diff --git a/GenOnlineService/AppLog.cs b/GenOnlineService/AppLog.cs new file mode 100644 index 0000000..dfc73ba --- /dev/null +++ b/GenOnlineService/AppLog.cs @@ -0,0 +1,73 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU Affero General Public License for more details. +** +** You should have received a copy of the GNU Affero General Public License +** along with this program. If not, see . +*/ + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace GenOnlineService +{ + // Logger for types without constructor injection. + public static class AppLog + { + public static ILogger For(Type category) + { + return new DeferredLogger(category.FullName ?? category.Name); + } + + public static ILogger For() + { + return For(typeof(T)); + } + + // static fields can initialize before ServiceLocator.Services is set, so resolve on first use + private sealed class DeferredLogger : ILogger + { + private readonly string _category; + private ILogger? _logger; + + public DeferredLogger(string category) + { + _category = category; + } + + private ILogger Inner + { + get + { + if (_logger != null) + { + return _logger; + } + + ILoggerFactory? factory = ServiceLocator.Services?.GetService(); + return factory == null ? NullLogger.Instance : (_logger = factory.CreateLogger(_category)); + } + } + + public IDisposable? BeginScope(TState state) where TState : notnull => Inner.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => Inner.IsEnabled(logLevel); + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Inner.Log(logLevel, eventId, state, exception, formatter); + } + } + } +} diff --git a/GenOnlineService/AppMetrics.cs b/GenOnlineService/AppMetrics.cs new file mode 100644 index 0000000..4baefbf --- /dev/null +++ b/GenOnlineService/AppMetrics.cs @@ -0,0 +1,213 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +*/ + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace GenOnlineService +{ + public static class AppMetrics + { + public const string MeterName = "GenOnlineService"; + public const string ActivitySourceName = MeterName; + public const string MySqlActivitySourceName = "MySqlConnector"; + public const string MySqlMeterName = "MySqlConnector"; + + private static readonly Meter s_meter = new(MeterName); + private static readonly ActivitySource s_activitySource = new(ActivitySourceName); + + private static int s_onlinePlayers; + private static int s_activeLobbies; + private static int s_webSocketSendQueueDepth; + private static double s_externalPublicationBacklogAgeSeconds; + private static readonly ConcurrentDictionary s_webSocketConnections = new(); + private static readonly ConcurrentDictionary s_matchmakingQueuedPlayers = new(); + private static readonly ConcurrentDictionary s_backgroundJobLastSuccess = new(); + + private static readonly Counter s_authenticationAttempts = + s_meter.CreateCounter("genonline.authentication.attempts", "{attempt}", "Authentication attempts by flow and outcome"); + private static readonly Counter s_webSocketFailures = + s_meter.CreateCounter("genonline.websocket.failures", "{message}", "Inbound or outbound WebSocket messages that failed"); + private static readonly Histogram s_matchmakingWaitDuration = + s_meter.CreateHistogram("genonline.matchmaking.wait.duration", "s", "Time in the matchmaking queue by exit outcome"); + private static readonly Counter s_matchmakingMatches = + s_meter.CreateCounter("genonline.matchmaking.matches", "{match}", "Matchmaking matches started or aborted during setup"); + private static readonly Counter s_backgroundJobFailures = + s_meter.CreateCounter("genonline.background_job.failures", "{run}", "Failed background job runs"); + private static readonly Counter s_externalPublicationFailures = + s_meter.CreateCounter("genonline.external_publication.failures", "{attempt}", "Failed external leaderboard publication attempts"); + private static readonly Counter s_securityActions = + s_meter.CreateCounter("genonline.security.actions", "{action}", "Rate limit rejections, moderation and anti-cheat actions"); + + static AppMetrics() + { + s_meter.CreateObservableGauge("genonline.players.online", () => Volatile.Read(ref s_onlinePlayers), "{player}", "Players currently online"); + s_meter.CreateObservableGauge("genonline.lobbies.active", () => Volatile.Read(ref s_activeLobbies), "{lobby}", "Lobbies currently active"); + s_meter.CreateObservableGauge("genonline.websocket.connections.active", + () => s_webSocketConnections.Select(entry => new Measurement(entry.Value, new KeyValuePair("session.type", entry.Key.ToMetricLabel()))), + "{connection}", "Open WebSocket connections by session type"); + s_meter.CreateObservableGauge("genonline.websocket.send_queue.depth", () => Volatile.Read(ref s_webSocketSendQueueDepth), "{message}", "Messages waiting for WebSocket delivery"); + s_meter.CreateObservableGauge("genonline.matchmaking.queue.players", + () => s_matchmakingQueuedPlayers.Select(entry => new Measurement(entry.Value, new KeyValuePair("playlist", entry.Key))), + "{player}", "Players in the matchmaking queue by playlist"); + s_meter.CreateObservableGauge("genonline.background_job.last_success.time", + () => s_backgroundJobLastSuccess.Select(entry => new Measurement(entry.Value, new KeyValuePair("job", entry.Key))), + "s", "Unix time of the last successful run by job"); + s_meter.CreateObservableGauge("genonline.external_publication.backlog.age", () => Volatile.Read(ref s_externalPublicationBacklogAgeSeconds), "s", "Age of the oldest unpublished match"); + } + + public static void RecordServiceSnapshot(int activeLobbies, int onlinePlayers) + { + Volatile.Write(ref s_activeLobbies, activeLobbies); + Volatile.Write(ref s_onlinePlayers, onlinePlayers); + } + + public static void RecordAuthentication(string flow, string source, string outcome) + { + s_authenticationAttempts.Add(1, new TagList { { "flow", flow }, { "source", source }, { "outcome", outcome } }); + } + + // records one attempt on dispose, with the outcome known at that point + public static IDisposable TrackAuthentication(string flow, string source, Func getOutcome) + { + return new DeferredRecord(() => RecordAuthentication(flow, source, getOutcome())); + } + + public static void RecordWebSocketConnections(EUserSessionType sessionType, int connections) + { + s_webSocketConnections[sessionType] = connections; + } + + public static void RecordWebSocketFailure(string direction, string reason) + { + s_webSocketFailures.Add(1, new TagList { { "direction", direction }, { "reason", reason } }); + } + + public static void AdjustWebSocketSendQueueDepth(int delta) + { + if (Interlocked.Add(ref s_webSocketSendQueueDepth, delta) < 0) + { + Interlocked.Exchange(ref s_webSocketSendQueueDepth, 0); + } + } + + public static void RecordMatchmakingQueue(string playlist, int players) + { + s_matchmakingQueuedPlayers[playlist] = players; + } + + public static void RecordMatchmakingQueueExit(string playlist, string outcome, TimeSpan waited) + { + s_matchmakingWaitDuration.Record(Math.Max(0, waited.TotalSeconds), new TagList { { "playlist", playlist }, { "outcome", outcome } }); + } + + public static void RecordMatchmakingMatch(string playlist, string outcome) + { + s_matchmakingMatches.Add(1, new TagList { { "playlist", playlist }, { "outcome", outcome } }); + } + + public static void RecordBackgroundJob(string job, bool succeeded) + { + if (succeeded) + { + s_backgroundJobLastSuccess[job] = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + } + else + { + s_backgroundJobFailures.Add(1, new TagList { { "job", job } }); + } + } + + public static void RecordExternalPublicationFailure(string outcome) + { + s_externalPublicationFailures.Add(1, new TagList { { "outcome", outcome } }); + } + + public static void RecordExternalPublicationBacklogAge(TimeSpan age) + { + Volatile.Write(ref s_externalPublicationBacklogAgeSeconds, Math.Max(0, age.TotalSeconds)); + } + + public static void RecordSecurityAction(string action, string outcome) + { + s_securityActions.Add(1, new TagList { { "action", action }, { "outcome", outcome } }); + } + + public static Activity? StartActivity( + string name, + ActivityKind kind = ActivityKind.Internal, + params KeyValuePair[] attributes) + { + Activity? activity = s_activitySource.StartActivity(name, kind); + if (activity != null) + { + foreach (KeyValuePair attribute in attributes) + { + activity.SetTag(attribute.Key, attribute.Value); + } + } + return activity; + } + + public static void RecordException(Activity? activity, Exception exception) + { + if (activity == null) + { + return; + } + + activity.SetStatus(ActivityStatusCode.Error, exception.Message); + activity.AddEvent(new ActivityEvent( + "exception", + tags: new ActivityTagsCollection + { + { "exception.type", exception.GetType().FullName }, + { "exception.message", exception.Message }, + { "exception.stacktrace", exception.ToString() } + })); + } + + private sealed class DeferredRecord : IDisposable + { + private Action? _record; + + public DeferredRecord(Action record) + { + _record = record; + } + + public void Dispose() + { + try + { + Interlocked.Exchange(ref _record, null)?.Invoke(); + } + catch + { + // Telemetry is fail-open. + } + } + } + } + // Lower-case enum labels for metrics and spans, cached to avoid per-call allocations. + public static class MetricLabels + { + private static class Cache where T : struct, Enum + { + public static readonly ConcurrentDictionary Names = new(); + } + + public static string ToMetricLabel(this T value) where T : struct, Enum + { + return Cache.Names.GetOrAdd(value, static v => v.ToString().ToLowerInvariant()); + } + } +} diff --git a/GenOnlineService/Constants.cs b/GenOnlineService/Constants.cs index d717c00..1aed4d2 100644 --- a/GenOnlineService/Constants.cs +++ b/GenOnlineService/Constants.cs @@ -22,6 +22,7 @@ using Microsoft.EntityFrameworkCore; using System; using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Globalization; using System.Net; @@ -373,17 +374,33 @@ public enum EKnownClients // TODO static class WebSocketManager { + private static readonly ILogger s_log = AppLog.For(typeof(WebSocketManager)); + public static int g_PeakConnectionCount = 0; - public static async Task CreateSession(AppDbContext _db, EUserSessionType sessionType, bool bIsReconnect, Int64 ownerID, KnownClients.EKnownClients client_id, string ipAddr, string strContinent, string strCountry, double dLatitude, double dLongitude, bool bIsAdmin) + + private static void RecordConnectionSnapshot() { - string strDisplayName = await Database.Users.GetDisplayName(_db, ownerID); + foreach (var entry in m_dictWebsockets) + { + AppMetrics.RecordWebSocketConnections(entry.Key, entry.Value.Count); + } + } + public static Task CreateSession(AppDbContext _db, EUserSessionType sessionType, bool bIsReconnect, Int64 ownerID, KnownClients.EKnownClients client_id, string ipAddr, string strContinent, string strCountry, double dLatitude, double dLongitude, bool bIsAdmin) + { + return ServiceLocator.Services.GetRequiredService().OpenSessionAsync(_db, sessionType, bIsReconnect, ownerID, client_id, ipAddr, strContinent, strCountry, dLatitude, dLongitude, bIsAdmin); + } + + internal static async Task InstallSession(AppDbContext _db, EUserSessionType sessionType, bool bIsReconnect, Int64 ownerID, KnownClients.EKnownClients client_id, string ipAddr, string strContinent, string strCountry, double dLatitude, double dLongitude, bool bIsAdmin) + { + string strDisplayName = await Database.Users.GetDisplayName(_db, ownerID); + await Database.Users.UpdateLastLoginData(_db, ownerID, ipAddr); // if we have cache data, that means its a reconnect, noraml connections go through login flows which reset cache data UserSession? userCacheData = WebSocketManager.GetSessionFromUser(ownerID, sessionType); if (bIsReconnect) { // this is a reconnect, re-use cache - Console.WriteLine("--> WEBSOCKET RECONNECT"); + s_log.LogDebug("WebSocket reconnect for user {UserId} ({SessionType})", ownerID, sessionType); // if its a reconnect, and we dont have cache OR shared data, its probably a server restart, so return null if (userCacheData == null) @@ -425,18 +442,11 @@ public static async Task CreateSession(AppDbContext _db, } else { - Console.WriteLine("--> WEBSOCKET CONNECT"); + s_log.LogDebug("WebSocket connect for user {UserId} ({SessionType})", ownerID, sessionType); // how many other sessions do they have online? bool bIsFirstSessionForUser = WebSocketManager.GetAllDataFromUser(ownerID).Count == 0; - // kill any existing sessions for this user of same session type - if (m_dictWebsockets[sessionType].TryGetValue(ownerID, out UserWebSocketInstance? existingSession)) - { - Console.WriteLine("Killing existing session for {0} ({1})", ownerID, strDisplayName); - await DeleteSession(ownerID, sessionType, existingSession, !bIsReconnect); - } - // get and cache social container UserSocialContainer socialContainer = new(); socialContainer.Friends = await Database.Social.GetFriends(_db, ownerID); @@ -483,15 +493,14 @@ public static async Task CreateSession(AppDbContext _db, // close the previous socket first: it may still be open, and its send loop must stop reading the session's queue if (m_dictWebsockets[sessionType].TryRemove(ownerID, out UserWebSocketInstance? supersededSess) && supersededSess != null) { - Console.WriteLine("Closing superseded websocket for {0} ({1})", ownerID, strDisplayName); + s_log.LogDebug("Closing superseded websocket for {UserId} ({DisplayName})", ownerID, strDisplayName); await supersededSess.CloseAsync(WebSocketCloseStatus.NormalClosure, "Superseded by a newer connection"); } UserWebSocketInstance newSess = new UserWebSocketInstance(sessionType, ownerID, userCacheData); m_dictWebsockets[sessionType][ownerID] = newSess; + RecordConnectionSnapshot(); - // update last login and last ip - await Database.Users.UpdateLastLoginData(_db, ownerID, ipAddr); // TODO_EFCORE: Optimize this, dont iterate all the time int numSessions = WebSocketManager.GetNumberOfUsersOnline(); @@ -588,149 +597,51 @@ public static async Task CheckForTimeouts() } } - foreach (var userData in lstCacheEntriesToDestroy) - { - // A reconnect between the snapshot above and now would have replaced this user's - // session with a live one; only clear if the SAME session object is still registered - // and still abandoned+expired, so a fresh reconnect never has its live session torn - // down (kicked from its lobby, deregistered from matchmaking) by a stale sweep entry. - UserSession? currentSession = GetSessionFromUser(userData.UserID, userData.SessionType); - if (!ShouldStillClearAbandonedSession(currentSession, userData.Session)) - { - continue; - } - - await ClearDataFromUser(userData.UserID, userData.SessionType); - } - } - - public static async Task DeleteSession(Int64 user_id, EUserSessionType sessionType, UserWebSocketInstance? oldWS, bool bShouldInvalidatePlayerCacheToBlockReconnect) - { - UserSession? sourceData = WebSocketManager.GetSessionFromUser(user_id, sessionType); - SharedUserData? sourceSharedData = WebSocketManager.GetSharedDataForUser(user_id); - - bool bIsStaleSocket = false; - - if (oldWS != null) - { - // has this websocket already been superseded by a newer connection for the same user? if so it is a - // stale/zombie socket and must not be allowed to abandon or tear down the live session - if (m_dictWebsockets[sessionType].TryGetValue(user_id, out UserWebSocketInstance? currentWS)) - { - bIsStaleSocket = !ReferenceEquals(currentWS, oldWS); - } - - try - { - // dont remove by ID, user could have re-opened another websocket open via reconnection, remove by instance, if its not there, thats OK, it was already closed and the new instance is a reconnect - var item = m_dictWebsockets[sessionType].First(kvp => kvp.Value == oldWS); // safe to lookup by sessionType here since we only ever remove old WS of the same type - m_dictWebsockets[sessionType].Remove(item.Key, out UserWebSocketInstance? destroyedSess); - - if (destroyedSess != null) - { - await destroyedSess.CloseAsync(WebSocketCloseStatus.NormalClosure, "User signed in from another point of presence [A]"); - } - } - catch - { - - } - - if (bIsStaleSocket) - { - // nothing else to do - the user is still connected on the newer socket - await oldWS.CloseAsync(WebSocketCloseStatus.NormalClosure, "Superseded by a newer connection"); - return; - } - } - - if (bShouldInvalidatePlayerCacheToBlockReconnect) - { - // NOTE: ClearDataFromUser owns the shared data ref count decrement (and GC) for this path - doing it - // again here would drop the count twice for a single disconnect and tear down the shared data while - // the user still has other live sessions - await WebSocketManager.ClearDataFromUser(user_id, sessionType); - } - else - { - // mark it as abandoned for now to start the expiration timer - if (sourceData != null) - { - sourceData.MarkAbandoned(); - - // If the player was in an active game when their connection dropped, record the - // abandon time NOW (before any lobby-structure cleanup runs). This timestamp is - // the authoritative "who quit first" signal used by DetermineLobbyWinnerIfNotPresent, - // and must be captured here rather than in RemoveMember, because RemoveMember may - // execute much later (e.g. 30 s after the first abandonment) or at an unexpected - // time (e.g. when the other player reconnects with a fresh session and their old - // session is forcibly evicted, causing them to appear as "last to leave" even - // though they stayed in the game longer than the opponent who quit first). - try - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - if (sourceData.currentLobbyID != -1) - { - Lobby? ingameLobby = lobbyManager.GetLobby(sourceData.currentLobbyID); - if (ingameLobby != null && ingameLobby.State == ELobbyState.INGAME) - { - ingameLobby.RecordPlayerIngameAbandon(user_id); - } - else - { - Console.WriteLine($"[WARN] DeleteSession(false): skipped RecordPlayerIngameAbandon for user {user_id} — lobby={sourceData.currentLobbyID} found={ingameLobby != null} state={ingameLobby?.State}"); - } - } - else - { - Console.WriteLine($"[WARN] DeleteSession(false): skipped RecordPlayerIngameAbandon for user {user_id} — currentLobbyID==-1 (session has no lobby)"); - } - } - catch (Exception ex) - { - Console.WriteLine($"[WARN] Failed to record in-game abandon for user {user_id}: {ex.Message}"); - } - } - } - - // NOTE: They only went offline if ref count became 0, otherwise they're still online somewhere else - if (sourceData != null && sourceSharedData != null && sourceSharedData.NeedsGC()) - { - // TODO_SOCIAL: Move this to a class - // inform any friends who are online that this person just came online - WebSocketMessage_Social_FriendStatusChanged friendStatusChangedEvent = new(); - friendStatusChangedEvent.msg_id = (int)EWebSocketMessageID.SOCIAL_FRIEND_ONLINE_STATUS_CHANGED; - friendStatusChangedEvent.display_name = sourceSharedData.m_strDisplayName; - friendStatusChangedEvent.online = false; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(friendStatusChangedEvent)); - - if (sourceData != null) - { - // friends are reciprocal so we can just iterate our friends - foreach (Int64 friendID in sourceSharedData.GetSocialContainer().Friends) - { - WebsocketHelper.SendToAllSessionsOfUser(friendID, bytesJSON); - } - } - } - - int numSessions = WebSocketManager.GetNumberOfUsersOnline(); ; - Console.Title = String.Format("GenOnline - {0} players", numSessions); - - try - { - if (oldWS != null) - { - // Close the WS directly, dont rely on session data as it may be linked to something else at this point - await oldWS.CloseAsync(WebSocketCloseStatus.NormalClosure, "Session being deleted"); - } - } - catch - { - - } - } - + foreach (var userData in lstCacheEntriesToDestroy) + await ServiceLocator.Services.GetRequiredService().ExpireSessionAsync(userData.Session); + } + + public static Task DeleteSession(Int64 userID, EUserSessionType sessionType, UserWebSocketInstance? oldWS, bool destroySession, UserSession? expectedSession = null) + { + var session = expectedSession ?? oldWS?.OwnerSession ?? GetSessionFromUser(userID, sessionType); + return ServiceLocator.Services.GetRequiredService().DisconnectSessionAsync(session, oldWS, destroySession); + } + + internal static UserWebSocketInstance? DetachSocket(UserSession session) + { + var sockets = m_dictWebsockets[session.GetSessionType()]; + if (!sockets.TryGetValue(session.m_UserID, out var socket) || !ReferenceEquals(socket.OwnerSession, session)) return null; + if (!sockets.TryRemove(new KeyValuePair(session.m_UserID, socket))) return null; + RecordConnectionSnapshot(); + return socket; + } + + internal static bool RemoveSessionData(UserSession session) + { + if (!m_dictUserSessions[session.GetSessionType()].TryRemove(new KeyValuePair(session.m_UserID, session))) + return false; + session.ReleasePendingWebsocketSendAccounting(); + if (m_dictSharedUserData.TryGetValue(session.m_UserID, out var shared)) + { + shared.DecrementRefCount(); + if (shared.NeedsGC()) + { + m_dictSharedUserData.TryRemove(new KeyValuePair(session.m_UserID, shared)); + var message = new WebSocketMessage_Social_FriendStatusChanged + { + msg_id = (int)EWebSocketMessageID.SOCIAL_FRIEND_ONLINE_STATUS_CHANGED, + display_name = shared.m_strDisplayName, + online = false + }; + byte[] payload = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message)); + foreach (long friendID in shared.GetSocialContainer().Friends) + WebsocketHelper.SendToAllSessionsOfUser(friendID, payload); + } + } + Console.Title = $"GenOnline - {GetNumberOfUsersOnline()} players"; + return true; + } + /* public static ChatSession? GetWebSocketForUser(Int64 userID) { @@ -748,7 +659,7 @@ public static async Task DeleteSession(Int64 user_id, EUserSessionType sessionTy // TODO_EFCORE: Just use a weakref to the websocket from the user session instead of lookups public static UserWebSocketInstance? GetWebSocketForSession(UserSession session) { - if (m_dictWebsockets[session.GetSessionType()].TryGetValue(session.m_UserID, out UserWebSocketInstance? retVal)) + if (m_dictWebsockets[session.GetSessionType()].TryGetValue(session.m_UserID, out UserWebSocketInstance? retVal) && ReferenceEquals(retVal.OwnerSession, session)) { return retVal; } @@ -851,52 +762,22 @@ public static async Task DisconnectUser(Int64 userID, byte[] finalMessage) await userSession.FlushWebsocketSendsAsync(TimeSpan.FromSeconds(2)); } - await DeleteSession(userID, userSession.GetSessionType(), oldWS, true); + await DeleteSession(userID, userSession.GetSessionType(), oldWS, true, userSession); } catch (Exception ex) { - Console.WriteLine($"[ERROR] DisconnectUser failed for user {userID}, session {userSession.GetSessionType()}: {ex.Message}"); - SentrySdk.CaptureException(ex); - } - } - } - - public static async Task ClearDataFromUser(Int64 userID, EUserSessionType sessionType) - { - // NOTE: This is when a player is truly disconnected and we can destroy session, remove form lobby etc, websocket disconnect doesnt mean that because the clietn reconnects - try - { - UserSession? userData = null; - - if (m_dictUserSessions[sessionType].ContainsKey(userID)) - { - userData = m_dictUserSessions[sessionType][userID]; + s_log.LogError(ex, "DisconnectUser failed for user {UserId}, session {SessionType}", userID, userSession.GetSessionType()); } - await SessionHelpers.FullyDestroyPlayerSession(userID, userData, true); - - // decrement ref count on shared data - if (m_dictSharedUserData.TryGetValue(userID, out SharedUserData? sharedData)) - { - sharedData.DecrementRefCount(); - if (sharedData.NeedsGC()) // cleanup if necessary - { - m_dictSharedUserData.Remove(userID, out var removedSharedData); - } - } - else - { - Console.WriteLine("Error: Could not find shared data for user {0} when deleting session", userID); - } - } - catch - { - } - - return m_dictUserSessions[sessionType].Remove(userID, out var itemRemoved); } - + public static Task ClearDataFromUser(Int64 userID, EUserSessionType sessionType) + { + var session = GetSessionFromUser(userID, sessionType); + return ServiceLocator.Services.GetRequiredService().ClearSessionAsync(session); + } + + // helpers private static readonly object g_dirtyLobbyRoomsLock = new(); private static readonly HashSet g_dirtyLobbyRooms = new(); @@ -1175,6 +1056,12 @@ private static Int64 Get(string key, Int64 defaultValue) public class UserSession { + private static readonly ILogger s_log = AppLog.For(typeof(UserSession)); + + private int retired; + internal bool IsRetired => Volatile.Read(ref retired) != 0; + internal void Retire() => Interlocked.Exchange(ref retired, 1); + public Int64 m_UserID = -1; public string m_strContinent; @@ -1188,6 +1075,7 @@ public class UserSession // Matchmaking data public UInt16 MatchmakingPlaylistID = 0; + public long MatchmakingStartedTimestamp = 0; public ConcurrentList MatchmakingMapIndicies = new(); internal bool IsRegisteredForMatchmaking { get; set; } = false; internal SemaphoreSlim MatchmakingStateLock { get; } = new(1, 1); @@ -1286,12 +1174,17 @@ public void QueueWebsocketSend(byte[] bytesJSON) return; } - if (!m_outbound.Writer.TryWrite(bytesJSON)) + if (m_outbound.Writer.TryWrite(bytesJSON)) { + AppMetrics.AdjustWebSocketSendQueueDepth(1); + } + else + { + AppMetrics.RecordWebSocketFailure("outbound", "queue_full"); // too far behind to catch up: drop the connection so the client reconnects if (WebSocketManager.GetWebSocketForSession(this)?.AbortIfOpen() == true) { - Console.WriteLine("[WebSocket] Outbound queue full for {0}, connection aborted", m_UserID); + s_log.LogWarning("Outbound websocket queue full for user {UserId}; connection aborted", m_UserID); } } } @@ -1326,6 +1219,16 @@ public async Task CloseWebsocket(WebSocketCloseStatus rea public ChannelReader OutboundWebsocketSends => m_outbound.Reader; + // the session is gone: take its unsent messages off the queue-depth gauge + public void ReleasePendingWebsocketSendAccounting() + { + int pending = m_outbound.Reader.Count; + if (pending > 0) + { + AppMetrics.AdjustWebSocketSendQueueDepth(-pending); + } + } + public bool NeedsCleanup() { // Grace period an abandoned (no live websocket) session gets before it's torn down, @@ -1441,7 +1344,8 @@ public bool TryUpdateSessionNetworkRoom(Int16 newRoomID) { if (!RoomCatalog.TryResolveTargetRoomID(newRoomID, out Int16 targetRoomID)) { - Console.WriteLine($"Rejected invalid network room selection {newRoomID} from user {m_UserID}; keeping selected room {selectedNetworkRoomID}."); + s_log.LogWarning("Rejected invalid network room selection {RequestedRoomId} from user {UserId}, keeping selected room {SelectedRoomId}", + newRoomID, m_UserID, selectedNetworkRoomID); return false; } @@ -1479,6 +1383,8 @@ public void UpdateSessionLobbyID(Int64 newLobbyID) public class UserWebSocketInstance { + private static readonly ILogger s_log = AppLog.For(typeof(UserWebSocketInstance)); + // cached user data, useful public EUserSessionType m_SessionType = EUserSessionType.None; public Int64 m_UserID = -1; @@ -1513,7 +1419,11 @@ public void QueueKeepAlive() } private WebSocket? m_SockInternal = null; - private readonly UserSession m_OwnerSession; + private readonly object transportLock = new(); + private bool closed; + private readonly UserSession m_OwnerSession; + internal UserSession OwnerSession => m_OwnerSession; + internal bool IsOpen => m_SockInternal?.State == WebSocketState.Open; private readonly CancellationTokenSource m_sendLoopStop = new CancellationTokenSource(); private Task m_sendLoop = Task.CompletedTask; @@ -1524,12 +1434,20 @@ public UserWebSocketInstance(EUserSessionType sessionType, Int64 ownerID, UserSe m_OwnerSession = ownerSession; } - public void AttachWebsocket(WebSocket sock) - { - m_SockInternal = sock; - m_sendLoop = Task.Run(RunSendLoop); - } - + public void AttachWebsocket(WebSocket sock) + { + lock (transportLock) + { + if (closed) + { + sock.Abort(); + return; + } + m_SockInternal = sock; + m_sendLoop = Task.Run(RunSendLoop); + } + } + public bool AbortIfOpen() { WebSocket? sock = m_SockInternal; @@ -1558,7 +1476,10 @@ private async Task RunSendLoop() return; } - outbound.TryRead(out _); + if (outbound.TryRead(out _)) + { + AppMetrics.AdjustWebSocketSendQueueDepth(-1); + } } } } @@ -1567,7 +1488,7 @@ private async Task RunSendLoop() } catch (Exception ex) { - Console.WriteLine($"[WebSocket] Send loop failed for {m_UserID}: {ex}"); + s_log.LogError(ex, "Websocket send loop failed for user {UserId}", m_UserID); } } @@ -1587,17 +1508,20 @@ private async Task SendFrameAsync(byte[] buffer) } catch (OperationCanceledException) { - Console.WriteLine("[WebSocket] Send to {0} stalled for {1}s, connection aborted", m_UserID, c_SendStallTimeout.TotalSeconds); + AppMetrics.RecordWebSocketFailure("outbound", "timeout"); + s_log.LogWarning("Websocket send to user {UserId} stalled for {StallSeconds}s; connection aborted", m_UserID, c_SendStallTimeout.TotalSeconds); return false; } catch { + AppMetrics.RecordWebSocketFailure("outbound", "error"); return false; } } public async Task CloseAsync(WebSocketCloseStatus closeStatus, string? statusDescription) { + lock (transportLock) closed = true; // stop reading the queue first so the next connection's send loop is the only reader m_sendLoopStop.Cancel(); @@ -1630,6 +1554,8 @@ public enum ESessionAccessType }; public static class SessionHelpers { + private static readonly ILogger s_log = AppLog.For(typeof(SessionHelpers)); + public static bool SessionTypeHasAccessTo(EUserSessionType sessType, ESessionAccessType accessType) { if (sessType == EUserSessionType.GameClient) // client can do anything @@ -1647,61 +1573,23 @@ public static bool SessionTypeHasAccessTo(EUserSessionType sessType, ESessionAcc return false; } - public static async Task FullyDestroyPlayerSession(Int64 user_id, UserSession? userData, bool bMigrateLobbyIfPresent) - { - // NOTE: Dont assume userData is valid, use user_id for user id - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine("FullyDestroyPlayerSession for user {0}", user_id); - Console.ForegroundColor = ConsoleColor.Gray; - - // invalidate any TURN credentials - TURNCredentialManager.DeleteCredentialsForUser(user_id); - - // TODO: Implement single point of presence? gets dicey if multiple logins - // TODO: Dont destroy this, just mark inactive/offline, we use this as a saved credential system - - // session tied to this token (keep other ones attached to user_id, could be other machines) - // TODO_JWT: Remove table fully + set logged out - //await m_Inst.Query("DELETE FROM sessions WHERE user_id={0} AND session_type={1};", user_id, (int)ESessionType.Game); - - // leave any lobby - Console.WriteLine("[Source 2] User {0} Leave Any Lobby", user_id); - - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - await lobbyManager.LeaveAnyLobby(user_id); - - await lobbyManager.CleanupUserLobbiesNotStarted(user_id); - - // remove from any matchmaking - if (userData != null) - { - await MatchmakingManager.DeregisterPlayer(userData); - } - - // TODO: Client needs to handle this... itll start returning 404 - } - public async static Task SetUsedLoggedIn(Int64 userID, KnownClients.EKnownClients clientID, EUserSessionType sessionType) { // TODO_EFCORE: website uses this index as 1 (60hz) to 0 (30hz), update it to use new enum + support new clients, also need to update DB to match // TODO_EFCORE: Move away from db for this and just have website login call endpoint on service //UInt16 clientID = clientIDStr == "gen_online_60hz" ? (UInt16)1 : (UInt16)0; - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine("StartSession deleing other sessions for user {0}", userID); - Console.ForegroundColor = ConsoleColor.Gray; + s_log.LogInformation("StartSession deleting other sessions for user {UserId}", userID); // kill any WS they had too, StartSession comes before WS connects // disconnect any other sessions with this ID UserSession? sess = GenOnlineService.WebSocketManager.GetSessionFromUser(userID, sessionType); if (sess != null) { - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine("Found duplicate session for user {0}", userID); - Console.ForegroundColor = ConsoleColor.Gray; + s_log.LogInformation("Found duplicate session for user {UserId}", userID); UserWebSocketInstance? oldWS = GenOnlineService.WebSocketManager.GetWebSocketForSession(sess); - await GenOnlineService.WebSocketManager.DeleteSession(userID, sessionType, oldWS, false); + await GenOnlineService.WebSocketManager.DeleteSession(userID, sessionType, oldWS, false, sess); } } } @@ -2397,6 +2285,8 @@ public class TURNResponse public static class S3CredentialManager { + private static readonly ILogger s_log = AppLog.For(typeof(S3CredentialManager)); + private static AmazonS3Client m_s3client = null; private static bool s_uploadsEnabled = false; @@ -2412,7 +2302,7 @@ public static void Initialize() // When uploads are disabled, the S3 settings are intentionally optional. if (!s_uploadsEnabled) { - Console.WriteLine("MatchData: upload_match_data is false, replay and screenshot uploads are disabled."); + s_log.LogInformation("MatchData: upload_match_data is false, replay and screenshot uploads are disabled"); return; } @@ -2567,6 +2457,8 @@ private static void GetS3Config(out int TTL, out string access_key, out string s public static class TURNCredentialManager { + private static readonly ILogger s_log = AppLog.For(typeof(TURNCredentialManager)); + private static ConcurrentDictionary g_DictTURNUsernames = new(); private static void GetTURNConfig(out int TTL, out string token, out string key, out bool bShouldInvalidateTokensAutomatically) @@ -2683,13 +2575,13 @@ private static void GetTURNConfig(out int TTL, out string token, out string key, //client.DefaultRequestHeaders.Add("Content-Type", "application/json"); try { - Console.WriteLine("Start req turn credentials at {0}", Environment.TickCount); + var sw = Stopwatch.StartNew(); string strURI = String.Format("https://rtc.live.cloudflare.com/v1/turn/keys/{0}/credentials/generate-ice-servers", TurnKey); HttpResponseMessage response = await client.PostAsync(strURI, requestContent); if (response.IsSuccessStatusCode) { - Console.WriteLine("Finish req turn credentials at {0}", Environment.TickCount); + s_log.LogDebug("TURN credential request for user {UserId} took {ElapsedMs}ms", userID, sw.ElapsedMilliseconds); string responseBody = await response.Content.ReadAsStringAsync(); TURNResponse? resp = JsonSerializer.Deserialize(responseBody); @@ -2795,7 +2687,8 @@ public static async Task DeleteCredentialsForUser(Int64 userID) try { string strURI = String.Format("https://rtc.live.cloudflare.com/v1/turn/keys/{0}/credentials/{1}/revoke", TurnKey, strTURNUsername); - HttpResponseMessage response = await client.PostAsync(strURI, requestContent); + using HttpResponseMessage response = await client.PostAsync(strURI, requestContent); + response.EnsureSuccessStatusCode(); } catch { @@ -3025,6 +2918,9 @@ public class WebSocketMessage_Social_NewFriendRequest : WebSocketMessage public class WebSocketMessage_FullMeshConnectivityCheckOutcome: WebSocketMessage { + public long mesh_check_id { get; set; } + public int attempt { get; set; } + public string reason { get; set; } = string.Empty; public bool mesh_complete { get; set; } public List missing_connections { get; set; } = new(); } diff --git a/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs b/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs index 24954bb..1595664 100644 --- a/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs +++ b/GenOnlineService/Controllers/CheckLogin/CheckLoginController.cs @@ -77,6 +77,10 @@ public async Task Post( public async Task Post_InternalHandler(string jsonData, string ipAddr, string webSocketUrl, bool bIsMonitor = false) { POST_CheckLogin_Result result = new POST_CheckLogin_Result(); + using IDisposable authenticationMetric = AppMetrics.TrackAuthentication( + "check_login", + bIsMonitor ? "monitor" : "client", + () => result.result.ToMetricLabel()); // Must have an IP... if (bIsMonitor) diff --git a/GenOnlineService/Controllers/Lobbies/LobbiesController.cs b/GenOnlineService/Controllers/Lobbies/LobbiesController.cs index 167eb72..65f9323 100644 --- a/GenOnlineService/Controllers/Lobbies/LobbiesController.cs +++ b/GenOnlineService/Controllers/Lobbies/LobbiesController.cs @@ -328,8 +328,6 @@ public async Task Put() if (playerSession != null) { - // cleanup any zombie lobbies - await _lobbyManager.CleanupUserLobbiesNotStarted(user_id); await using var db = await _dbFactory.CreateDbContextAsync(); string strDisplayName = await Database.Users.GetDisplayName(db, user_id); @@ -342,14 +340,9 @@ public async Task Put() result.result = 1; result.lobby_id = newLobbyID; - // TODO: What if this fails? just let them proceed? just means only direct connect people will be able to play - // get some turn credentials - TURNCredentialContainer? turnCredentials = await TURNCredentialManager.CreateCredentialsForUser(user_id); - if (turnCredentials != null) - { - result.turn_username = turnCredentials.m_strUsername; - result.turn_token = turnCredentials.m_strToken; - } + var credentials = await _lobbyManager.CreateLobbyRelayCredentialsAsync(playerSession, newLobbyID); + result.turn_username = credentials.Username; + result.turn_token = credentials.Token; } else { diff --git a/GenOnlineService/Controllers/Lobby/LobbyController.cs b/GenOnlineService/Controllers/Lobby/LobbyController.cs index 336e93b..db7a085 100644 --- a/GenOnlineService/Controllers/Lobby/LobbyController.cs +++ b/GenOnlineService/Controllers/Lobby/LobbyController.cs @@ -211,65 +211,18 @@ public async Task Get(string lobby_id) [Authorize(Roles = "GameClient")] public async Task Delete(Int64 lobbyID) { - RouteHandler_DELETE_Lobby_Result result = new RouteHandler_DELETE_Lobby_Result(); - - using (var reader = new StreamReader(HttpContext.Request.Body)) + var result = new RouteHandler_DELETE_Lobby_Result(); + using var reader = new StreamReader(HttpContext.Request.Body); + await reader.ReadToEndAsync(); + try { - string jsonData = await reader.ReadToEndAsync(); - var options = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; - - try - { - // TODO: Dont let them join more than 1 lobby - // need a lobby ID - int leavingPersonSlot = -1; - Int64 user_id = TokenHelper.GetUserID(this); - EUserSessionType sessionType = TokenHelper.GetSessionType(this); - if (user_id != -1 && SessionHelpers.SessionTypeHasAccessTo(sessionType, ESessionAccessType.Gameplay)) - { - Lobby? lobby = _lobbyManager.GetLobby(lobbyID); - if (lobby != null) - { - foreach (var member in lobby.Members) - { - if (member.SlotState == EPlayerType.SLOT_PLAYER) - { - if (member.UserID == user_id) - { - leavingPersonSlot = member.SlotIndex; - } - } - } - } - - Console.WriteLine("[Source 1] User {0} Leave Any Lobby", user_id); - await _lobbyManager.LeaveAnyLobby(user_id); - - // cleanup TURN credentials - TURNCredentialManager.DeleteCredentialsForUser(user_id); - - // clear our lobby ID - UserSession? sourceData = WebSocketManager.GetSessionFromUser(user_id, sessionType); - - if (sourceData != null) - { - sourceData.UpdateSessionLobbyID(-1); - // NOTE: We dont update the match history match ID here, that is done by the match history service - } - - result.success = true; - } - - } - catch - { - return result; - } + result.success = await _lobbyManager.LeaveAsync(new LeaveLobbyCommand(lobbyID, + TokenHelper.GetUserID(this), TokenHelper.GetSessionType(this))); } - + catch + { + return result; + } return result; } @@ -410,94 +363,57 @@ enum ELobbyUpdatePermissions [Authorize(Roles = "GameClient")] public async Task Post(Int64 lobbyID) { - RouteHandler_POST_Lobby_Result result = new RouteHandler_POST_Lobby_Result(); - - using (var reader = new StreamReader(HttpContext.Request.Body)) - { - string jsonData = await reader.ReadToEndAsync(); - var options = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; - - try - { - // TODO: Dont let them join more than 1 lobby - - var data = JsonSerializer.Deserialize>(jsonData, options); - - if (data != null - && data.ContainsKey("field") - ) - { - // TODO_NGMP: For all things which modify lobbies, we should check if the player is really in the lobby - // get requesting user - Int64 user_id = TokenHelper.GetUserID(this); - if (user_id != -1) - { - Lobby? lobby = _lobbyManager.GetLobby(lobbyID); - - if (lobby != null) - { - // check the user is in the lobby, otherwise bail - LobbyMember? SourceMember = lobby.GetMemberFromUserID(user_id); - - if (SourceMember == null) - { - Response.StatusCode = (int)HttpStatusCode.Unauthorized; - return result; - } - - if (!TryParseLobbyUpdateField(data["field"].GetInt32(), out ELobbyUpdateField field)) - { - Response.StatusCode = (int)HttpStatusCode.BadRequest; - return result; - } - - // check permissions - ELobbyUpdatePermissions updatePerms = g_dictLobbyUpdatePermissionsTable[field]; - - if (updatePerms == ELobbyUpdatePermissions.LobbyOwner) // check owner - { - if (user_id != lobby.Owner) - { - Response.StatusCode = (int)HttpStatusCode.Unauthorized; - result.success = false; - return result; - } - } - - // Every mutation below touches this lobby's Members, slot state, or ready state, - // so the whole field-update dispatch runs as one atomic operation under the - // per-lobby gate instead of each setter mutating state unguarded. - // A kick can't finish the actual removal in here: RemoveMember acquires this - // same lobby's gate, which is not reentrant, so ApplyLobbyFieldUpdateAsync only - // validates the target and hands the user ID back for PerformKickAsync to - // process once the gate below has been released. - Int64? kickedUserID = await lobby.RunExclusiveAsync(() => ApplyLobbyFieldUpdateAsync(lobby, SourceMember, field, data)); - - if (kickedUserID.HasValue) - { - await PerformKickAsync(lobby, kickedUserID.Value); - } - } - } - - } - - } - catch - { + var result = new RouteHandler_POST_Lobby_Result(); + using var reader = new StreamReader(HttpContext.Request.Body); + string jsonData = await reader.ReadToEndAsync(); + try + { + var data = JsonSerializer.Deserialize>(jsonData); + if (data == null || !data.ContainsKey("field")) return result; + long userID = TokenHelper.GetUserID(this); + if (userID == -1) return result; + var lobby = _lobbyManager.GetLobby(lobbyID); + if (lobby == null) return result; + + if (data["field"].GetInt32() == (int)ELobbyUpdateField.HOST_ACTION_KICK_USER) + { + long? targetID = data.TryGetValue("userid", out var target) && target.ValueKind == JsonValueKind.Number + && target.TryGetInt64(out long parsedTarget) ? parsedTarget : null; + var status = await _lobbyManager.KickAsync(new KickLobbyMemberCommand(lobbyID, userID, targetID)); + if (status is LobbyMembershipStatus.ActorNotMember or LobbyMembershipStatus.ActorNotHost) + Response.StatusCode = (int)HttpStatusCode.Unauthorized; + // Legacy kick responses leave success false, even when removal succeeds. return result; } + + await lobby.RunExclusiveAsync(async () => + { + var sourceMember = lobby.GetMemberFromUserID(userID); + if (sourceMember == null) + { + Response.StatusCode = (int)HttpStatusCode.Unauthorized; + return; + } + if (!TryParseLobbyUpdateField(data["field"].GetInt32(), out ELobbyUpdateField field)) + { + Response.StatusCode = (int)HttpStatusCode.BadRequest; + return; + } + if (g_dictLobbyUpdatePermissionsTable[field] == ELobbyUpdatePermissions.LobbyOwner && userID != lobby.Owner) + { + Response.StatusCode = (int)HttpStatusCode.Unauthorized; + return; + } + await ApplyLobbyFieldUpdateAsync(lobby, sourceMember, field, data); + }); } - + catch + { + return result; + } return result; } - // Rejects any wire value that isn't a real EPlayerType member, so a malformed or - // out-of-range slot_state from HOST_ACTION_SET_SLOT_STATE can't be cast into an enum value - // the rest of the lobby code (SetPlayerSlotState, IsAI, serialization, ...) never expects. internal static bool TryParseSlotState(UInt16 rawValue, out EPlayerType slotState) { slotState = (EPlayerType)rawValue; @@ -505,23 +421,14 @@ internal static bool TryParseSlotState(UInt16 rawValue, out EPlayerType slotStat return Enum.IsDefined(typeof(EPlayerType), slotState) && slotState != EPlayerType.SLOT_PLAYER; } - // Rejects any wire value that isn't a real ELobbyUpdateField member, so an unknown/malformed - // field can't be cast into an enum value that then gets indexed straight into - // g_dictLobbyUpdatePermissionsTable (which previously threw KeyNotFoundException, silently - // swallowed by Post's outer catch, for anything out of range). internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateField field) { field = (ELobbyUpdateField)rawValue; return Enum.IsDefined(typeof(ELobbyUpdateField), field); } - // Runs the per-field lobby update dispatch. Must only be called from inside - // lobby.RunExclusiveAsync: every branch here mutates this lobby's Members, slot - // state/fields, or ready state, and previously ran completely unguarded. - // Returns the kicked user's ID when HOST_ACTION_KICK_USER validated a real target, so the - // caller can run the actual removal after releasing the gate; null for every other field - // (including a kick request with no valid target). - private async Task ApplyLobbyFieldUpdateAsync(Lobby lobby, LobbyMember SourceMember, ELobbyUpdateField field, Dictionary data) + // Caller holds the lobby gate. + private async Task ApplyLobbyFieldUpdateAsync(Lobby lobby, LobbyMember SourceMember, ELobbyUpdateField field, Dictionary data) { // reset everyones ready states when anything changes (minus dummy actions) if (field != ELobbyUpdateField.HOST_ACTION_FORCE_START @@ -547,7 +454,7 @@ internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateFiel if ((strMap != null && strMap.Length > 255) || (strMapPath != null && strMapPath.Length > 512)) { Response.StatusCode = (int)HttpStatusCode.BadRequest; - return null; + return; } if (strMap != null && strMapPath != null) @@ -630,22 +537,6 @@ internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateFiel SourceMember.UpdateHasMap(bHasMap); } } - else if (field == ELobbyUpdateField.HOST_ACTION_KICK_USER) - { - if (data.ContainsKey("userid")) - { - Int64 KickedUserID = data["userid"].GetInt64(); - - // the target must actually be in THIS lobby, otherwise a host could wipe the - // TURN credentials / lobby state of any arbitrary online player. Checked here, - // under the gate, so it can't race a concurrent leave/kick of the same target; - // the actual removal runs after the gate is released (see PerformKickAsync). - if (lobby.GetMemberFromUserID(KickedUserID) != null) - { - return KickedUserID; - } - } - } else if (field == ELobbyUpdateField.HOST_ACTION_SET_SLOT_STATE) { UInt16 slot_index = data["slot_index"].GetUInt16(); @@ -653,7 +544,7 @@ internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateFiel if (!TryParseSlotState(data["slot_state"].GetUInt16(), out EPlayerType slot_state)) { Response.StatusCode = (int)HttpStatusCode.BadRequest; - return null; + return; } // human slots are left alone; kicking is how a host removes a player @@ -796,165 +687,44 @@ internal static bool TryParseLobbyUpdateField(int rawValue, out ELobbyUpdateFiel } } - return null; - } - - // Runs the parts of a kick that RemoveMember itself performs (leaving the lobby, which - // acquires this same lobby's gate) plus TURN/session cleanup - all after - // ApplyLobbyFieldUpdateAsync's gate has been released, never from inside it. - private async Task PerformKickAsync(Lobby lobby, Int64 kickedUserID) - { - // TODO: we should communicate the kick to the user... - await _lobbyManager.LeaveSpecificLobby(kickedUserID, lobby.LobbyID); - - // cleanup TURN credentials - TURNCredentialManager.DeleteCredentialsForUser(kickedUserID); - - // clear our lobby ID - UserSession? sourceData = WebSocketManager.GetSessionFromUser(kickedUserID, EUserSessionType.GameClient); // user being kicked must be a game client - - if (sourceData != null) - { - sourceData.UpdateSessionLobbyID(-1); - // NOTE: We dont update the match history match ID here, that is done by the match history service - } - - // we have to manually send to the kicked user... they won't get the dirty lobby update anymore - await lobby.DirtyRetransmitToSingleMember(kickedUserID); + return; } [HttpPut("{lobbyID}")] [Authorize(Roles = "GameClient")] public async Task Put(Int64 lobbyID) { - RouteHandler_PUT_Lobby_Result result = new RouteHandler_PUT_Lobby_Result(); - - using (var reader = new StreamReader(HttpContext.Request.Body)) - { - string jsonData = await reader.ReadToEndAsync(); - - var options = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; - - try - { - // TODO: Dont let them join more than 1 lobby - - var data = JsonSerializer.Deserialize>(jsonData, options); - - if (data != null - && data.ContainsKey("preferred_port") - && data.ContainsKey("anticheat_id") - ) - { - - Lobby? lobby = _lobbyManager.GetLobby(lobbyID); - - if (lobby != null) - { - Int64 user_id = TokenHelper.GetUserID(this); - EUserSessionType sessionType = TokenHelper.GetSessionType(this); - if (user_id != -1 && SessionHelpers.SessionTypeHasAccessTo(sessionType, ESessionAccessType.Gameplay)) - { - if (lobby.LobbyType == ELobbyType.QuickMatch && !lobby.MatchedUserIDs.Contains(user_id)) - { - Response.StatusCode = (int)HttpStatusCode.Forbidden; - result.success = false; - return result; - } - - UInt16 userPreferredPort = data["preferred_port"].GetUInt16(); - bool bHasMap = data["has_map"].GetBoolean(); - EKnownAnticheatID anticheatID = (EKnownAnticheatID)data["anticheat_id"].GetInt32(); - - // does the lobby have a password? - bool bLobbyPassworded = lobby.IsPassworded; - bool bUserProvidedPassword = data.ContainsKey("password"); - - // we need a password - if (bLobbyPassworded) - { - if (bUserProvidedPassword) - { - // do the passwords match? - string? strUserProvidedPassword = data["password"].GetString(); - - // if it doesnt match, bail, otherwise just proceed as normal - if (strUserProvidedPassword != lobby.Password) - { - Response.StatusCode = (int)HttpStatusCode.Unauthorized; - result.success = false; - return result; - } - } - else - { - // no password, no access - Response.StatusCode = (int)HttpStatusCode.Unauthorized; - result.success = false; - return result; - } - } - - // must be running same AC - if (anticheatID != lobby.AnticheatID) - { - Response.StatusCode = (int)HttpStatusCode.ExpectationFailed; - result.success = false; - return result; - } - - UserSession? playerSession = WebSocketManager.GetSessionFromUser(user_id, sessionType); - - if (playerSession != null) - { - // leave any lobby - await _lobbyManager.LeaveAnyLobby(user_id); - - await using var db = await _dbFactory.CreateDbContextAsync(); - string strDisplayName = await Database.Users.GetDisplayName(db, user_id); - bool bJoinedSuccessfully = await _lobbyManager.JoinLobby(db, lobby, playerSession, strDisplayName, userPreferredPort, bHasMap); - - result.success = bJoinedSuccessfully; - - if (!bJoinedSuccessfully) // this basically means full, didnt find a slot in correct state - { - Response.StatusCode = (int)HttpStatusCode.NotAcceptable; - } - else - { - // TODO: What if this fails? just let them proceed? just means only direct connect people will be able to play - // get some turn credentials - TURNCredentialContainer? turnCredentials = await TURNCredentialManager.CreateCredentialsForUser(user_id); - if (turnCredentials != null) - { - result.turn_username = turnCredentials.m_strUsername; - result.turn_token = turnCredentials.m_strToken; - } - - Response.StatusCode = (int)HttpStatusCode.OK; - } - } - else - { - Response.StatusCode = (int)HttpStatusCode.Unauthorized; - } - } - - } - } - - } - catch - { - return result; - } - } - - return result; - - } + var result = new RouteHandler_PUT_Lobby_Result(); + using var reader = new StreamReader(HttpContext.Request.Body); + string jsonData = await reader.ReadToEndAsync(); + try + { + var data = JsonSerializer.Deserialize>(jsonData); + if (data == null || !data.ContainsKey("preferred_port") || !data.ContainsKey("anticheat_id")) return result; + var joined = await _lobbyManager.JoinAsync(new JoinLobbyCommand(lobbyID, + TokenHelper.GetUserID(this), TokenHelper.GetSessionType(this), + data["preferred_port"].GetUInt16(), data.TryGetValue("has_map", out var hasMap) && hasMap.GetBoolean(), + (EKnownAnticheatID)data["anticheat_id"].GetInt32(), + data.TryGetValue("password", out var password) ? password.GetString() : null)); + result.success = joined.Status == LobbyMembershipStatus.Success; + result.turn_username = joined.TurnUsername; + result.turn_token = joined.TurnToken; + Response.StatusCode = (int)MapMembershipStatus(joined.Status); + } + catch + { + return result; + } + return result; + } + + private static HttpStatusCode MapMembershipStatus(LobbyMembershipStatus status) => status switch + { + LobbyMembershipStatus.ActorNotMember or LobbyMembershipStatus.ActorNotHost or LobbyMembershipStatus.SessionUnavailable or LobbyMembershipStatus.PasswordMismatch => HttpStatusCode.Unauthorized, + LobbyMembershipStatus.NotMatched => HttpStatusCode.Forbidden, + LobbyMembershipStatus.AnticheatMismatch => HttpStatusCode.ExpectationFailed, + LobbyMembershipStatus.Success or LobbyMembershipStatus.NoChange => HttpStatusCode.OK, + _ => HttpStatusCode.NotAcceptable + }; } } diff --git a/GenOnlineService/Controllers/LoginCode/LoginCodeController.cs b/GenOnlineService/Controllers/LoginCode/LoginCodeController.cs index 99824c4..f773216 100644 --- a/GenOnlineService/Controllers/LoginCode/LoginCodeController.cs +++ b/GenOnlineService/Controllers/LoginCode/LoginCodeController.cs @@ -150,10 +150,15 @@ public override Type GetReturnType() [Route("env/{environment}/contract/{contract_version}/[controller]")] public class LoginCode : ControllerBase { + private static readonly ILogger s_log = AppLog.For(typeof(LoginCode)); [HttpPost] public async Task Post([FromHeader(Name = "X-Api-Key")] string apiKey) { + using IDisposable authenticationMetric = AppMetrics.TrackAuthentication( + "login_code_callback", + "partner", + () => Response.StatusCode < 400 ? "success" : $"http_{Response.StatusCode}"); if (string.IsNullOrEmpty(apiKey)) { Response.StatusCode = (int)HttpStatusCode.Unauthorized; @@ -203,22 +208,26 @@ public async Task Post([FromHeader(Name = "X-Api-Key")] string apiKey) if (!bUpdated) { + s_log.LogWarning("Login-code callback referenced a missing, expired, or completed pending login for user {UserId}", user_id); Response.StatusCode = (int)HttpStatusCode.NotFound; } } - catch + catch (Exception ex) { + s_log.LogWarning(ex, "Login-code callback had invalid field values"); Response.StatusCode = (int)HttpStatusCode.PreconditionFailed; } } else { + s_log.LogWarning("Login-code callback was missing required fields"); Response.StatusCode = (int)HttpStatusCode.Unauthorized; } } - catch + catch (Exception ex) { + s_log.LogError(ex, "Login-code callback could not be processed"); Response.StatusCode = (int)HttpStatusCode.InternalServerError; } } @@ -232,6 +241,7 @@ public async Task Get() result.login_code = GenerateLoginCode(); // a collision with a live code must not be handed out to a second client result.success = PendingLoginManager.AddPendingLogin(result.login_code); + AppMetrics.RecordAuthentication("login_code_issue", "client", result.success ? "success" : "collision"); return result; } diff --git a/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs b/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs index 97adc87..64e238d 100644 --- a/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs +++ b/GenOnlineService/Controllers/LoginWithToken/LoginWithTokenController.cs @@ -84,6 +84,10 @@ public async Task Post_InternalHandler(string jsonData, string ipAddr } POST_LoginWithToken_Result result = new POST_LoginWithToken_Result(); + using IDisposable authenticationMetric = AppMetrics.TrackAuthentication( + "login_with_token", + bWasMonitor ? "monitor" : "client", + () => result.result.ToMetricLabel()); var options = new JsonSerializerOptions { diff --git a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs index 1de844d..c7956d9 100644 --- a/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs +++ b/GenOnlineService/Controllers/Matchmaking/MatchmakingController.cs @@ -88,8 +88,9 @@ public MatchmakingController(ILogger logger) } } } - catch + catch (Exception ex) { + _logger.LogError(ex, "Malformed or failed matchmaking registration request"); Response.StatusCode = (int)HttpStatusCode.InternalServerError; } } @@ -128,7 +129,7 @@ public async Task Delete() if (playerSession != null) { - await MatchmakingManager.DeregisterPlayer(playerSession); + await MatchmakingManager.DeregisterPlayerAsync(playerSession); } } } diff --git a/GenOnlineService/Controllers/Monitoring/MonitoringController.cs b/GenOnlineService/Controllers/Monitoring/MonitoringController.cs index 58bff4b..4dde8c3 100644 --- a/GenOnlineService/Controllers/Monitoring/MonitoringController.cs +++ b/GenOnlineService/Controllers/Monitoring/MonitoringController.cs @@ -228,8 +228,9 @@ public async Task Monitor_Database() GenOnlineService.Controllers.LoginWithToken.POST_LoginWithToken_Result internalResult = (GenOnlineService.Controllers.LoginWithToken.POST_LoginWithToken_Result)await loginWithTokenController.Post_InternalHandler("{\"challenge\": \"abc\", \"token\": \"iamatest\", \"client_id\": \"gen_online_60hz\"}", IPAddress.Loopback.ToString(), Program.BuildWebSocketUrl(Request, environment, contractVersion)); return internalResult; } - catch + catch (Exception ex) { + _logger.LogError(ex, "Monitoring LoginWithToken check failed"); Response.StatusCode = (int)HttpStatusCode.InternalServerError; } } @@ -279,8 +280,9 @@ public APIResult Monitor_Uptime() return internalResult; } - catch + catch (Exception ex) { + _logger.LogError(ex, "Monitoring VersionCheck failed"); Response.StatusCode = (int)HttpStatusCode.InternalServerError; } } @@ -308,8 +310,9 @@ public APIResult Monitor_Uptime() APIResult internalResult = await checkLoginController.Post_InternalHandler("{\"challenge\": \"abc\", \"nonce\": \"def\", \"code\": \"iamatest\", \"client_id\": \"gen_online_30hz\"}", IPAddress.Loopback.ToString(), Program.BuildWebSocketUrl(Request, environment, contractVersion)); return internalResult; } - catch + catch (Exception ex) { + _logger.LogError(ex, "Monitoring CheckLogin check failed"); Response.StatusCode = (int)HttpStatusCode.InternalServerError; } } diff --git a/GenOnlineService/Controllers/RefreshToken/RefreshTokenController.cs b/GenOnlineService/Controllers/RefreshToken/RefreshTokenController.cs index 1c65a3a..b2e2511 100644 --- a/GenOnlineService/Controllers/RefreshToken/RefreshTokenController.cs +++ b/GenOnlineService/Controllers/RefreshToken/RefreshTokenController.cs @@ -49,10 +49,12 @@ public override Type GetReturnType() public class RefreshTokenController : ControllerBase { private readonly IDbContextFactory _dbFactory; + private readonly ILogger _logger; - public RefreshTokenController(IDbContextFactory dbFactory) + public RefreshTokenController(IDbContextFactory dbFactory, ILogger logger) { _dbFactory = dbFactory; + _logger = logger; } [HttpPost(Name = "PostRefreshToken")] @@ -64,6 +66,10 @@ public async Task Post() public async Task Post_InternalHandler(string ipAddr) { POST_RefreshToken_Result result = new POST_RefreshToken_Result(); + using IDisposable authenticationMetric = AppMetrics.TrackAuthentication( + "refresh_token", + "client", + () => result.result.ToMetricLabel()); try { @@ -128,8 +134,7 @@ public async Task Post_InternalHandler(string ipAddr) } catch (Exception ex) { - Console.WriteLine($"[ERROR] RefreshToken failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "RefreshToken failed"); result.result = EPendingLoginState.LoginFailed; Response.StatusCode = (int)HttpStatusCode.InternalServerError; diff --git a/GenOnlineService/Controllers/VersionCheck/VersionCheckController.cs b/GenOnlineService/Controllers/VersionCheck/VersionCheckController.cs index c8b41aa..bc138aa 100644 --- a/GenOnlineService/Controllers/VersionCheck/VersionCheckController.cs +++ b/GenOnlineService/Controllers/VersionCheck/VersionCheckController.cs @@ -128,6 +128,7 @@ public async Task Get() class VersionHelper { + private static readonly ILogger s_log = AppLog.For(typeof(VersionHelper)); public static APIResult Get_ManifestHandler() { GET_VersionManifest_Result manifest = new GET_VersionManifest_Result(); @@ -140,8 +141,9 @@ public static APIResult Get_ManifestHandler() manifest.execrc_30 = CRC32Calculator.CalculateCRC32(Path.Combine(Directory.GetCurrentDirectory(), "crcfiles", "GeneralsOnlineZH_30.exe")); manifest.execrc_60 = CRC32Calculator.CalculateCRC32(Path.Combine(Directory.GetCurrentDirectory(), "crcfiles", "GeneralsOnlineZH_60.exe")); } - catch (Exception) + catch (Exception ex) { + s_log.LogError(ex, "Failed to calculate executable CRCs for the version manifest"); // Keep default 0 values on failure } #endif @@ -242,9 +244,9 @@ async Task GetHTTPSize(string url) return response.Content.Headers.ContentLength.Value; } } - catch (HttpRequestException) + catch (HttpRequestException ex) { - // TODO: Log exception or handle error + s_log.LogWarning(ex, "Failed to determine patch download size"); } return -1; // Return -1 if the size could not be determined diff --git a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs index ce70dfd..78df6c7 100644 --- a/GenOnlineService/Controllers/WebSocket/WebSocketController.cs +++ b/GenOnlineService/Controllers/WebSocket/WebSocketController.cs @@ -23,6 +23,7 @@ using Microsoft.EntityFrameworkCore; using System; using System.Buffers; +using System.Diagnostics; using System.Net.WebSockets; using System.Security.Claims; using System.Text; @@ -34,11 +35,41 @@ public class WebSocketController : ControllerBase { private readonly LobbyManager _lobbyManager; private readonly IDbContextFactory _dbFactory; + private readonly ILogger _logger; + private static readonly ILogger s_log = AppLog.For(typeof(WebSocketController)); + private static readonly System.Collections.Concurrent.ConcurrentDictionary s_lastMessageFailureLog = new(); + private static readonly long s_messageFailureLogIntervalTicks = Stopwatch.Frequency * 60; + + private static bool ShouldLogMessageFailure(Int64 userID) + { + if (s_lastMessageFailureLog.Count > 10000) + { + s_lastMessageFailureLog.Clear(); + } + + long now = Stopwatch.GetTimestamp(); + bool shouldLog = false; + s_lastMessageFailureLog.AddOrUpdate( + userID, + _ => + { + shouldLog = true; + return now; + }, + (_, last) => + { + shouldLog = now - last >= s_messageFailureLogIntervalTicks; + return shouldLog ? now : last; + }); + return shouldLog; + } + - public WebSocketController(LobbyManager lobbyManager, IDbContextFactory dbFactory) + public WebSocketController(LobbyManager lobbyManager, IDbContextFactory dbFactory, ILogger logger) { _lobbyManager = lobbyManager; _dbFactory = dbFactory; + _logger = logger; } // longer chat is dropped rather than relayed @@ -52,6 +83,7 @@ public WebSocketController(LobbyManager lobbyManager, IDbContextFactory payload = buffer.AsSpan(); + int payloadLength = payload.Length; WSMessageEnvelope envelope; try @@ -477,10 +507,19 @@ private async Task ProcessWSMessage(UserWebSocketInstance sourceWS, UserSession catch { // malformed + AppMetrics.RecordWebSocketFailure("inbound", "malformed"); return; } EWebSocketMessageID msgID = (EWebSocketMessageID)envelope.msg_id; + string messageType = Enum.IsDefined(msgID) ? msgID.ToMetricLabel() : "unknown"; + string messageOutcome = messageType == "unknown" ? "ignored" : "success"; + using Activity? messageActivity = AppMetrics.StartActivity( + "websocket.message.process", + ActivityKind.Consumer, + new("messaging.system", "websocket"), + new("messaging.operation.type", "process"), + new("message.type", messageType)); // Only allocate a Dictionary when we actually need arbitrary fields Dictionary? data = null; @@ -1015,55 +1054,15 @@ await lobby.RunExclusiveAsync(() => } } } - else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_HOST_REQUESTS_BEGIN) - { - // Host has requested this, as part of the start game flow - - // must be in a lobby - Lobby? lobbyInfo = null; - if (sourceUserSession.currentLobbyID != -1) - { - // must be lobby owner too - lobbyInfo = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); - - if (lobbyInfo == null || lobbyInfo.Owner != sourceUserSession.m_UserID) - { - return; - } - } - - if (lobbyInfo == null) - { - return; - } - - // lock slots (more people joining when we're already doing connectivity checks won't help the situation) - // Awaited separately from StartFullMeshConnectivityCheck below: each is its own - // gated operation on the lobby, so neither ever nests inside the other's gate use. - await lobbyInfo.CloseOpenSlots(); - - // mark lobby as in progress of full mesh connectivity checks - await lobbyInfo.StartFullMeshConnectivityCheck(); - - // start full mesh connectivity checks - lobbyInfo.SendFullMeshConnectivityCheckRequestToMembers(); - } - else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) - { - // process a response from a user - WebSocketMessage_FullMeshConnectivityCheckResponseFromUser? fullMeshMsg = - JsonSerializer.Deserialize(payload, JsonOpts); - - // store response - if (fullMeshMsg != null) - { - Lobby? lobby = _lobbyManager.GetLobby(sourceUserSession.currentLobbyID); - if (lobby != null) - { - await lobby.StoreFullMeshConnectivityResponse(sourceUserSession.m_UserID, fullMeshMsg); - } - } - } + else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_HOST_REQUESTS_BEGIN) + { + await _lobbyManager.StartMeshCheckAsync(sourceUserSession, sourceWS); + } + else if (msgID == EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE) + { + var response = JsonSerializer.Deserialize(payload, JsonOpts); + if (response != null) await _lobbyManager.StoreMeshResponseAsync(sourceUserSession, sourceWS, response); + } else if (msgID == EWebSocketMessageID.NETWORK_CONNECTION_CLIENT_REQUEST_SIGNALLING) { WebSocketMessage_RequestSignaling? signalingRequest = @@ -1314,10 +1313,22 @@ await Database.AntiCheat.FlagAccountForReview_Module( } } } - catch + catch (Exception ex) + { + messageOutcome = "error"; + AppMetrics.RecordException(messageActivity, ex); + AppMetrics.RecordWebSocketFailure("inbound", "error"); + // log at Error at most once per user per minute, metrics and spans still count every failure + _logger.Log( + ShouldLogMessageFailure(sourceUserSession.m_UserID) ? LogLevel.Error : LogLevel.Debug, + ex, + "WebSocket message {MessageType} failed for user {UserId}", + messageType, + sourceUserSession.m_UserID); + } + finally { - // swallow per-message exceptions to avoid killing the loop - // you can add Sentry logging here if desired + messageActivity?.SetTag("message.outcome", messageOutcome); } } } diff --git a/GenOnlineService/Database/Database.AntiCheat.cs b/GenOnlineService/Database/Database.AntiCheat.cs index 25ff2a1..f4140da 100644 --- a/GenOnlineService/Database/Database.AntiCheat.cs +++ b/GenOnlineService/Database/Database.AntiCheat.cs @@ -16,6 +16,7 @@ ** along with this program. If not, see . */ +using GenOnlineService; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using System.Text.Json; @@ -181,6 +182,8 @@ namespace Database { public static class AntiCheat { + private static readonly ILogger s_log = AppLog.For(typeof(AntiCheat)); + public static async Task AC_BanUser(AppDbContext db, long userId, string reason) { try @@ -194,11 +197,12 @@ await db.Users .SetProperty(u => u.BanVerifiedBy, "x64") .SetProperty(u => u.BanAliases, "") ); + AppMetrics.RecordSecurityAction("anticheat_", "success"); } catch (Exception ex) { - Console.WriteLine($"[ERROR] AC_BanUser failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + AppMetrics.RecordSecurityAction("anticheat_", "error"); + s_log.LogError(ex, "AC_BanUser failed"); } } @@ -221,11 +225,12 @@ public static async Task FlagAccountForReview( }); await db.SaveChangesAsync(); + AppMetrics.RecordSecurityAction("anticheat_", "success"); } catch (Exception ex) { - Console.WriteLine($"[ERROR] FlagAccountForReview failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + AppMetrics.RecordSecurityAction("anticheat_", "error"); + s_log.LogError(ex, "FlagAccountForReview failed"); } } @@ -250,11 +255,12 @@ public static async Task FlagAccountForReview_Module( }); await db.SaveChangesAsync(); + AppMetrics.RecordSecurityAction("anticheat_", "success"); } catch (Exception ex) { - Console.WriteLine($"[ERROR] FlagAccountForReview_Module failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + AppMetrics.RecordSecurityAction("anticheat_", "error"); + s_log.LogError(ex, "FlagAccountForReview_Module failed"); } } @@ -276,11 +282,12 @@ public static async Task FlagAccountForReview_SuspectProbes( }); await db.SaveChangesAsync(); + AppMetrics.RecordSecurityAction("anticheat_", "success"); } catch (Exception ex) { - Console.WriteLine($"[ERROR] FlagAccountForReview_SuspectProbes failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + AppMetrics.RecordSecurityAction("anticheat_", "error"); + s_log.LogError(ex, "FlagAccountForReview_SuspectProbes failed"); } } @@ -299,11 +306,12 @@ public static async Task FlagAccountForReview_NewAccount_FirstMatches( }); await db.SaveChangesAsync(); + AppMetrics.RecordSecurityAction("anticheat_", "success"); } catch (Exception ex) { - Console.WriteLine($"[ERROR] FlagAccountForReview_NewAccount_FirstMatches failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + AppMetrics.RecordSecurityAction("anticheat_", "error"); + s_log.LogError(ex, "FlagAccountForReview_NewAccount_FirstMatches failed"); } } diff --git a/GenOnlineService/Database/Database.ConnectionOutcomes.cs b/GenOnlineService/Database/Database.ConnectionOutcomes.cs index 5f90445..f296951 100644 --- a/GenOnlineService/Database/Database.ConnectionOutcomes.cs +++ b/GenOnlineService/Database/Database.ConnectionOutcomes.cs @@ -61,6 +61,8 @@ namespace Database { public static class ConnectionOutcomes { + private static readonly ILogger s_log = AppLog.For(typeof(ConnectionOutcomes)); + public static async Task StoreConnectionOutcome( AppDbContext db, EIPVersion protocol, @@ -125,8 +127,7 @@ await db.ConnectionOutcomes } catch (Exception ex) { - Console.WriteLine($"[ERROR] StoreConnectionOutcome failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "StoreConnectionOutcome failed"); } } diff --git a/GenOnlineService/Database/Database.DailyStats.cs b/GenOnlineService/Database/Database.DailyStats.cs index b6e3118..103f7f5 100644 --- a/GenOnlineService/Database/Database.DailyStats.cs +++ b/GenOnlineService/Database/Database.DailyStats.cs @@ -16,6 +16,7 @@ ** along with this program. If not, see . */ +using GenOnlineService; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using System.Text.Json; @@ -64,6 +65,8 @@ public void Configure(EntityTypeBuilder builder) public static class DailyStatsManager { + private static readonly ILogger s_log = AppLog.For(typeof(DailyStatsManager)); + public static DailyStat g_StatsContainer = new(); public static async Task LoadFromDB(AppDbContext db) @@ -81,8 +84,7 @@ public static async Task LoadFromDB(AppDbContext db) } catch (Exception ex) { - Console.WriteLine($"[ERROR] DailyStats.LoadFromDB failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "DailyStats.LoadFromDB failed"); g_StatsContainer = new DailyStat(); } } @@ -114,8 +116,7 @@ public static async Task SaveToDB(AppDbContext db) } catch (Exception ex) { - Console.WriteLine($"[ERROR] DailyStats.SaveToDB failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "DailyStats.SaveToDB failed"); } } @@ -144,8 +145,7 @@ public static void RegisterOutcome(int army, bool bWon) } catch (Exception ex) { - Console.WriteLine($"[ERROR] RegisterOutcome failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "RegisterOutcome failed"); } } } \ No newline at end of file diff --git a/GenOnlineService/Database/Database.Leaderboards.cs b/GenOnlineService/Database/Database.Leaderboards.cs index 69c41df..c10976d 100644 --- a/GenOnlineService/Database/Database.Leaderboards.cs +++ b/GenOnlineService/Database/Database.Leaderboards.cs @@ -16,6 +16,7 @@ ** along with this program. If not, see . */ +using GenOnlineService; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using System; @@ -146,6 +147,8 @@ namespace Database { public static class Leaderboards { + private static readonly ILogger s_log = AppLog.For(typeof(Leaderboards)); + public struct LeaderboardPoints { @@ -265,8 +268,7 @@ public static async Task CreateUserEntriesIfNotExists(AppDbContext db, long play } catch (Exception ex) { - Console.WriteLine($"[ERROR] CreateUserEntriesIfNotExists failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CreateUserEntriesIfNotExists failed"); } } @@ -338,8 +340,7 @@ public async static ValueTask> GetBulkLeader } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetBulkLeaderboardData failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetBulkLeaderboardData failed"); } return results; diff --git a/GenOnlineService/Database/Database.MatchHistory.cs b/GenOnlineService/Database/Database.MatchHistory.cs index dc944bf..c040834 100644 --- a/GenOnlineService/Database/Database.MatchHistory.cs +++ b/GenOnlineService/Database/Database.MatchHistory.cs @@ -261,6 +261,8 @@ namespace Database // TODO_EFCORE: Consider moving to zero-serialization model public static class MatchHistory { + private static readonly ILogger s_log = AppLog.For(typeof(MatchHistory)); + private static readonly Expression>[] _slotSelectors = { m => m.MemberSlot0, @@ -415,8 +417,7 @@ public static async Task CommitPlayerOutcome( } catch (Exception ex) { - Console.WriteLine($"[ERROR] CommitPlayerOutcome failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CommitPlayerOutcome failed"); } } @@ -561,8 +562,7 @@ public static async Task CreatePlaceholderMatchHistory( } catch (Exception ex) { - Console.WriteLine($"[ERROR] CreatePlaceholderMatchHistory failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CreatePlaceholderMatchHistory failed"); return 0; } } @@ -714,11 +714,12 @@ public static async Task DetermineLobbyWinnerIfNotPresent( // in-game) over TimeMemberLeft (recorded when the player was structurally removed from the // lobby, which can happen much later, or earlier due to a fresh-session reconnect, skewing // the order relative to the actual in-game quit order). - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID} LobbyID={lobby.LobbyID}: no winner in DB — timestamp fallback. IngameAbandon={lobby.TimePlayerAbandonedIngame.Count} MemberLeft={lobby.TimeMemberLeft.Count}"); + s_log.LogInformation("Match {MatchId} lobby {LobbyId}: no winner in DB, falling back to abandon timestamps. IngameAbandon={IngameAbandonCount} MemberLeft={MemberLeftCount}", + lobby.MatchID, lobby.LobbyID, lobby.TimePlayerAbandonedIngame.Count, lobby.TimeMemberLeft.Count); foreach (var _kv in lobby.TimePlayerAbandonedIngame) - Console.WriteLine($"[WinnerDet] IngameAbandon: user={_kv.Key} at={_kv.Value:O}"); + s_log.LogDebug("Match {MatchId}: IngameAbandon user={UserId} at={AbandonTime}", lobby.MatchID, _kv.Key, _kv.Value); foreach (var _kv in lobby.TimeMemberLeft) - Console.WriteLine($"[WinnerDet] MemberLeft: user={_kv.Key} at={_kv.Value:O}"); + s_log.LogDebug("Match {MatchId}: MemberLeft user={UserId} at={LeftTime}", lobby.MatchID, _kv.Key, _kv.Value); // 6a. In a 1v1, drop anyone who reported a loss - see BuildFallbackCandidates. string strRosterType = await db.MatchHistory .Where(m => m.MatchId == (long)lobby.MatchID) @@ -726,7 +727,8 @@ public static async Task DetermineLobbyWinnerIfNotPresent( .FirstOrDefaultAsync() ?? String.Empty; List lstCandidateSlots = BuildFallbackCandidates(members, strRosterType, lobby.ReportedOutcomes); - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: rosterType='{strRosterType}' reported={lobby.ReportedOutcomes.Count} candidates=[{String.Join(",", lstCandidateSlots)}]"); + s_log.LogInformation("Match {MatchId}: rosterType={RosterType} reported={ReportedOutcomeCount} candidates=[{CandidateSlots}]", + lobby.MatchID, strRosterType, lobby.ReportedOutcomes.Count, String.Join(",", lstCandidateSlots)); DateTime latestLeave = DateTime.MinValue; MatchdataMemberModel? lastPlayerNullable = null; @@ -757,7 +759,8 @@ public static async Task DetermineLobbyWinnerIfNotPresent( abandonSource = "MemberLeft"; } - Console.WriteLine($"[WinnerDet] Slot={kv.Key} user={model.user_id} side={model.side} team={model.team} time={abandonTime:O} src={abandonSource}"); + s_log.LogDebug("Match {MatchId}: slot={Slot} user={UserId} side={Side} team={Team} time={AbandonTime} src={AbandonSource}", + lobby.MatchID, kv.Key, model.user_id, model.side, model.team, abandonTime, abandonSource); if (abandonTime > latestLeave) { latestLeave = abandonTime; @@ -772,12 +775,13 @@ public static async Task DetermineLobbyWinnerIfNotPresent( { lastSlot = lstCandidateSlots[0]; lastPlayerNullable = members[lastSlot]; - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: sole remaining candidate slot={lastSlot} user={lastPlayerNullable.Value.user_id} has no abandon timestamp — awarding anyway."); + s_log.LogInformation("Match {MatchId}: sole remaining candidate slot={Slot} user={UserId} has no abandon timestamp, awarding anyway", + lobby.MatchID, lastSlot, lastPlayerNullable.Value.user_id); } if (lastPlayerNullable == null) { - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: no valid abandon timestamps found — fully inconclusive, clearing won flags."); + s_log.LogWarning("Match {MatchId}: no valid abandon timestamps found, fully inconclusive, clearing won flags", lobby.MatchID); foreach (var kv in members) { if (kv.Value.side == Constants.OBSERVER_SIDE_VALUE) @@ -790,7 +794,8 @@ public static async Task DetermineLobbyWinnerIfNotPresent( MatchdataMemberModel lastPlayer = lastPlayerNullable.Value; int winningTeam = lastPlayer.team; - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: winner selected → user={lastPlayer.user_id} slot={lastSlot} team={winningTeam} at={latestLeave:O}"); + s_log.LogInformation("Match {MatchId}: winner selected, user={UserId} slot={Slot} team={Team} at={LeaveTime}", + lobby.MatchID, lastPlayer.user_id, lastSlot, winningTeam, latestLeave); // 7. Mark last player + teammates as winners, explicitly clear everyone else. foreach (var kv in members) @@ -802,13 +807,14 @@ public static async Task DetermineLobbyWinnerIfNotPresent( bool isWinner = model.user_id == lastPlayer.user_id || (winningTeam != -1 && model.team == winningTeam); - Console.WriteLine($"[WinnerDet] Match={lobby.MatchID}: marking slot={kv.Key} user={model.user_id} as {(isWinner ? "WINNER" : "loser")}."); + s_log.LogDebug("Match {MatchId}: marking slot={Slot} user={UserId} as {Result}", + lobby.MatchID, kv.Key, model.user_id, isWinner ? "WINNER" : "loser"); await UpdateMatchHistorySetWinFlag(db, lobby.MatchID, kv.Key, isWinner, cancellationToken); } } catch (Exception ex) { - Console.WriteLine($"[ERROR] DetermineLobbyWinnerIfNotPresent failed: {ex.Message}"); + s_log.LogError(ex, "DetermineLobbyWinnerIfNotPresent failed"); throw; } } @@ -855,7 +861,7 @@ public static async Task UpdateMatchHistorySetWinFlag( } catch (Exception ex) { - Console.WriteLine($"[ERROR] UpdateMatchHistorySetWinFlag failed: {ex.Message}"); + s_log.LogError(ex, "UpdateMatchHistorySetWinFlag failed"); throw; } } @@ -942,8 +948,7 @@ public static async Task GetMatchesInRange( } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetMatchesInRange failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetMatchesInRange failed"); } return collection; @@ -997,8 +1002,7 @@ public static async Task GetMatchesSince( } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetMatchesSince failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetMatchesSince failed"); } return collection; @@ -1025,8 +1029,7 @@ public static async Task GetHighestMatchID(AppDbContext db) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetHighestMatchID failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetHighestMatchID failed"); return -1; } } @@ -1118,6 +1121,18 @@ public static async Task> GetPendingExternalPu .ToList(); } + public static async Task GetOldestUnpublishedMatchTime( + AppDbContext db, + CancellationToken cancellationToken) + { + return await ( + from publication in db.ExternalPublications.AsNoTracking() + join match in db.MatchHistory.AsNoTracking() on publication.MatchId equals match.MatchId + where publication.PublishedAt == null && match.Finished + select (DateTime?)match.TimeFinished) + .MinAsync(cancellationToken); + } + public static async Task MarkExternalPublicationSucceeded( AppDbContext db, ulong matchId, @@ -1295,8 +1310,7 @@ await UpdatePeriodEloAndLeaderboardsAsync( } catch (Exception ex) { - Console.WriteLine($"[ERROR] UpdateLeaderboardAndElo failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UpdateLeaderboardAndElo failed"); } } private static async Task> LoadMatchMembersAsync( @@ -1349,7 +1363,8 @@ private static async Task UpdateCurrentEloAsync( if (a.user_id >= b.user_id) continue; - Console.WriteLine($"[ELO] Pairing a={a.user_id}(won={a.won}) vs b={b.user_id}(won={b.won}) → result={(a.won ? "PlayerAWins" : "PlayerBWins")}"); + s_log.LogDebug("ELO pairing a={PlayerA}(won={AWon}) vs b={PlayerB}(won={BWon}), result={Result}", + a.user_id, a.won, b.user_id, b.won, a.won ? "PlayerAWins" : "PlayerBWins"); ref EloData A = ref CollectionsMarshal.GetValueRefOrAddDefault( dictElo, a.user_id, out _); diff --git a/GenOnlineService/Database/Database.PlayerStats.cs b/GenOnlineService/Database/Database.PlayerStats.cs index 597380c..2183d66 100644 --- a/GenOnlineService/Database/Database.PlayerStats.cs +++ b/GenOnlineService/Database/Database.PlayerStats.cs @@ -59,6 +59,8 @@ namespace Database { public static class UserStats { + private static readonly ILogger s_log = AppLog.For(typeof(UserStats)); + private static readonly Func> _getUserStatsJson = EF.CompileAsyncQuery( (AppDbContext db, long userId) => @@ -90,8 +92,7 @@ public static async Task GetPlayerStats( } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetPlayerStats failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetPlayerStats failed"); return new PlayerStats(userId, EloConfig.BaseRating, 0, EloConfig.BaseRating); } @@ -123,8 +124,7 @@ public static async Task GetPlayerStats( } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetPlayerStats failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetPlayerStats failed"); } return ps; @@ -186,8 +186,7 @@ await db.UserStats } catch (Exception ex) { - Console.WriteLine($"[ERROR] UpdatePlayerStat failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UpdatePlayerStat failed"); } } diff --git a/GenOnlineService/Database/Database.ServiceStats.cs b/GenOnlineService/Database/Database.ServiceStats.cs index 44ef93b..f049a24 100644 --- a/GenOnlineService/Database/Database.ServiceStats.cs +++ b/GenOnlineService/Database/Database.ServiceStats.cs @@ -16,6 +16,7 @@ ** along with this program. If not, see . */ +using GenOnlineService; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; using System.Text.Json; @@ -55,6 +56,8 @@ namespace Database { public static class ServiceStats { + private static readonly ILogger s_log = AppLog.For(typeof(ServiceStats)); + public static readonly Func> FindStatTracked = EF.CompileAsyncQuery( (AppDbContext db, int day, int hour) => @@ -114,8 +117,7 @@ public static async Task CommitStats( } catch (Exception ex) { - Console.WriteLine($"[ERROR] CommitStats failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CommitStats failed"); } } diff --git a/GenOnlineService/Database/Database.Social.cs b/GenOnlineService/Database/Database.Social.cs index 63ae0f2..42d034d 100644 --- a/GenOnlineService/Database/Database.Social.cs +++ b/GenOnlineService/Database/Database.Social.cs @@ -93,6 +93,8 @@ namespace Database { public static class Social { + private static readonly ILogger s_log = AppLog.For(typeof(Social)); + private static readonly Func> _getFriends = EF.CompileAsyncQuery( (AppDbContext db, long userId) => @@ -138,8 +140,7 @@ public static async Task> GetFriends(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetFriends failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetFriends failed"); } return result; @@ -154,8 +155,7 @@ public static async Task CountFriends(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] CountFriends failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CountFriends failed"); // Treat an unreadable count as full so the friends limit is never bypassed. return int.MaxValue; @@ -174,8 +174,7 @@ public static async Task> GetBlocked(AppDbContext db, long sourceU } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetBlocked failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetBlocked failed"); } return result; @@ -193,8 +192,7 @@ public static async Task> GetPendingFriendsRequests(AppDbContext d } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetPendingFriendsRequests failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetPendingFriendsRequests failed"); } return result; @@ -213,8 +211,7 @@ await db.FriendRequests } catch (Exception ex) { - Console.WriteLine($"[ERROR] RemovePendingFriendRequest failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "RemovePendingFriendRequest failed"); } } @@ -246,8 +243,7 @@ public static async Task CreateFriendship(AppDbContext db, long userId1, long us if (ex.InnerException is MySqlConnector.MySqlException mysqlEx && mysqlEx.Number == 1062) return; - Console.WriteLine($"[ERROR] CreateFriendship failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "CreateFriendship failed"); } } @@ -263,8 +259,7 @@ await db.Friends } catch (Exception ex) { - Console.WriteLine($"[ERROR] RemoveFriendship failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "RemoveFriendship failed"); } } @@ -282,8 +277,7 @@ public static async Task AddBlock(AppDbContext db, long sourceUserId, long targe } catch (Exception ex) { - Console.WriteLine($"[ERROR] AddBlock failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "AddBlock failed"); } } @@ -297,8 +291,7 @@ await db.BlockedUsers } catch (Exception ex) { - Console.WriteLine($"[ERROR] RemoveBlock failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "RemoveBlock failed"); } } @@ -316,8 +309,7 @@ public static async Task AddPendingFriendRequest(AppDbContext db, long sourceUse } catch (Exception ex) { - Console.WriteLine($"[ERROR] AddPendingFriendRequest failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "AddPendingFriendRequest failed"); } } diff --git a/GenOnlineService/Database/Database.User.cs b/GenOnlineService/Database/Database.User.cs index af66d34..24a01b2 100644 --- a/GenOnlineService/Database/Database.User.cs +++ b/GenOnlineService/Database/Database.User.cs @@ -157,6 +157,8 @@ namespace Database { public static class UserDevices { + private static readonly ILogger s_log = AppLog.For(typeof(UserDevices)); + public static readonly Func> FindDevice = EF.CompileAsyncQuery( (AppDbContext db, long userId, string h0, string h1, string h2) => @@ -226,8 +228,7 @@ public static async Task RegisterUserDevice( } catch (Exception ex) { - Console.WriteLine($"[ERROR] RegisterUserDevice failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "RegisterUserDevice failed"); } } } @@ -235,6 +236,8 @@ public static async Task RegisterUserDevice( public static class Users { + private static readonly ILogger s_log = AppLog.For(typeof(Users)); + private static readonly Func> GetEloData = EF.CompileAsyncQuery( (AppDbContext db, long userId) => @@ -341,8 +344,7 @@ public static async Task> GetBulkELOData( } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetBulkELOData failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetBulkELOData failed"); } return results; @@ -396,8 +398,7 @@ public static async Task IsUserAdmin(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] IsUserAdmin failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "IsUserAdmin failed"); return false; } } @@ -418,8 +419,7 @@ public static async Task> GetDisplayNameBulk(AppDbConte } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetDisplayNameBulk failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetDisplayNameBulk failed"); } return dict; @@ -433,8 +433,7 @@ public static async Task IsUserBanned(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] IsUserBanned failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "IsUserBanned failed"); return false; } } @@ -447,8 +446,7 @@ public static async Task IsUserBanned(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetUserBanStatus failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetUserBanStatus failed"); return null; } } @@ -462,8 +460,7 @@ public static async Task GetDisplayName(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetDisplayName failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetDisplayName failed"); return string.Empty; } } @@ -476,8 +473,7 @@ public static async Task GetDisplayName(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetUserLobbyPreferences failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetUserLobbyPreferences failed"); return null; } } @@ -495,8 +491,7 @@ public static async Task SetFavorite_LimitSuperweapons( } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetFavorite_LimitSuperweapons failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetFavorite_LimitSuperweapons failed"); } } @@ -515,8 +510,7 @@ public static async Task GetELOData(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetELOData API call failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetELOData API call failed"); } try @@ -528,8 +522,7 @@ public static async Task GetELOData(AppDbContext db, long userId) } catch (Exception ex) { - Console.WriteLine($"[ERROR] GetELOData fallback failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "GetELOData fallback failed"); } return new EloData(EloConfig.BaseRating, EloConfig.BaseRating, 0); @@ -547,8 +540,7 @@ public static async Task SetFavorite_Map( } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetFavorite_Map failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetFavorite_Map failed"); } } @@ -565,8 +557,7 @@ await db.Users } catch (Exception ex) { - Console.WriteLine($"[ERROR] UpdateLastLoginData failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UpdateLastLoginData failed"); } } @@ -582,8 +573,7 @@ public static async Task SetFavorite_StartingMoney( } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetFavorite_StartingMoney failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetFavorite_StartingMoney failed"); } } @@ -598,8 +588,7 @@ public static async Task SetFavorite_Side( } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetFavorite_Side failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetFavorite_Side failed"); } } @@ -623,8 +612,7 @@ await db.Users } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetDisplayName failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetDisplayName failed"); return false; } } @@ -640,8 +628,7 @@ public static async Task SetFavorite_Color( } catch (Exception ex) { - Console.WriteLine($"[ERROR] SetFavorite_Color failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SetFavorite_Color failed"); } } @@ -667,8 +654,7 @@ await db.Users } catch (Exception ex) { - Console.WriteLine($"[ERROR] SaveELOData failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "SaveELOData failed"); } } diff --git a/GenOnlineService/Discord.cs b/GenOnlineService/Discord.cs index 85814d9..75c24d7 100644 --- a/GenOnlineService/Discord.cs +++ b/GenOnlineService/Discord.cs @@ -98,6 +98,7 @@ public static string FormatString(string strFormat, params object[] strParams) public class DiscordBot { + private static readonly Microsoft.Extensions.Logging.ILogger s_log = AppLog.For(typeof(DiscordBot)); enum EBotAction { PushScriptedMessage @@ -114,7 +115,7 @@ public DiscordBot() _ = InitAsync().ContinueWith(t => { if (t.IsFaulted) - Console.WriteLine("Discord initialization failed: " + t.Exception); + s_log.LogError(t.Exception, "Discord initialization failed"); }, TaskContinuationOptions.OnlyOnFaulted); #endif } @@ -705,8 +706,7 @@ private async Task OnMessageReceived(SocketMessage message) private static Task LogAsync(LogMessage log) { - Console.WriteLine(log.ToString()); - System.Diagnostics.Debug.WriteLine(log.ToString()); + s_log.LogDebug("Discord: {DiscordLogMessage}", log.ToString()); return Task.CompletedTask; } diff --git a/GenOnlineService/ExternalLeaderboardPublicationQueue.cs b/GenOnlineService/ExternalLeaderboardPublicationQueue.cs index 5efe8b1..2d9588d 100644 --- a/GenOnlineService/ExternalLeaderboardPublicationQueue.cs +++ b/GenOnlineService/ExternalLeaderboardPublicationQueue.cs @@ -21,10 +21,12 @@ public sealed class ExternalLeaderboardPublicationWorker : BackgroundService private const int c_MaxBatchSize = 8; private readonly IDbContextFactory _dbFactory; + private readonly ILogger _logger; - public ExternalLeaderboardPublicationWorker(IDbContextFactory dbFactory) + public ExternalLeaderboardPublicationWorker(IDbContextFactory dbFactory, ILogger logger) { _dbFactory = dbFactory; + _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -43,8 +45,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } catch (Exception ex) { - Console.WriteLine($"[ERROR] External leaderboard publication poll failed: {ex}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "External leaderboard publication poll failed"); } try @@ -86,15 +87,30 @@ private async Task PublishPendingMatches(CancellationToken stoppingToken) } catch (Exception ex) { - Console.WriteLine($"[ERROR] External publication bookkeeping failed for match {item.MatchId}: {ex}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "External publication bookkeeping failed for match {MatchId}", item.MatchId); throw; } } + await RecordBacklogAge(stoppingToken); return pending.Count; } + // separate from publishing so a telemetry query failure never blocks it + private async Task RecordBacklogAge(CancellationToken stoppingToken) + { + try + { + await using AppDbContext db = await _dbFactory.CreateDbContextAsync(stoppingToken); + DateTime? oldest = await Database.MatchHistory.GetOldestUnpublishedMatchTime(db, stoppingToken); + AppMetrics.RecordExternalPublicationBacklogAge(oldest is { } finishedAt ? DateTime.UtcNow - finishedAt : TimeSpan.Zero); + } + catch (Exception ex) when (!stoppingToken.IsCancellationRequested) + { + _logger.LogWarning(ex, "External publication backlog age could not be read"); + } + } + private async Task PublishItem( Database.ExternalPublicationWorkItem item, CancellationToken stoppingToken) @@ -124,8 +140,7 @@ private async Task PublishItem( } catch (Exception ex) { - Console.WriteLine($"[WARNING] Match {item.MatchId} was ingested, but its optional ratings response could not be applied: {ex}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "Match {MatchId} was ingested, but its optional ratings response could not be applied", item.MatchId); } await using AppDbContext completionDb = await _dbFactory.CreateDbContextAsync(stoppingToken); @@ -141,7 +156,7 @@ await Database.MatchHistory.MarkExternalPublicationSucceeded( } catch (Exception ex) { - Console.WriteLine($"[WARNING] Failed to publish match {item.MatchId} to the external leaderboard (attempt {item.Attempt}): {ex.Message}"); + _logger.LogWarning(ex, "Failed to publish match {MatchId} to the external leaderboard on attempt {Attempt}", item.MatchId, item.Attempt); await RecordFailure(item, ex, stoppingToken); } } @@ -155,7 +170,7 @@ private async Task ApplyRatingsResponse( if (string.IsNullOrWhiteSpace(responseBody)) { if (matchEntry.lobby_type == ELobbyType.QuickMatch) - Console.WriteLine($"[WARNING] External Match Ingest response for QuickMatch {matchId} contained no ratings body."); + _logger.LogWarning("External Match Ingest response for QuickMatch {MatchId} contained no ratings body", matchId); return; } @@ -166,14 +181,13 @@ private async Task ApplyRatingsResponse( } catch (JsonException ex) { - Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} could not be deserialized: {ex.Message}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "External Match Ingest response for match {MatchId} could not be deserialized", matchId); return; } if (refreshResponse?.data == null) { - Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained no ratings data."); + _logger.LogWarning("External Match Ingest response for match {MatchId} contained no ratings data", matchId); return; } @@ -187,13 +201,13 @@ private async Task ApplyRatingsResponse( { if (!expectedPlayerIds.Contains(userId)) { - Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained unexpected player_id {userId}; skipping (ELO left unchanged)."); + _logger.LogWarning("External Match Ingest response for match {MatchId} contained unexpected player ID {UserId}; skipping", matchId, userId); continue; } if (updatedPlayer?.overall == null || updatedPlayer.season == null) { - Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} contained incomplete ratings for player_id {userId}; skipping."); + _logger.LogWarning("External Match Ingest response for match {MatchId} contained incomplete ratings for player ID {UserId}; skipping", matchId, userId); continue; } @@ -223,7 +237,7 @@ private async Task ApplyRatingsResponse( if (saved) savedUpdates.Add((userId, data)); else - Console.WriteLine($"[WARNING] External Match Ingest response for match {matchId} referenced missing user_id {userId}; skipping."); + _logger.LogWarning("External Match Ingest response for match {MatchId} referenced missing user ID {UserId}; skipping", matchId, userId); } await transaction.CommitAsync(stoppingToken); @@ -260,10 +274,10 @@ await Database.MatchHistory.MarkExternalPublicationFailed( publicationException.Message, stoppingToken); + AppMetrics.RecordExternalPublicationFailure(retry ? "retrying" : "exhausted"); if (!retry) { - Console.WriteLine($"[ERROR] External publication for match {item.MatchId} stopped after {item.Attempt} attempts."); - SentrySdk.CaptureException(publicationException); + _logger.LogError(publicationException, "External publication for match {MatchId} stopped after {Attempt} attempts", item.MatchId, item.Attempt); } } diff --git a/GenOnlineService/ExternalLeaderboardsClient.cs b/GenOnlineService/ExternalLeaderboardsClient.cs index af697f3..1968183 100644 --- a/GenOnlineService/ExternalLeaderboardsClient.cs +++ b/GenOnlineService/ExternalLeaderboardsClient.cs @@ -33,6 +33,7 @@ public class EloRefreshRating public static class ExternalLeaderboardsClient { + private static readonly ILogger s_log = AppLog.For(typeof(ExternalLeaderboardsClient)); private static IConfigurationSection GetExternalLeaderboardsConfigSection() { if (Program.g_Config == null) @@ -145,7 +146,7 @@ public static async Task PostMatchResultAsync(MatchHistory_Entry matchEn { sw.Stop(); - Console.WriteLine($"[INFO] External Match Ingest POST Response for match {matchID} was received in {sw.ElapsedMilliseconds}ms (status: {response.StatusCode})."); + s_log.LogDebug("External Match Ingest POST response for match {MatchId} received in {ElapsedMilliseconds} ms with status {StatusCode}", matchID, sw.ElapsedMilliseconds, response.StatusCode); responseBody = await response.Content.ReadAsStringAsync(cancellationToken); if (!response.IsSuccessStatusCode) @@ -182,11 +183,11 @@ public static async Task PostMatchResultAsync(MatchHistory_Entry matchEn using (var response = await client.SendAsync(request)) { sw.Stop(); - Console.WriteLine($"[INFO] External ELO API call for player {playerId} took {sw.ElapsedMilliseconds}ms (status: {response.StatusCode})."); + s_log.LogDebug("External ELO API call for player {PlayerId} took {ElapsedMilliseconds} ms with status {StatusCode}", playerId, sw.ElapsedMilliseconds, response.StatusCode); if (!response.IsSuccessStatusCode) { - Console.WriteLine($"[ERROR] External ELO API call failed for player {playerId} with status: {response.StatusCode}"); + s_log.LogError("External ELO API call failed for player {PlayerId} with status {StatusCode}", playerId, response.StatusCode); return null; } @@ -194,7 +195,7 @@ public static async Task PostMatchResultAsync(MatchHistory_Entry matchEn var result = JsonSerializer.Deserialize(responseBody); if (result?.data == null || !result.data.TryGetValue(playerId, out var entry)) { - Console.WriteLine($"[ERROR] External ELO API response for player {playerId} did not contain that player_id or could not be deserialized: {responseBody}"); + s_log.LogError("External ELO API response for player {PlayerId} did not contain that player ID or could not be deserialized", playerId); return null; } @@ -205,7 +206,7 @@ public static async Task PostMatchResultAsync(MatchHistory_Entry matchEn } catch (Exception ex) { - Console.WriteLine($"[ERROR] Exception during external ELO API call for player {playerId}: {ex.Message}"); + s_log.LogError(ex, "Exception during external ELO API call for player {PlayerId}", playerId); return null; } } diff --git a/GenOnlineService/GenOnlineService.csproj b/GenOnlineService/GenOnlineService.csproj index 7ea8fd5..626d4c9 100644 --- a/GenOnlineService/GenOnlineService.csproj +++ b/GenOnlineService/GenOnlineService.csproj @@ -47,9 +47,16 @@ - + + + + + + + + diff --git a/GenOnlineService/LobbyManager.Membership.cs b/GenOnlineService/LobbyManager.Membership.cs new file mode 100644 index 0000000..245f583 --- /dev/null +++ b/GenOnlineService/LobbyManager.Membership.cs @@ -0,0 +1,177 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU Affero General Public License for more details. +** +** You should have received a copy of the GNU Affero General Public License +** along with this program. If not, see . +*/ + +using Database; +using Microsoft.EntityFrameworkCore; + +namespace GenOnlineService; + +public enum LobbyMembershipStatus +{ + Success, + NoChange, + ActorNotMember, + ActorNotHost, + NotMatched, + AnticheatMismatch, + SessionUnavailable, + PasswordMismatch, + NotInSetup, + AlreadyMember, + NoOpenSlot, + BlockedByOwner, + FriendshipRequired, + PreferencesUnavailable +} + +public sealed record JoinLobbyCommand(long LobbyID, long UserID, EUserSessionType SessionType, + ushort PreferredPort, bool HasMap, EKnownAnticheatID AnticheatID, string? Password); +public sealed record LeaveLobbyCommand(long LobbyID, long UserID, EUserSessionType SessionType); +public sealed record KickLobbyMemberCommand(long LobbyID, long ActorUserID, long? TargetUserID); +public sealed record JoinLobbyResult(LobbyMembershipStatus Status, string TurnUsername = "", string TurnToken = ""); + +public partial class LobbyManager +{ + // Bounded stripes serialize a user's membership transitions across lobbies. Always + // acquire this before a lobby gate; never acquire another stripe while holding one. + private readonly SemaphoreSlim[] membershipGates = Enumerable.Range(0, 256).Select(_ => new SemaphoreSlim(1, 1)).ToArray(); + + private async Task AcquireMembershipGateAsync(long userID) + { + var gate = membershipGates[(int)((ulong)userID % (uint)membershipGates.Length)]; + await gate.WaitAsync(); + return new MembershipLease(gate); + } + + private sealed class MembershipLease(SemaphoreSlim gate) : IDisposable + { + public void Dispose() => gate.Release(); + } + + private readonly IDbContextFactory membershipDbFactory; + private readonly LobbyMembershipConnections membershipConnections; + + public async Task JoinAsync(JoinLobbyCommand command) + { + using var membership = await AcquireMembershipGateAsync(command.UserID); + var lobby = GetLobby(command.LobbyID); + if (lobby == null) return new(LobbyMembershipStatus.NoChange); + if (command.UserID == -1 || !SessionHelpers.SessionTypeHasAccessTo(command.SessionType, ESessionAccessType.Gameplay)) + return new(LobbyMembershipStatus.NoChange); + + var session = membershipConnections.GetSession(command.UserID, command.SessionType); + if (session == null || session.IsRetired) return new(LobbyMembershipStatus.SessionUnavailable); + + await using var db = await membershipDbFactory.CreateDbContextAsync(); + var displayName = await Users.GetDisplayName(db, command.UserID); + var preferences = await Users.GetUserLobbyPreferences(db, command.UserID); + if (preferences == null) return new(LobbyMembershipStatus.PreferencesUnavailable); + var sources = m_dictLobbies.Values.Where(source => source.GetMemberFromUserID(command.UserID) != null).ToArray(); + var status = await Lobby.TransferMemberAsync(lobby, sources, command, session, displayName, preferences); + if (status != LobbyMembershipStatus.Success) return new(status); + + var credentials = await membershipConnections.CreateRelayCredentialsAsync(command.UserID); + return new(status, credentials.Username, credentials.Token); + } + + public async Task LeaveAsync(LeaveLobbyCommand command) + { + if (command.UserID == -1 || !SessionHelpers.SessionTypeHasAccessTo(command.SessionType, ESessionAccessType.Gameplay)) + return false; + using var membership = await AcquireMembershipGateAsync(command.UserID); + var lobby = GetLobby(command.LobbyID); + if (lobby == null) return false; + return await RemoveMembershipAsync(lobby, command.UserID) != null; + } + + public async Task KickAsync(KickLobbyMemberCommand command) + { + using var membership = await AcquireMembershipGateAsync(command.TargetUserID ?? command.ActorUserID); + var lobby = GetLobby(command.LobbyID); + if (lobby == null) return LobbyMembershipStatus.NoChange; + var status = await lobby.KickMemberAsync(command); + if (status == LobbyMembershipStatus.Success && command.TargetUserID is long targetID) + { + await ClearSessionMembershipStateAsync(targetID, EUserSessionType.GameClient); + // Removed members no longer receive ordinary dirty lobby updates. + await lobby.DirtyRetransmitToSingleMember(targetID); + } + return status; + } + + private async Task ClearSessionMembershipStateAsync(long userID, EUserSessionType sessionType) + { + try + { + await membershipConnections.DeleteRelayCredentialsAsync(userID); + } + finally + { + membershipConnections.GetSession(userID, sessionType)?.UpdateSessionLobbyID(-1); + } + // Match-history membership is owned by the match-history service. + } + + internal async Task RemoveMatchmakingMemberAsync(Lobby lobby, LobbyMember member, bool sendPeerTeardown = false) + { + using var membership = await AcquireMembershipGateAsync(member.UserID); + if (!member.GetSession().TryGetTarget(out var session) || !IsCurrentSession(session) || session.IsRetired) return; + if (sendPeerTeardown) lobby.SendPeerTeardownToDepartingMember(member); + await RemoveMembershipAsync(lobby, member.UserID, member); + } + + // Caller holds the user's membership gate through removal and session cleanup. + private async Task RemoveMembershipAsync(Lobby lobby, long userID, LobbyMember? expectedMember = null) + { + var removed = await lobby.RemoveMemberAsync(userID, expectedMember); + if (removed != null && removed.GetSession().TryGetTarget(out var session) + && IsCurrentSession(session) && !session.IsRetired && session.currentLobbyID == lobby.LobbyID) + await ClearSessionMembershipStateAsync(userID, session.GetSessionType()); + return removed; + } + + internal async Task TryRequeueMatchmakingAsync(UserSession session, long expectedLobbyID, byte[] requeueActionJSON) + { + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session) || session.IsRetired + || (session.currentLobbyID != -1 && session.currentLobbyID != expectedLobbyID)) return false; + if (session.currentLobbyID == expectedLobbyID && expectedLobbyID != -1) + await ClearSessionMembershipStateAsync(session.m_UserID, session.GetSessionType()); + return await MatchmakingManager.TryRequeueRegisteredPlayer(session, requeueActionJSON); + } + + public async Task LeaveAllLobbiesAsync(long userID) + { + using var membership = await AcquireMembershipGateAsync(userID); + await LeaveMembershipsAsync(userID); + } + + private async Task LeaveMembershipsAsync(long userID) + { + foreach (Lobby lobby in m_dictLobbies.Values) + await RemoveMembershipAsync(lobby, userID); + } + internal async Task<(string Username, string Token)> CreateLobbyRelayCredentialsAsync(UserSession session, long lobbyID) + { + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session) || session.IsRetired || session.currentLobbyID != lobbyID) + return ("", ""); + return await membershipConnections.CreateRelayCredentialsAsync(session.m_UserID); + } + +} diff --git a/GenOnlineService/LobbyManager.Sessions.cs b/GenOnlineService/LobbyManager.Sessions.cs new file mode 100644 index 0000000..723962e --- /dev/null +++ b/GenOnlineService/LobbyManager.Sessions.cs @@ -0,0 +1,152 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU Affero General Public License for more details. +** +** You should have received a copy of the GNU Affero General Public License +** along with this program. If not, see . +*/ + +using Database; +using System.Net.WebSockets; + +namespace GenOnlineService; + +public partial class LobbyManager +{ + internal async Task OpenSessionAsync(AppDbContext db, EUserSessionType sessionType, bool reconnect, + long userID, KnownClients.EKnownClients clientID, string ipAddress, string continent, string country, + double latitude, double longitude, bool admin) + { + using var membership = await AcquireMembershipGateAsync(userID); + var previous = membershipConnections.GetSession(userID, sessionType); + if (!reconnect && previous != null) await DestroySessionAsync(previous); + if (reconnect && (previous == null || previous.IsRetired)) return null; + var installed = await WebSocketManager.InstallSession(db, sessionType, reconnect, userID, clientID, + ipAddress, continent, country, latitude, longitude, admin); + if (reconnect && previous != null) await InvalidateSessionMeshAsync(previous); + return installed; + } + + internal async Task DisconnectSessionAsync(UserSession? session, UserWebSocketInstance? socket, bool destroySession) + { + if (session == null) return; + using (await AcquireMembershipGateAsync(session.m_UserID)) + { + if (IsCurrentSession(session) && ReferenceEquals(WebSocketManager.GetWebSocketForSession(session), socket)) + { + if (destroySession) await DestroySessionAsync(session); + else + { + WebSocketManager.DetachSocket(session); + await InvalidateSessionMeshAsync(session); + session.MarkAbandoned(); + var lobby = GetLobby(session.currentLobbyID); + if (lobby?.State == ELobbyState.INGAME) lobby.RecordPlayerIngameAbandon(session.m_UserID); + } + } + if (socket != null) await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Session being deleted"); + } + } + + internal Task ClearSessionAsync(UserSession? session) => RemoveSessionAsync(session, expiredOnly: false); + internal Task ExpireSessionAsync(UserSession session) => RemoveSessionAsync(session, expiredOnly: true); + + private async Task RemoveSessionAsync(UserSession? session, bool expiredOnly) + { + if (session == null) return false; + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session)) return false; + if (expiredOnly && !WebSocketManager.ShouldStillClearAbandonedSession(session, session)) return false; + await DestroySessionAsync(session); + return true; + } + + private bool IsCurrentSession(UserSession session) => + ReferenceEquals(membershipConnections.GetSession(session.m_UserID, session.GetSessionType()), session); + + private async Task DestroySessionAsync(UserSession session) + { + session.Retire(); + bool gameplay = SessionHelpers.SessionTypeHasAccessTo(session.GetSessionType(), ESessionAccessType.Gameplay); + try + { + if (gameplay) + { + await InvalidateSessionMeshAsync(session); + await MatchmakingManager.DetachSessionAsync(session); + await LeaveMembershipsAsync(session.m_UserID); + await CleanupUserLobbiesNotStartedAsync(session.m_UserID); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Destroying session for user {UserId} failed", session.m_UserID); + throw; + } + finally + { + try + { + if (gameplay) await membershipConnections.DeleteRelayCredentialsAsync(session.m_UserID); + } + finally + { + var socket = WebSocketManager.DetachSocket(session); + try + { + WebSocketManager.RemoveSessionData(session); + } + finally + { + // Send loops never invoke lifecycle operations. Stop the old queue reader + // before allowing a reconnect to attach another reader to the same session. + if (socket != null) await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Session replaced or expired"); + } + } + } + } + + private async Task InvalidateSessionMeshAsync(UserSession session) + { + if (!SessionHelpers.SessionTypeHasAccessTo(session.GetSessionType(), ESessionAccessType.Gameplay)) return; + var lobby = GetLobby(session.currentLobbyID); + if (lobby != null) await lobby.InvalidateMeshForSessionAsync(session); + } + + internal async Task StartMeshCheckAsync(UserSession session, UserWebSocketInstance socket) + { + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session) || session.IsRetired || !ReferenceEquals(WebSocketManager.GetWebSocketForSession(session), socket)) return; + var lobby = GetLobby(session.currentLobbyID); + if (lobby != null) await lobby.StartMeshCheckAsync(session, socket); + } + + internal async Task StoreMeshResponseAsync(UserSession session, UserWebSocketInstance socket, + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + { + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session) || session.IsRetired || !ReferenceEquals(WebSocketManager.GetWebSocketForSession(session), socket)) return; + var lobby = GetLobby(session.currentLobbyID); + if (lobby != null) await lobby.StoreMeshResponseAsync(session, socket, response); + } + + internal async Task CancelMatchmakingAsync(UserSession session) + { + using var membership = await AcquireMembershipGateAsync(session.m_UserID); + if (!IsCurrentSession(session) || session.IsRetired) return; + var detached = await MatchmakingManager.DetachSessionAsync(session); + foreach (var entry in detached.Members) + await RemoveMembershipAsync(entry.Lobby, session.m_UserID, entry.Member); + if (!detached.CancellationRejected) await LeaveMembershipsAsync(session.m_UserID); + } +} diff --git a/GenOnlineService/LobbyManager.cs b/GenOnlineService/LobbyManager.cs index 5a677d7..86ac1e8 100644 --- a/GenOnlineService/LobbyManager.cs +++ b/GenOnlineService/LobbyManager.cs @@ -66,18 +66,18 @@ internal static bool ShouldRetry(bool meshComplete, bool hasLegacyResponse, int internal static class FullMeshCheckSettings { // how long an attempt waits for every connection before judging it - internal static int AttemptWindowMS => Get("full_mesh_check_attempt_window_ms", 8000); + internal static int AttemptWindowMs => Get("full_mesh_check_attempt_window_ms", 8000); // how often an incomplete attempt asks members for a fresh snapshot - internal static int SnapshotIntervalMS => Get("full_mesh_check_snapshot_interval_ms", 1000); + internal static int SnapshotIntervalMs => Get("full_mesh_check_snapshot_interval_ms", 1000); // time given to re-signalled connections before the next attempt - internal static int RetryDelayMS => Get("full_mesh_check_retry_delay_ms", 3000); + internal static int RetryDelayMs => Get("full_mesh_check_retry_delay_ms", 3000); internal static int MaxAttempts => Get("full_mesh_check_max_attempts", 2); // upper bound for a whole check, used as the clients' setup timeout - internal static int MaxDurationMS => (AttemptWindowMS * MaxAttempts) + (RetryDelayMS * (MaxAttempts - 1)); + internal static int MaxDurationMs => (AttemptWindowMs * MaxAttempts) + (RetryDelayMs * (MaxAttempts - 1)); private static int Get(string key, int defaultValue) { @@ -88,6 +88,8 @@ private static int Get(string key, int defaultValue) public class Lobby { + private static readonly ILogger s_log = AppLog.For(typeof(Lobby)); + public Int64 LobbyID { get; private set; } = -1; public Int64 Owner { get; private set; } = -1; public string Name { get; private set; } = ""; @@ -111,7 +113,7 @@ public class Lobby // SemaphoreSlim, so there is exactly one lock to reason about and no lock-ordering hazard // between the two. It is NOT reentrant: a callback passed to RunExclusiveAsync must never // call back into RunExclusiveAsync on this same lobby, or it will deadlock - callers that are - // already inside a callback (e.g. DoHostMigration, called from RemoveMember's callback) must + // already inside a callback (e.g. DoHostMigration, called from RemoveMemberAsync's callback) must // mutate state directly instead. Sending a websocket message from inside a callback is safe: // QueueWebsocketSend only enqueues bytes onto an in-memory bounded channel (a non-blocking // TryWrite) - it never performs real network I/O itself, so it cannot block while the gate is @@ -262,19 +264,108 @@ public async Task RegisterProbeResponse_Malformed_Type2(Int64 userID) } // End AC Probes - public async Task StartFullMeshConnectivityCheck() + private sealed record MeshParticipant(LobbyMember Member, UserSession Session, UserWebSocketInstance? Socket); + private Dictionary m_MeshParticipants = new(); + private MeshParticipant? m_MeshHost; + private long m_MembershipVersion; + private long m_MembershipVersionAtMeshCheckStart; + [JsonIgnore] + public string? LastFullMeshCheckFailureReason { get; private set; } + + public Task StartMeshCheckAsync(UserSession? requester = null, UserWebSocketInstance? socket = null) { - await RunExclusiveAsync(() => + return RunExclusiveAsync(() => { + var host = GetMemberFromUserID(Owner); + if (State != ELobbyState.GAME_SETUP || host == null || !host.GetSession().TryGetTarget(out var hostSession) + || (requester != null && (!ReferenceEquals(hostSession, requester) + || !ReferenceEquals(WebSocketManager.GetWebSocketForSession(requester), socket)))) + return Task.FromResult(0L); + + if (requester != null) + { + foreach (var member in Members) + if (member.SlotState == EPlayerType.SLOT_OPEN) member.SetPlayerSlotState(EPlayerType.SLOT_CLOSED); + DirtyRetransmit(); + } FullMeshCheckID = Interlocked.Increment(ref s_NextFullMeshCheckID); FullMeshCheckAttempt = 1; + m_MembershipVersionAtMeshCheckStart = m_MembershipVersion; + m_MeshParticipants = new(); + foreach (var member in Members.Where(member => member.IsHuman())) + if (member.GetSession().TryGetTarget(out var session)) + m_MeshParticipants[member.UserID] = new(member, session, WebSocketManager.GetWebSocketForSession(session)); + m_MeshHost = m_MeshParticipants.GetValueOrDefault(Owner); m_TimeToRetryFullMeshChecks = -1; LastFullMeshConnectivityCheckOutcome = null; + LastFullMeshCheckFailureReason = null; BeginFullMeshConnectivityCheckAttempt(); - return Task.CompletedTask; + if (EnsureMeshCheckIsCurrentLocked()) SendFullMeshConnectivityCheckRequestToMembers(); + return Task.FromResult(FullMeshCheckID); }); } + private bool IsCurrentMeshParticipant(MeshParticipant participant) => + ReferenceEquals(GetMemberFromUserID(participant.Session.m_UserID), participant.Member) + && participant.Member.GetSession().TryGetTarget(out var session) && ReferenceEquals(session, participant.Session) + && !session.IsRetired && participant.Socket?.IsOpen == true + && ReferenceEquals(WebSocketManager.GetWebSocketForSession(session), participant.Socket); + + private bool EnsureMeshCheckIsCurrentLocked() + { + if (FullMeshCheckID == 0) return false; + string? reason = State != ELobbyState.GAME_SETUP ? "lobby_state_changed" + : m_MembershipVersion != m_MembershipVersionAtMeshCheckStart + || m_MeshParticipants.Count != GetNumberOfHumans() ? "membership_changed" + : m_MeshParticipants.Values.Any(participant => !IsCurrentMeshParticipant(participant)) ? "session_changed" + : m_MeshHost == null || Owner != m_MeshHost.Session.m_UserID ? "host_changed" : null; + if (reason == null) return true; + InvalidateMeshCheck(reason); + return false; + } + + private void InvalidateMeshCheck(string reason) + { + if (PendingFullMeshConnectivityChecks) CompleteMeshCheck(false, reason, new()); + else if (FullMeshCheckID != 0) + { + LastFullMeshConnectivityCheckOutcome = false; + LastFullMeshCheckFailureReason = reason; + } + } + + internal Task InvalidateMeshForSessionAsync(UserSession session) => RunExclusiveAsync(() => + { + if (m_MeshParticipants.Values.Any(participant => ReferenceEquals(participant.Session, session))) + InvalidateMeshCheck("session_changed"); + return Task.CompletedTask; + }); + + public Task GetValidatedMeshCheckResultAsync(long checkID) => RunExclusiveAsync(() => + { + EnsureMeshCheckIsCurrentLocked(); + return Task.FromResult(checkID == FullMeshCheckID && checkID != 0 + ? LastFullMeshConnectivityCheckOutcome : (bool?)false); + }); + + private void CompleteMeshCheck(bool success, string? reason, List missing) + { + if (!PendingFullMeshConnectivityChecks) return; + PendingFullMeshConnectivityChecks = false; + TimeStartFullMeshChecks = -1; + m_TimeToRetryFullMeshChecks = -1; + LastFullMeshConnectivityCheckOutcome = success; + LastFullMeshCheckFailureReason = reason; + if (m_MeshHost == null || Owner != m_MeshHost.Session.m_UserID || !IsCurrentMeshParticipant(m_MeshHost)) return; + var outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome + { + msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST, + mesh_check_id = FullMeshCheckID, attempt = FullMeshCheckAttempt, + mesh_complete = success, reason = reason ?? string.Empty, missing_connections = missing + }; + m_MeshHost.Session.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(outcome))); + } + private void BeginFullMeshConnectivityCheckAttempt() { PendingFullMeshConnectivityChecks = true; @@ -282,7 +373,7 @@ private void BeginFullMeshConnectivityCheckAttempt() m_FullMeshConnecting = new(); m_bCurrentAttemptHasLegacyResponse = false; TimeStartFullMeshChecks = Environment.TickCount64; - m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMS; + m_TimeNextFullMeshSnapshotRequest = TimeStartFullMeshChecks + FullMeshCheckSettings.SnapshotIntervalMs; } private bool IsFullMeshPairStillConnecting(Int64 userA, Int64 userB) @@ -305,13 +396,13 @@ private void LogFullMeshCheckAttempt(bool bMeshComplete, List $"{p.Item1}<->{p.Item2}")); - Console.WriteLine("[Lobby {0}] Mesh check {1} attempt {2}/{3}: {4} after {5} ms with {6} humans{7}", + s_log.LogInformation("Lobby {LobbyId}: mesh check {MeshCheckId} attempt {Attempt}/{MaxAttempts} {MeshResult} after {ElapsedMs} ms with {HumanCount} humans{MeshDetail}", LobbyID, FullMeshCheckID, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts, bMeshComplete ? "complete" : "incomplete", elapsedMS, GetNumberOfHumans(), bMeshComplete ? "" : $", missing {strMissing}{(strConnecting.Length > 0 ? $" (still connecting {strConnecting})" : "")}"); } - public void SendFullMeshConnectivityCheckRequestToMembers() + private void SendFullMeshConnectivityCheckRequestToMembers() { WebSocketMessage_FullMeshConnectivityCheckRequest startCommand = new WebSocketMessage_FullMeshConnectivityCheckRequest(); startCommand.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE; @@ -352,7 +443,7 @@ private void RestartSignallingForMissingConnections(List continue; } - Console.WriteLine("[Lobby {0}] Re-signalling {1} <-> {2} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); + s_log.LogDebug("Lobby {LobbyId}: re-signalling {SourceUserId} <-> {TargetUserId} before mesh check retry", LobbyID, sourceMember.UserID, targetMember.UserID); SendStartSignallingToMember(sourceMember, targetMember); SendStartSignallingToMember(targetMember, sourceMember); @@ -373,31 +464,21 @@ private void SendStartSignallingToMember(LobbyMember recipient, LobbyMember peer } } - public async Task StoreFullMeshConnectivityResponse(Int64 sourceUser, WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) + public Task StoreMeshResponseAsync(UserSession source, UserWebSocketInstance socket, + WebSocketMessage_FullMeshConnectivityCheckResponseFromUser response) => RunExclusiveAsync(() => { - await RunExclusiveAsync(() => - { - bool bLegacyResponse = FullMeshCheckProtocol.IsLegacyResponse(response); - bool bMatchesCurrentAttempt = FullMeshCheckProtocol.MatchesCurrentAttempt( - response, - FullMeshCheckID, - FullMeshCheckAttempt); - - LobbyMember? sourceMember = GetMemberFromUserID(sourceUser); - if (PendingFullMeshConnectivityChecks - && m_TimeToRetryFullMeshChecks == -1 - && sourceMember?.IsHuman() == true - && bMatchesCurrentAttempt) - { - m_bCurrentAttemptHasLegacyResponse |= bLegacyResponse; - FullMeshConnectivityChecks[sourceUser] = new ConcurrentList(response.connectivity_map); - m_FullMeshConnecting[sourceUser] = new HashSet(response.connecting_map); - } - - ProcessPendingFullMeshConnectivityChecksInternal(); + // Legacy zero IDs cannot distinguish successive checks on the same socket. + if (!EnsureMeshCheckIsCurrentLocked() || !PendingFullMeshConnectivityChecks || m_TimeToRetryFullMeshChecks != -1 + || !m_MeshParticipants.TryGetValue(source.m_UserID, out var participant) + || !ReferenceEquals(participant.Session, source) || !ReferenceEquals(participant.Socket, socket) + || !FullMeshCheckProtocol.MatchesCurrentAttempt(response, FullMeshCheckID, FullMeshCheckAttempt)) return Task.CompletedTask; - }); - } + m_bCurrentAttemptHasLegacyResponse |= FullMeshCheckProtocol.IsLegacyResponse(response); + FullMeshConnectivityChecks[source.m_UserID] = new(response.connectivity_map ?? new()); + m_FullMeshConnecting[source.m_UserID] = new(response.connecting_map ?? new()); + ProcessPendingFullMeshConnectivityChecksInternal(); + return Task.CompletedTask; + }); public async Task ProcessPendingFullMeshConnectivityChecks() { @@ -411,172 +492,43 @@ await RunExclusiveAsync(() => // Must only be called while holding m_LobbyGate (via RunExclusiveAsync). private void ProcessPendingFullMeshConnectivityChecksInternal() { - if (!PendingFullMeshConnectivityChecks) - { - return; - } - - // Give re-signalled connections time to establish before starting the retry. + if (!EnsureMeshCheckIsCurrentLocked() || !PendingFullMeshConnectivityChecks) return; if (m_TimeToRetryFullMeshChecks != -1) { - if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) - { - return; - } - + if (Environment.TickCount64 < m_TimeToRetryFullMeshChecks) return; m_TimeToRetryFullMeshChecks = -1; BeginFullMeshConnectivityCheckAttempt(); SendFullMeshConnectivityCheckRequestToMembers(); return; } - { - bool bDoneChecks = false; - - // judged as soon as anyone replies: members who haven't replied yet count as missing, and gaps before - // the window closes are re-polled below, which also recovers replies lost to a reconnect - bool bWindowElapsed = (Environment.TickCount64 - TimeStartFullMeshChecks) >= FullMeshCheckSettings.AttemptWindowMS; - bDoneChecks = bWindowElapsed || !FullMeshConnectivityChecks.IsEmpty; - - List lstMissingConnections = new(); - - if (bDoneChecks) + var missing = new List(); + foreach (var source in m_MeshParticipants.Keys) + foreach (var target in m_MeshParticipants.Keys) + if (source != target && (!FullMeshConnectivityChecks.TryGetValue(source, out var connected) || !connected.Contains(target))) + missing.Add(new() { source_user_id = source, target_user_id = target }); + bool disabled = Program.g_Config?.GetSection("Core").GetValue("disable_full_mesh_check") == true; + bool allReported = m_MeshParticipants.Keys.All(FullMeshConnectivityChecks.ContainsKey); + bool complete = disabled || (allReported && missing.Count == 0); + bool attemptTimedOut = Environment.TickCount64 - TimeStartFullMeshChecks >= FullMeshCheckSettings.AttemptWindowMs; + if (!complete && !attemptTimedOut) + { + if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) { - // now verify each user has provided data for all other users - foreach (var userMap in FullMeshConnectivityChecks) - { - // a member who left mid-check no longer counts - if (GetMemberFromUserID(userMap.Key)?.IsHuman() != true) - { - continue; - } - - // foreach member in the lobby, check they are in userMap.Value - foreach (LobbyMember member in Members) - { - if (member.IsHuman()) - { - // useful for test -// if (member.UserID != userMap.Key && member.UserID == 1) -// { -// // register it -// MissingConnectionEntry missingConnectionEntry = new(); -// missingConnectionEntry.source_user_id = userMap.Key; -// missingConnectionEntry.target_user_id = member.UserID; -// lstMissingConnections.Add(missingConnectionEntry); -// } - - // wont have a connection to ourself - if (member.UserID != userMap.Key && !userMap.Value.Contains(member.UserID)) - { - // register it - MissingConnectionEntry missingConnectionEntry = new(); - missingConnectionEntry.source_user_id = userMap.Key; - missingConnectionEntry.target_user_id = member.UserID; - lstMissingConnections.Add(missingConnectionEntry); - } - } - } - } - - // A member that never reported cannot be assumed connected, so treat them as missing to everyone. - foreach (LobbyMember member in Members) - { - if (member.IsHuman() && !FullMeshConnectivityChecks.ContainsKey(member.UserID)) - { - foreach (LobbyMember otherMember in Members) - { - if (otherMember.IsHuman() && otherMember.UserID != member.UserID) - { - MissingConnectionEntry missingConnectionEntry = new(); - missingConnectionEntry.source_user_id = member.UserID; - missingConnectionEntry.target_user_id = otherMember.UserID; - lstMissingConnections.Add(missingConnectionEntry); - } - } - } - } - - bool bDisableMeshCheck = false; - if (Program.g_Config != null) - { - IConfiguration? coreSettings = Program.g_Config.GetSection("Core"); - - if (coreSettings != null) - { - bDisableMeshCheck = coreSettings.GetValue("disable_full_mesh_check"); - } - } - - - - bool bMeshComplete = bDisableMeshCheck || lstMissingConnections.Count == 0; - - // members report a snapshot, so a gap before the window closes may be a connection still forming: - // keep asking for fresh snapshots rather than re-signalling it early - if (!bMeshComplete && !bWindowElapsed) - { - if (Environment.TickCount64 >= m_TimeNextFullMeshSnapshotRequest) - { - m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMS; - SendFullMeshConnectivityCheckRequestToMembers(); - } - - return; - } - - LogFullMeshCheckAttempt(bMeshComplete, lstMissingConnections); - - if (FullMeshCheckProtocol.ShouldRetry( - bMeshComplete, - m_bCurrentAttemptHasLegacyResponse, - FullMeshCheckAttempt, - FullMeshCheckSettings.MaxAttempts)) - { - ++FullMeshCheckAttempt; - - // a pair still negotiating would be torn down by a re-signal; the next attempt re-checks it - RestartSignallingForMissingConnections(lstMissingConnections - .Where(c => !IsFullMeshPairStillConnecting(c.source_user_id, c.target_user_id)) - .ToList()); - m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMS; - return; - } - - // inform host that we are done - // start full mesh connectivity checks - WebSocketMessage_FullMeshConnectivityCheckOutcome outcome = new WebSocketMessage_FullMeshConnectivityCheckOutcome(); - outcome.msg_id = (int)EWebSocketMessageID.FULL_MESH_CONNECTIVITY_CHECK_RESPONSE_COMPLETE_TO_HOST; - - if (bDisableMeshCheck) - { - outcome.mesh_complete = true; - outcome.missing_connections = new List(); - } - else - { - outcome.mesh_complete = lstMissingConnections.Count == 0; - outcome.missing_connections = lstMissingConnections; - } - - LastFullMeshConnectivityCheckOutcome = outcome.mesh_complete; - - // TODO_EFCORE: Later, these should really use lobby list instead of getting session from ID - - // send to host - UserSession? hostSession = WebSocketManager.GetSessionFromUser(Owner, EUserSessionType.GameClient); // host should be a game client - if (hostSession != null) - { - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(outcome)); - hostSession.QueueWebsocketSend(bytesJSON); - } - - // reset state - PendingFullMeshConnectivityChecks = false; - TimeStartFullMeshChecks = -1; - m_TimeToRetryFullMeshChecks = -1; + m_TimeNextFullMeshSnapshotRequest = Environment.TickCount64 + FullMeshCheckSettings.SnapshotIntervalMs; + SendFullMeshConnectivityCheckRequestToMembers(); } + return; } + LogFullMeshCheckAttempt(complete, missing); + if (FullMeshCheckProtocol.ShouldRetry(complete, m_bCurrentAttemptHasLegacyResponse, FullMeshCheckAttempt, FullMeshCheckSettings.MaxAttempts)) + { + ++FullMeshCheckAttempt; + RestartSignallingForMissingConnections(missing.Where(connection => !IsFullMeshPairStillConnecting(connection.source_user_id, connection.target_user_id)).ToList()); + m_TimeToRetryFullMeshChecks = Environment.TickCount64 + FullMeshCheckSettings.RetryDelayMs; + return; + } + CompleteMeshCheck(complete, complete ? null : allReported ? "missing_connections" : "timeout", disabled ? new() : missing); } public void AddPassword(string password) @@ -720,8 +672,8 @@ public int MaxPlayers public ELobbyJoinability LobbyJoinability { get; private set; } = ELobbyJoinability.Public; // public by default - public const int maxLobbySize = 8; - public LobbyMember[] Members { get; private set; } = new LobbyMember[maxLobbySize]; + public const int MaxLobbySize = 8; + public LobbyMember[] Members { get; private set; } = new LobbyMember[MaxLobbySize]; public EKnownAnticheatID AnticheatID { get; private set; } = EKnownAnticheatID.NONE; @@ -755,7 +707,7 @@ public void RecordPlayerIngameAbandon(Int64 userId) DateTime abandonTime = DateTime.UtcNow; if (TimePlayerAbandonedIngame.TryAdd(userId, abandonTime)) { - Console.WriteLine("[Lobby {0}] Recorded in-game abandon for user {1} at {2:O}", LobbyID, userId, abandonTime); + s_log.LogInformation("Lobby {LobbyId}: recorded in-game abandon for user {UserId} at {AbandonTime}", LobbyID, userId, abandonTime); } } @@ -767,7 +719,7 @@ public void ClearPlayerIngameAbandon(Int64 userId) { if (TimePlayerAbandonedIngame.TryRemove(userId, out _)) { - Console.WriteLine("[Lobby {0}] Cleared in-game abandon record for reconnected user {1}", LobbyID, userId); + s_log.LogInformation("Lobby {LobbyId}: cleared in-game abandon record for reconnected user {UserId}", LobbyID, userId); } } @@ -780,7 +732,7 @@ public void RecordPlayerOutcome(Int64 userId, bool bWon) { if (ReportedOutcomes.TryAdd(userId, bWon)) { - Console.WriteLine("[Lobby {0}] Recorded reported outcome for user {1}: won={2}", LobbyID, userId, bWon); + s_log.LogInformation("Lobby {LobbyId}: recorded reported outcome for user {UserId}, won={Won}", LobbyID, userId, bWon); } } @@ -847,11 +799,11 @@ public Lobby(Int64 lobby_id, UserSession owner, string name, ELobbyState state, RNGSeed = rng_seed; MaximumCameraHeight = max_cam_height; LobbyType = lobbyType; - Members = new LobbyMember[maxLobbySize]; + Members = new LobbyMember[MaxLobbySize]; AnticheatID = inAnticheatID; // create default slots - for (UInt16 i = 0; i < maxLobbySize; ++i) + for (UInt16 i = 0; i < MaxLobbySize; ++i) { LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, i < max_players ? EPlayerType.SLOT_OPEN : EPlayerType.SLOT_CLOSED, i, true); Members[i] = placeholderMember; @@ -861,8 +813,8 @@ public Lobby(Int64 lobby_id, UserSession owner, string name, ELobbyState state, public event Action? OnLobbyNeedsDestroyed; - // Must only be called while holding m_LobbyGate (from within RemoveMember's callback): it reads - // and mutates Members/Owner and must observe RemoveMember's slot clear atomically with any + // Must only be called while holding m_LobbyGate (from within RemoveMemberAsync's callback): it reads + // and mutates Members/Owner and must observe RemoveMemberAsync's slot clear atomically with any // concurrent join/leave/migration. private void OnAfterPlayerLeftLocked(Int64 leavingUserID) { @@ -874,9 +826,7 @@ private void OnAfterPlayerLeftLocked(Int64 leavingUserID) if (numHumanMembers == 0) { - Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine("DeleteLobby: Source A"); - Console.ForegroundColor = ConsoleColor.Gray; + s_log.LogInformation("Lobby {LobbyId}: last human member left, requesting destruction", LobbyID); OnLobbyNeedsDestroyed?.Invoke(this); } @@ -934,9 +884,9 @@ public async Task FinalizeACChecks() } // Runs under the per-lobby gate: this mutates slot state and previously ran unguarded, - // racing against AddMember/RemoveMember/other slot updates. Sequential with (never nested - // inside) StartFullMeshConnectivityCheck's own gate use - callers await this first and then - // await StartFullMeshConnectivityCheck as a separate gated operation, so the gate is never + // racing against AddMember/RemoveMemberAsync/other slot updates. Sequential with (never nested + // inside) StartMeshCheckAsync's own gate use - callers await this first and then + // await StartMeshCheckAsync as a separate gated operation, so the gate is never // re-entered. public async Task CloseOpenSlots() { @@ -956,7 +906,7 @@ await RunExclusiveAsync(() => } // Must only be called while holding m_LobbyGate (currently only true from - // OnAfterPlayerLeftLocked, itself only called from within RemoveMember's callback). + // OnAfterPlayerLeftLocked, itself only called from within RemoveMemberAsync's callback). private void DoHostMigration() { Int64 oldOwner = Owner; @@ -972,8 +922,10 @@ private void DoHostMigration() Owner = member.UserID; member.UpdateSlotIndex(0); - Members[0] = member; - Members[oldSlot] = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, oldSlot, true); + var migrated = (LobbyMember[])Members.Clone(); + migrated[0] = member; + migrated[oldSlot] = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, oldSlot, true); + Members = migrated; member.SetReadyState(true); DirtyRetransmitLobbyList(); @@ -1071,7 +1023,7 @@ public async Task Tick() UserSession? sess = WebSocketManager.GetSessionFromUser(session.m_UserID, session.GetSessionType()); if (sess != null) { - Console.WriteLine("[DIRTY LOBBY] Sending WS lobby update for lobby {0}", LobbyID); + s_log.LogDebug("Lobby {LobbyId}: sending WebSocket lobby update", LobbyID); sess.QueueWebsocketSend(bytesJSON); } } @@ -1084,73 +1036,81 @@ public async Task Tick() } } - public async Task AddMember(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) + internal static async Task TransferMemberAsync(Lobby destination, IReadOnlyCollection sources, + JoinLobbyCommand command, UserSession session, string displayName, UserLobbyPreferences preferences) + { + var locked = sources.Append(destination).Distinct().OrderBy(lobby => lobby.LobbyID).ToArray(); + int acquired = 0; + try + { + // The caller holds the user's membership gate. All multi-lobby operations use + // ascending IDs; only synchronous state changes and queue writes follow. + foreach (var lobby in locked) + { + await lobby.m_LobbyGate.WaitAsync(); + ++acquired; + } + var admission = destination.ValidateJoin(command); + if (admission != LobbyMembershipStatus.Success) return admission; + var added = destination.AddMemberLocked(session, displayName, command.PreferredPort, command.HasMap, preferences); + if (added.Member == null) return added.Status; + // Destination admission and insertion must succeed before releasing old membership. + foreach (var source in sources) + { + if (ReferenceEquals(source, destination)) continue; + var member = source.GetMemberFromUserID(command.UserID); + if (member != null && source.RemoveMemberLocked(member)) source.PublishMemberRemoval(member); + } + destination.PublishMemberAddition(session, added.Member); + return LobbyMembershipStatus.Success; + } + finally + { + for (int index = acquired - 1; index >= 0; --index) locked[index].m_LobbyGate.Release(); + } + } + + internal Task TryMarkCompleteIfEmptyAsync(long? expectedOwner = null, bool setupOnly = false) => + RunExclusiveAsync(() => + { + if ((expectedOwner.HasValue && Owner != expectedOwner.Value) || (setupOnly && State != ELobbyState.GAME_SETUP) + || GetNumberOfHumans() != 0) return Task.FromResult(false); + State = ELobbyState.COMPLETE; + DirtyRetransmitLobbyList(); + return Task.FromResult(true); + }); + + internal Task AddCreatorAsync(UserSession session, string displayName, ushort preferredPort, UserLobbyPreferences preferences) => + RunExclusiveAsync(() => + { + var added = AddMemberLocked(session, displayName, preferredPort, true, preferences); + if (added.Member != null) PublishMemberAddition(session, added.Member); + return Task.FromResult(added.Status); + }); + + private (LobbyMembershipStatus Status, LobbyMember? Member) AddMemberLocked(UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap, UserLobbyPreferences lobbyPrefs) { - // NOTE: AddMember runs inside the per-lobby gate, so timing + slot determination can no - // longer result in two concurrent joins picking the same slot. - return await RunExclusiveAsync(async () => + if (State != ELobbyState.GAME_SETUP) return (LobbyMembershipStatus.NotInSetup, null); + if (GetMemberFromUserID(playerSession.m_UserID) != null) return (LobbyMembershipStatus.AlreadyMember, null); + + int openSlot = Array.FindIndex(Members, member => member.SlotState == EPlayerType.SLOT_OPEN); + if (openSlot < 0) return (LobbyMembershipStatus.NoOpenSlot, null); + ushort slotIndex = (ushort)openSlot; + + // QuickMatch applies social eligibility during matchmaking. + if (LobbyType == ELobbyType.CustomGame) { - if (State != ELobbyState.GAME_SETUP) + var ownerData = WebSocketManager.GetSharedDataForUser(Owner); + if (ownerData != null) { - return false; + var social = ownerData.GetSocialContainer(); + if (social.Blocked.Contains(playerSession.m_UserID)) return (LobbyMembershipStatus.BlockedByOwner, null); + if (LobbyJoinability == ELobbyJoinability.FriendsOnly && !social.Friends.Contains(playerSession.m_UserID)) + return (LobbyMembershipStatus.FriendshipRequired, null); } + } - // NOTE: this must be inside the lock, otherwise two concurrent joins for the same user can both pass - // the check and end up occupying two slots - LobbyMember? existingMember = GetMemberFromUserID(playerSession.m_UserID); - if (existingMember != null) // we're already in this lobby - { - return false; - } - - // find first open slot - bool bFoundSlot = false; - UInt16 slotIndex = 0; - foreach (var memberEntry in Members) - { - if (memberEntry.SlotState == EPlayerType.SLOT_OPEN) - { - // found a gap, use this slot index - bFoundSlot = true; - break; - } - ++slotIndex; - } - - if (!bFoundSlot) - { - return false; - } - - // Check social requirements (dont allow blocked in, and check friends only) - // SOCIAL: If the lobby owner has source user blocked, remove the lobby - // NOTE: Only check this for custom match, quick match checks it during matchmaking bucket stage - if (LobbyType == ELobbyType.CustomGame) - { - SharedUserData? lobbyOwnerSharedData = WebSocketManager.GetSharedDataForUser(Owner); // owner must be a game client - - if (lobbyOwnerSharedData != null) - { - // dont allow join if blocked - if (lobbyOwnerSharedData.GetSocialContainer().Blocked.Contains(playerSession.m_UserID)) - { - return false; - } - - // check joinability - if (LobbyJoinability == ELobbyJoinability.FriendsOnly) - { - // If it's friends only, return false if they aren't friends - if (!lobbyOwnerSharedData.GetSocialContainer().Friends.Contains(playerSession.m_UserID)) - { - return false; - } - } - } - } - - // de dupe names - string strOriginalDisplayName = strDisplayName; + string strOriginalDisplayName = strDisplayName; int dupesSeen = 0; string strNameLower = strDisplayName.ToLower(); foreach (var memberEntry in Members) @@ -1219,6 +1179,8 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa } Members[slotIndex] = newMember; + ++m_MembershipVersion; + InvalidateMeshCheck("membership_changed"); TimeMemberLeft[playerSession.m_UserID] = DateTime.UnixEpoch; // Lobby members leave public-room presence. @@ -1226,6 +1188,11 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa playerSession.UpdateSessionLobbyID(LobbyID); + return (LobbyMembershipStatus.Success, newMember); + } + + private void PublishMemberAddition(UserSession playerSession, LobbyMember newMember) + { // START NETWORK SIGNALLING // send all existing player to new player and vice-versa foreach (LobbyMember memberEntry in Members) @@ -1260,7 +1227,7 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa existingPlayerMsg.msg_id = (int)EWebSocketMessageID.NETWORK_CONNECTION_START_SIGNALLING; existingPlayerMsg.lobby_id = LobbyID; existingPlayerMsg.user_id = playerSession.m_UserID; - existingPlayerMsg.preferred_port = userPreferredPort; + existingPlayerMsg.preferred_port = newMember.Port; existingPlayerMsg.middleware_id = playerSession.GetMiddlewareID(); remoteSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(existingPlayerMsg))); @@ -1279,9 +1246,8 @@ public async Task AddMember(UserSession playerSession, string strDisplayNa // Notify lobby members and room browsers. DirtyRetransmitLobbyList(); - Console.WriteLine("User {0} joined lobby {1}: {2} (Slot was {3})", playerSession.m_UserID, LobbyID, true, slotIndex); - return true; - }); + s_log.LogInformation("User {UserId} joined lobby {LobbyId} in slot {SlotIndex}", playerSession.m_UserID, LobbyID, newMember.SlotIndex); + } public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) @@ -1315,39 +1281,63 @@ public void SendPeerTeardownToDepartingMember(LobbyMember departingMember) } } - public async Task RemoveMember(LobbyMember member) + internal async Task RemoveMemberAsync(long userID, LobbyMember? expectedMember = null) { - // Matchmaking cancellation and the client's explicit lobby leave can arrive concurrently. - // Claim the slot once, and run host migration in the same critical section as the slot - // clear, so teardown, host migration, and destruction callbacks stay atomic and idempotent - // (previously DoHostMigration ran after this gate was released, so it could interleave - // with a concurrent AddMember/RemoveMember and observe a half-updated Members/Owner). - bool bRemoved = await RunExclusiveAsync(() => + LobbyMember? removed = await RunExclusiveAsync(() => { - if (member.SlotIndex < 0 - || member.SlotIndex >= Members.Length - || !ReferenceEquals(Members[member.SlotIndex], member)) - { - return Task.FromResult(false); - } + var member = GetMemberFromUserID(userID); + return Task.FromResult(member != null && (expectedMember == null || ReferenceEquals(member, expectedMember)) + && RemoveMemberLocked(member) ? member : null); + }); + if (removed != null) PublishMemberRemoval(removed); + return removed; + } - LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); - Members[member.SlotIndex] = placeholderMember; - TimeMemberLeft[member.UserID] = DateTime.UtcNow; - - OnAfterPlayerLeftLocked(member.UserID); - - return Task.FromResult(true); - }); + internal async Task KickMemberAsync(KickLobbyMemberCommand command) + { + LobbyMember? removed = null; + var status = await RunExclusiveAsync(() => + { + if (GetMemberFromUserID(command.ActorUserID) == null) + return Task.FromResult(LobbyMembershipStatus.ActorNotMember); + if (Owner != command.ActorUserID) + return Task.FromResult(LobbyMembershipStatus.ActorNotHost); + if (command.TargetUserID == Owner) + return Task.FromResult(LobbyMembershipStatus.NoChange); + var member = command.TargetUserID.HasValue ? GetMemberFromUserID(command.TargetUserID.Value) : null; + if (member != null && RemoveMemberLocked(member)) removed = member; + return Task.FromResult(removed != null ? LobbyMembershipStatus.Success : LobbyMembershipStatus.NoChange); + }); + if (removed != null) PublishMemberRemoval(removed); + return status; + } - if (!bRemoved) + // Caller holds the lobby gate; validation and host migration form one transition. + private bool RemoveMemberLocked(LobbyMember member) + { + if (member.SlotIndex < 0 + || member.SlotIndex >= Members.Length + || !ReferenceEquals(Members[member.SlotIndex], member)) { - return; + return false; } + LobbyMember placeholderMember = new LobbyMember(this, null, -1, String.Empty, String.Empty, 0, -1, -1, -1, EPlayerType.SLOT_OPEN, member.SlotIndex, true); + Members[member.SlotIndex] = placeholderMember; + TimeMemberLeft[member.UserID] = DateTime.UtcNow; + + OnAfterPlayerLeftLocked(member.UserID); + ++m_MembershipVersion; + InvalidateMeshCheck("membership_changed"); + + return true; + } + + private void PublishMemberRemoval(LobbyMember member) + { // TODO_LOBBY: Optimize this Int64 UserID = member.UserID; - Console.WriteLine("User {0} left lobby {1}", UserID, LobbyID); + s_log.LogInformation("User {UserId} left lobby {LobbyId}", UserID, LobbyID); // AC dergister WebSocketMessage_ACDeregisterPlayer remotePlayerAcMsg = new WebSocketMessage_ACDeregisterPlayer(); @@ -1360,7 +1350,7 @@ public async Task RemoveMember(LobbyMember member) { if (remoteSession != null) { - Console.WriteLine("Sent AC deregister for user {0} to user {1}", member.UserID, remoteMember.UserID); + s_log.LogDebug("Lobby {LobbyId}: sent AC deregister for user {UserId} to user {RemoteUserId}", LobbyID, member.UserID, remoteMember.UserID); remoteSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerAcMsg))); } } @@ -1381,7 +1371,7 @@ public async Task RemoveMember(LobbyMember member) { if (remoteSession != null) { - Console.WriteLine("Sent network disconnect for user {0} to user {1}", member.UserID, remoteMember.UserID); + s_log.LogDebug("Lobby {LobbyId}: sent network disconnect for user {UserId} to user {RemoteUserId}", LobbyID, member.UserID, remoteMember.UserID); remoteSession.QueueWebsocketSend(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(remotePlayerMsg))); } } @@ -1390,6 +1380,15 @@ public async Task RemoveMember(LobbyMember member) } DirtyRetransmitLobbyList(); + } + + private LobbyMembershipStatus ValidateJoin(JoinLobbyCommand command) + { + if (LobbyType == ELobbyType.QuickMatch && !MatchedUserIDs.Contains(command.UserID)) + return LobbyMembershipStatus.NotMatched; + if (IsPassworded && command.Password != Password) + return LobbyMembershipStatus.PasswordMismatch; + return command.AnticheatID == AnticheatID ? LobbyMembershipStatus.Success : LobbyMembershipStatus.AnticheatMismatch; } public int GetNumberOfHumans() @@ -1450,7 +1449,7 @@ public async Task DirtyRetransmitToSingleMember(Int64 targetUserID) var session = WebSocketManager.GetSessionFromUser(targetUserID, EUserSessionType.GameClient); // lobby member must be a game client if (session != null) { - Console.WriteLine("[DIRTY LOBBY] Sending WS lobby update for lobby {0}", LobbyID); + s_log.LogDebug("Lobby {LobbyId}: sending WebSocket lobby update", LobbyID); WebSocketMessage_CurrentLobbyUpdate lobbyUpdate = new WebSocketMessage_CurrentLobbyUpdate(); lobbyUpdate.msg_id = (int)EWebSocketMessageID.LOBBY_CURRENT_LOBBY_UPDATE; @@ -1614,8 +1613,7 @@ await RunExclusiveAsync(() => } catch (Exception ex) { - Console.WriteLine($"[ERROR] UpdateState placeholder creation failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "Lobby {LobbyId}: UpdateState placeholder creation failed", LobbyID); } // calculate first probe time @@ -1838,7 +1836,7 @@ public enum EKnownAnticheatID } - public class LobbyManager + public partial class LobbyManager { private ConcurrentDictionary m_dictLobbies = new(); @@ -1855,10 +1853,14 @@ internal Int64 GenerateNextLobbyID() } private readonly IServiceProvider _services; + private readonly ILogger _logger; - public LobbyManager(IServiceProvider services) + public LobbyManager(IServiceProvider services, IDbContextFactory membershipDbFactory, LobbyMembershipConnections membershipConnections, ILogger logger) { _services = services; + _logger = logger; + this.membershipDbFactory = membershipDbFactory; + this.membershipConnections = membershipConnections; } public async Task Cleanup() @@ -1872,7 +1874,7 @@ public async Task Cleanup() TimeSpan timeSinceCreated = DateTime.UtcNow.Subtract(iterLobby.TimeCreated); if (iterLobby.GetNumberOfHumans() == 0 && timeSinceCreated.TotalMinutes >= 1.0) { - Console.WriteLine("Garbage collecting lobby {0}", iterLobby.LobbyID); + _logger.LogInformation("Garbage collecting lobby {LobbyId}", iterLobby.LobbyID); // mark for removal lstLobbiesToRemove.Add(iterLobby); @@ -1882,7 +1884,7 @@ public async Task Cleanup() foreach (Lobby lobbyToRemove in lstLobbiesToRemove) { // remove it, also commit it + update leaderboard - await DeleteLobby(lobbyToRemove); + if (await lobbyToRemove.TryMarkCompleteIfEmptyAsync()) await DeleteLobby(lobbyToRemove); } } @@ -1907,12 +1909,11 @@ public async Task ProcessLobbiesNeedingDestroyed() try { - await DeleteLobby(lobbyToDestroy); + if (await lobbyToDestroy.TryMarkCompleteIfEmptyAsync()) await DeleteLobby(lobbyToDestroy); } catch (Exception ex) { - Console.WriteLine($"[ERROR] Failed to destroy lobby {lobbyToDestroy.LobbyID}: {ex}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "Failed to destroy lobby {LobbyId}", lobbyToDestroy.LobbyID); } } } @@ -1921,12 +1922,14 @@ public async Task CreateLobby(AppDbContext _db, UserSession owningSession UInt16 hostPreferredPort, bool bVanillaTeams, bool bTrackStats, UInt32 default_starting_cash, bool bPassworded, String strPassword, Int16 parentNetworkRoom, bool bAllowObservers, UInt16 maxCamHeight, UInt32 exe_crc, UInt32 ini_crc, ELobbyType lobbyType, EKnownAnticheatID anticheatID) { - Console.WriteLine("Created lobby"); + using var membership = await AcquireMembershipGateAsync(owningSession.m_UserID); + if (!IsCurrentSession(owningSession) || owningSession.IsRetired) return -1; + _logger.LogInformation("Creating lobby for user {UserId}", owningSession.m_UserID); // cant own two lobbies at once, unless in gameplay - await CleanupUserLobbiesNotStarted(owningSession.m_UserID); + await CleanupUserLobbiesNotStartedAsync(owningSession.m_UserID); - Console.WriteLine("[Source 3] User {0} Leave Any Lobby", owningSession.m_UserID); - await this.LeaveAnyLobby(owningSession.m_UserID); + _logger.LogDebug("User {UserId} leaving any lobby before creating a lobby", owningSession.m_UserID); + await LeaveMembershipsAsync(owningSession.m_UserID); int rng_seed = new Random().Next(); @@ -1954,7 +1957,9 @@ public async Task CreateLobby(AppDbContext _db, UserSession owningSession if (lobbyType != ELobbyType.QuickMatch) // QuickMatch clients join after allocation. { - await JoinLobby(_db, newLobby, owningSession, strOwnerDisplayName, hostPreferredPort, true); + var preferences = await Database.Users.GetUserLobbyPreferences(_db, owningSession.m_UserID); + if (preferences != null) + await newLobby.AddCreatorAsync(owningSession, strOwnerDisplayName, hostPreferredPort, preferences); } // Send initial lobby state and invalidate room listings. @@ -1971,36 +1976,17 @@ public async Task Tick() } } - public async Task JoinLobby(AppDbContext _db, Lobby lobby, UserSession playerSession, string strDisplayName, UInt16 userPreferredPort, bool bHasMap) - { - UserLobbyPreferences? lobbyPrefs = await Database.Users.GetUserLobbyPreferences(_db, playerSession.m_UserID); - - if (lobbyPrefs != null) - { - bool bAdded = await lobby.AddMember(playerSession, strDisplayName, userPreferredPort, bHasMap, lobbyPrefs); - return bAdded; - } - - return false; - } - public int GetNumLobbies() { return m_dictLobbies.Count; } - public async Task CleanupUserLobbiesNotStarted(Int64 UserID) - { - List ownedLobbies = GetPlayerOwnedLobbies(UserID); - foreach (Lobby ownedLobby in ownedLobbies) - { - if (ownedLobby.State == ELobbyState.GAME_SETUP) // only those in setup, in game games can continue - { - await DeleteLobby(ownedLobby); - } - } - } - + private async Task CleanupUserLobbiesNotStartedAsync(Int64 userID) + { + foreach (var lobby in GetPlayerOwnedLobbies(userID)) + if (await lobby.TryMarkCompleteIfEmptyAsync(userID, setupOnly: true)) await DeleteLobby(lobby); + } + public List GetAllLobbies(Int16 selectedRoomID, bool bIncludePassword, bool bAllowInSetup, bool bAllowInGame, bool bAllowCompleted, bool bIncludeAllNetworkRooms) { List listLobbies = new List(); @@ -2126,33 +2112,6 @@ public List GetPlayerOwnedLobbies(Int64 userID) return lstLobbies; } - public async Task LeaveSpecificLobby(Int64 userID, Int64 lobbyID) - { - Lobby? targetLobby = GetLobby(lobbyID); - if (targetLobby != null) - { - LobbyMember? memberEntry = targetLobby.GetMemberFromUserID(userID); - if (memberEntry != null) - { - Console.WriteLine("User {0} Leave Specific Lobby", userID); - await targetLobby.RemoveMember(memberEntry); - } - } - } - - public async Task LeaveAnyLobby(Int64 userID) - { - foreach (Lobby lobbyInst in m_dictLobbies.Values) - { - LobbyMember? member = lobbyInst.GetMemberFromUserID(userID); - if (member != null) - { - Console.WriteLine("User {0} Leave Any Lobby", userID); - await lobbyInst.RemoveMember(member); - } - } - } - public async Task DeleteLobby(Lobby lobby) { try @@ -2182,8 +2141,7 @@ public async Task DeleteLobby(Lobby lobby) } catch (Exception ex) { - Console.WriteLine($"[ERROR] DeleteLobby failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + _logger.LogError(ex, "DeleteLobby failed"); return false; } } diff --git a/GenOnlineService/LobbyMembershipConnections.cs b/GenOnlineService/LobbyMembershipConnections.cs new file mode 100644 index 0000000..52a4a62 --- /dev/null +++ b/GenOnlineService/LobbyMembershipConnections.cs @@ -0,0 +1,46 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU Affero General Public License for more details. +** +** You should have received a copy of the GNU Affero General Public License +** along with this program. If not, see . +*/ + +namespace GenOnlineService; + +// Adapter for the legacy session and relay registries used by membership commands. +public sealed class LobbyMembershipConnections +{ + private static readonly ILogger s_log = AppLog.For(typeof(LobbyMembershipConnections)); + + public UserSession? GetSession(long userID, EUserSessionType sessionType) => + WebSocketManager.GetSessionFromUser(userID, sessionType); + + public Task DeleteRelayCredentialsAsync(long userID) => + TURNCredentialManager.DeleteCredentialsForUser(userID); + + public async Task<(string Username, string Token)> CreateRelayCredentialsAsync(long userID) + { + try + { + var credentials = await TURNCredentialManager.CreateCredentialsForUser(userID); + return (credentials?.m_strUsername ?? "", credentials?.m_strToken ?? ""); + } + catch (Exception ex) + { + // Relay failure does not undo a successful lobby join. + s_log.LogError(ex, "Creating relay credentials for user {UserId} failed", userID); + return ("", ""); + } + } +} diff --git a/GenOnlineService/MatchmakingManager.cs b/GenOnlineService/MatchmakingManager.cs index 95d02e3..4d15f8d 100644 --- a/GenOnlineService/MatchmakingManager.cs +++ b/GenOnlineService/MatchmakingManager.cs @@ -1,1973 +1,1900 @@ -/* -** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour -** Copyright (C) 2025 GeneralsOnline Development Team -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU Affero General Public License as -** published by the Free Software Foundation, either version 3 of the -** License, or (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU Affero General Public License for more details. -** -** You should have received a copy of the GNU Affero General Public License -** along with this program. If not, see . -*/ - -using Database; -using Discord.Rest; -using GenOnlineService; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; -using System; -using System.Collections.Concurrent; -using System.Net; -using System.Net.WebSockets; -using System.Security.Claims; -using System.Text; -using System.Text.Json; - -public class PlaylistMap -{ - public string Name { get; private set; } - public string Path { get; private set; } - public bool Custom { get; private set; } - public int MaxPlayers { get; private set; } - - public PlaylistMap(string strName, string strPath, bool bCustom, int maxPlayers) - { - Name = strName; - Path = strPath; - Custom = bCustom; - MaxPlayers = maxPlayers; - } -} - -public class ConcurrentList -{ - private List m_internalList; - private readonly object m_lockObj = new object(); - - public ConcurrentList() - { - m_internalList = new List(); - } - - public ConcurrentList(IEnumerable collection) - { - if (collection != null) - { - m_internalList = new List(collection); - } - else - { - m_internalList = new List(); - } - } - - public void AddRange(IEnumerable collection) - { - lock (m_lockObj) - { - m_internalList.AddRange(collection); - } - } - - public void Add(T item) - { - lock (m_lockObj) - { - m_internalList.Add(item); - } - } - - public void Clear() - { - lock (m_lockObj) - { - m_internalList.Clear(); - } - } - - public T this[int index] - { - get - { - lock (m_lockObj) - { - if (index < 0 || index >= m_internalList.Count) - throw new IndexOutOfRangeException("Index out of range."); - return m_internalList[index]; - } - } - } - - public int Count - { - get - { - lock (m_lockObj) - { - return m_internalList.Count; - } - } - } - - public bool Contains(T item) - { - lock (m_lockObj) - { - return m_internalList.Contains(item); - } - } - public bool Remove(T item) - { - lock (m_lockObj) - { - return m_internalList.Remove(item); - } - } - public List ToList() - { - lock (m_lockObj) - { - return m_internalList.ToList(); - } - } - public IEnumerator GetEnumerator() - { - lock (m_lockObj) - { - return m_internalList.ToList().GetEnumerator(); - } - } -} - -public class Playlist -{ - public UInt16 PlaylistID { get; private set; } - public string Name { get; private set; } - public int MinPlayers { get; private set; } - public int DesiredPlayers { get; private set; } - public int MinSelectedMaps { get; private set; } - - public bool AllowTeams { get; private set; } - public int TeamSize { get; private set; } - public bool AllowArmySelection { get; private set; } - public UInt16 GracePeriodAtMinPlayersMSec { get; private set; } - public List Maps { get; private set; } - - public Playlist(UInt16 a_PlaylistID, string a_strName, - int a_MinPlayers, int a_DesiredPlayers, int a_MinSelectedMaps, bool a_bAllowTeams, int a_TeamSize, bool a_bAllowArmySelection, UInt16 a_gracePeriodAtMinPlayersMSec, List allowedMaps) - { - PlaylistID = a_PlaylistID; - Name = a_strName; - MinPlayers = a_MinPlayers; - MinSelectedMaps = a_MinSelectedMaps; - DesiredPlayers = a_DesiredPlayers; - AllowTeams = a_bAllowTeams; - TeamSize = a_TeamSize; - AllowArmySelection = a_bAllowArmySelection; - GracePeriodAtMinPlayersMSec = a_gracePeriodAtMinPlayersMSec; - Maps = allowedMaps; - } -} - -static class MatchmakingManager -{ - // World Series 2026 Qualification in September requires the matchmaking to be - // based off the monthly ELO. - internal static int GetMatchmakingElo(PlayerStats stats) - { - DateTimeOffset now = DateTimeOffset.UtcNow; - DateTimeOffset septemberStart = new DateTimeOffset(2026, 9, 1, 0, 0, 0, TimeSpan.Zero); - DateTimeOffset octoberStart = new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero); - - return now >= septemberStart && now < octoberStart - ? stats.MonthlyEloRating - : stats.EloRating; - } - - public static void PlayerWidenSearch(UserSession playerSession) - { - // NOTE: we dont check the state of the bucket here, but it doesn't really matter since expanding the maps after it started won't do anything anyway - - // remove the map limitation - foreach (var kvPair in m_dictMatchmakingBuckets) - { - foreach (MatchmakingBucket mmBucket in kvPair.Value) - { - if (mmBucket.HasPlayer(playerSession)) - { - // get the maximum map set for this playlist - if (MatchmakingManager.g_Playlists.TryGetValue(mmBucket.PlaylistID, out Playlist? playlist)) - { - List lstAllMaps = new(); - for (int mapIndex = 0; mapIndex < playlist.Maps.Count; ++mapIndex) - { - lstAllMaps.Add(mapIndex); - } - - // are we the owner of the bucket? update the original list too - - MatchmakingBucketMember? bucketOwner = mmBucket.GetOwner(); - if (bucketOwner != null) - { - if (mmBucket.lstMapIndices.Count > 0 && bucketOwner.GetAssociatedSession() == playerSession) - { - mmBucket.lstMapIndices = new ConcurrentList(lstAllMaps); - } - } - - // update our player too - playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); - - // can't be in multiple buckets - return; - } - } - } - } - - // not in a bucket yet (still queued waiting to be sorted) - widen their selection anyway, otherwise the - // widen request is silently dropped and they get bucketed with their original map list - if (MatchmakingManager.g_Playlists.TryGetValue(playerSession.MatchmakingPlaylistID, out Playlist? queuedPlaylist)) - { - List lstAllMaps = new(); - for (int mapIndex = 0; mapIndex < queuedPlaylist.Maps.Count; ++mapIndex) - { - lstAllMaps.Add(mapIndex); - } - - playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); - } - } - - private static async Task SendMatchmakingMessage(UserSession cache, string message) - { - UserSession? sess = GenOnlineService.WebSocketManager.GetSessionFromUser(cache.m_UserID, cache.GetSessionType()); - if (sess != null) - { - WebSocketMessage_MatchmakingMessage msg = new WebSocketMessage_MatchmakingMessage(); - msg.msg_id = (int)EWebSocketMessageID.MATCHMAKING_MESSAGE; - msg.message = message; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(msg)); - - sess.QueueWebsocketSend(bytesJSON); - } - } - - public class MatchmakingBucketMember - { - WeakReference? m_SessionRef = null; - - public MatchmakingBucketMember(UserSession owningSession) - { - m_SessionRef = new WeakReference(owningSession); - } - - public bool Is(UserSession playerSession) - { - if (m_SessionRef != null) - { - if (m_SessionRef.TryGetTarget(out UserSession? thisSession)) - { - if (thisSession != null && thisSession == playerSession) - { - return true; - } - } - } - - return false; - } - - public UserSession? GetAssociatedSession() - { - if (m_SessionRef != null) - { - if (m_SessionRef.TryGetTarget(out UserSession? thisSession)) - { - return thisSession; - } - } - - return null; - } - } - - public class MatchmakingBucket - { - private ConcurrentList m_lstMembers = new(); - public ConcurrentList lstMapIndices { get; set; } = new(); - - private Int64 m_timeReachedMinPlayers = -1; - private bool m_bReachedMinPlayers = false; - private bool m_bWaitingOnLobbyJoins = false; - private bool m_bHasStartedCountdown = false; - private bool m_bWaitingOnMeshConnectivityChecks = false; - private bool m_bAutoStartInvalidated = false; - private bool m_bPendingDeletion = false; - private bool m_bMergedAway = false; - private bool m_bAbortInProgress = false; - private bool m_bStartCommitted = false; - private readonly object m_StateLock = new(); - - // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never - // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies - public bool IsMergedAway() - { - lock (m_StateLock) - { - return m_bMergedAway; - } - } - - public bool IsLockedForMatchStart() - { - lock (m_StateLock) - { - return m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; - } - } - - private bool IsPendingDeletion() - { - lock (m_StateLock) - { - return m_bPendingDeletion; - } - } - - public UInt16 PlaylistID { get; private set; } - public int MinPlayers { get; private set; } - public int DesiredPlayers { get; private set; } - - public UInt32 ExeCRC { get; private set; } - public UInt32 IniCRC { get; private set; } - - public EKnownAnticheatID AnticheatID { get; private set; } - - public int eloExpansionIteration { get; private set; } = 1; // 1 * EloExpansionValue, matches the initial value - - public void ExpandElo() - { - ++eloExpansionIteration; - m_LastELOExpansionTime = DateTime.Now; - } - - DateTime m_CreationTime = DateTime.Now; - DateTime m_LastELOExpansionTime = DateTime.Now; - - private TimeSpan TimeSinceLastEloExpansion() - { - TimeSpan timeDifference = DateTime.Now - m_LastELOExpansionTime; - return timeDifference; - } - - public MatchmakingBucketMember? GetOwner() - { - if (m_lstMembers.Count > 0) - { - return m_lstMembers[0]; - } - - return null; - } - - public void DetermineMap(out string strMapName, out string strMapPath) - { - // If they are in this bucket, they had SOME map overlap with the bucket creator, now we need to find the common ground between everyone - - // TODO_QUICKMATCH: what if we cant find a suitable map? - - // first condense the map list doesn to a list that has mutually agreed upon maps/preferences from all participants - //var mapSetFromBucketCreator = new HashSet(lstMapIndices); - - var perPlayerMapSet = new List>(); - var finalMapSet = new HashSet(lstMapIndices.ToList()); // we need to check intersection against this, so pre-populate it with the original bucket creation list, because that's the "biggest set" in theory - - // TODO_QUICKMATCH: Optimize this, it's inefficient - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - perPlayerMapSet.Add(new HashSet(memberSession.MatchmakingMapIndicies.ToList())); - } - } - - // Find shared values across all of perPlayerMapSet - if (perPlayerMapSet.Count > 0) - { - for (int i = 0; i < perPlayerMapSet.Count; i++) - { - finalMapSet.IntersectWith(perPlayerMapSet[i]); - } - } - - // remove any maps that aren't big enough (mainly applies to FFA's where maps may be 6 players but bucket could be 8 players) - // NOTE: iterate a real copy, we are mutating finalMapSet below - var copyMapSetForIter = new HashSet(finalMapSet); - foreach (int mapIndex in copyMapSetForIter) - { - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - if (mapIndex >= 0 && mapIndex < playlist.Maps.Count) - { - if (playlist.Maps[mapIndex].MaxPlayers < CurrentMemberCount()) - { - finalMapSet.Remove(mapIndex); - } - } - } - } - - // Randomly select a map from the map list - if (finalMapSet.Count > 0) - { - // Get the playlist for this bucket - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - var finalMapIndices = finalMapSet.ToList(); - int selectedIndex = Random.Shared.Next(finalMapIndices.Count); - int mapIndex = finalMapIndices[selectedIndex]; - - // Defensive: ensure index is valid for playlist.Maps - if (mapIndex >= 0 && mapIndex < playlist.Maps.Count) - { - strMapName = playlist.Maps[mapIndex].Name; - strMapPath = playlist.Maps[mapIndex].Path; - return; - } - } - } - else - { - // pick a sensible default (biggest map in playlist), probably not what the players asked for, but we cant play on no map - Console.WriteLine("WARNING: No mutually agreed upon map found for matchmaking bucket, falling back to largest map in playlist"); - - if (MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - int biggestCountSeen = 0; - PlaylistMap? mapToUse = null; - foreach (var map in playlist.Maps) - { - if (map.MaxPlayers > biggestCountSeen) - { - biggestCountSeen = map.MaxPlayers; - mapToUse = map; - } - } - - // TODO_QUICKMATCH: What if it's still null? don't think we can get into that state since we must have some kind of map in the playlist - if (mapToUse != null) - { - strMapName = mapToUse.Name; - strMapPath = mapToUse.Path; - return; - } - } - } - - // TODO_QUICKMATCH: What happens if you widen when already in a bucket? tell the user htey cant? you would need everyone to expand, or just expand for everyone? - - // Fallback: use first map from bucket creator's list if available - if (lstMapIndices.Count > 0 && MatchmakingManager.g_Playlists.TryGetValue(PlaylistID, out Playlist? fallbackPlaylist)) - { - int fallbackIndex = lstMapIndices[0]; - if (fallbackIndex >= 0 && fallbackIndex < fallbackPlaylist.Maps.Count) - { - strMapName = fallbackPlaylist.Maps[fallbackIndex].Name; - strMapPath = fallbackPlaylist.Maps[fallbackIndex].Path; - return; - } - } - - // If no map found, set to empty - strMapName = string.Empty; - strMapPath = string.Empty; - - } - - public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) - { - var mapSet = new HashSet(lstMapIndices.ToList()); - var mapSetRhs = new HashSet(lstRhs.ToList()); - foreach (int mapIndex in mapSetRhs) - { - if (mapSet.Contains(mapIndex)) - { - return true; - } - } - return false; - } - - public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) - { - if (IsPendingDeletion() || bucketToMerge.IsPendingDeletion()) - { - return false; - } - - // playlist must match - if (bucketToMerge.PlaylistID != this.PlaylistID) - { - return false; - } - - // either bucket already merged away this tick? it is stale, never touch it again - if (IsMergedAway() || bucketToMerge.IsMergedAway()) - { - return false; - } - - // if we're already counting down, dont let people join, just pretend we are full - if (IsLockedForMatchStart()) - { - return false; - } - - // the other bucket may have already been given a lobby this tick - its members are on their way into - // that lobby, so pulling them in here would matchmake them into a second lobby - if (bucketToMerge.IsLockedForMatchStart()) - { - return false; - } - - // must have space for all users in the rhs bucket, and CRCs must match - if (!HasSpaceForUsers(bucketToMerge.CurrentMemberCount(), bucketToMerge.ExeCRC, bucketToMerge.IniCRC, bucketToMerge.AnticheatID)) - { - return false; - } - - // must be within the eloThreshold - int eloExpansionToUse = (bucketToMerge.GetAvgElo() >= EloConfig.HighEloThreshold || GetAvgElo() >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; - if (!IsAvgEloWithinThreshold(bucketToMerge.GetAvgElo(), eloExpansionIteration * eloExpansionToUse)) - { - return false; - } - - // cant be blocked by any participant (or have any participant blocked) - // TODO_OPTIMIZE: This is O(n^2) - foreach (MatchmakingBucketMember rhsMember in bucketToMerge.m_lstMembers) - { - UserSession? rhsSession = rhsMember.GetAssociatedSession(); - - if (rhsSession != null) - { - if (IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(rhsSession, rhsSession.m_UserID)) - { - return false; - } - } - } - - - // must have overlap in our map selections - if (!bucketToMerge.DoMapSelectionsIntersect(this.lstMapIndices)) - { - return false; - } - - return true; - } - - public async Task MergeWithOtherBucket(MatchmakingBucket bucketToMerge) - { - // copy over players (skipping anyone already present, and dead sessions) - foreach (MatchmakingBucketMember rhsMember in bucketToMerge.m_lstMembers) - { - UserSession? rhsSession = rhsMember.GetAssociatedSession(); - if (rhsSession == null || HasPlayer(rhsSession)) - { - continue; - } - - this.m_lstMembers.Add(rhsMember); - } - - // the source bucket is now empty and flagged so it can never merge/accept players again - lock (bucketToMerge.m_StateLock) - { - bucketToMerge.m_bMergedAway = true; - } - bucketToMerge.m_lstMembers.Clear(); - - // nothing else to copy... everything else should match since we were a merge candidate - - // tell all players - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? session = member.GetAssociatedSession(); - if (session != null) - { - await SendMatchmakingMessage(session, String.Format("Your matchmaking bucket was merged with another bucket. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); - } - } - } - - public bool HasPlayer(UserSession playerSession) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - if (member.Is(playerSession)) - { - return true; - } - } - - return false; - } - - public bool RemovePlayer(UserSession playerSession, out bool bCancellationRejected) - { - bCancellationRejected = false; - lock (m_StateLock) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - if (member.Is(playerSession)) - { - if (m_bStartCommitted) - { - bCancellationRejected = true; - return false; - } - - if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) - { - m_bAutoStartInvalidated = true; - } - - m_lstMembers.Remove(member); - return true; - } - } - } - - return false; - } - - public int CurrentMemberCount() - { - return m_lstMembers.Count; - } - - // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, - // which both blocks the "everyone joined the lobby" check and skews the average elo - public bool PruneDeadMembers() - { - bool bInvalidatedSetup = false; - lock (m_StateLock) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - if (member.GetAssociatedSession() == null) - { - if (m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) - { - m_bAutoStartInvalidated = true; - bInvalidatedSetup = true; - } - - m_lstMembers.Remove(member); - } - } - } - - return bInvalidatedSetup; - } - - internal void MarkPendingDeletion() - { - lock (m_StateLock) - { - m_bPendingDeletion = true; - } - } - - // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs - public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) - { - // NOTE: We check blocking in both directions, joiner blocked them, or joiner is blocked by a player - SharedUserData? joiningUserData = joiningUserSession != null - ? GenOnlineService.WebSocketManager.GetSharedDataForUser(joiningUserSession.m_UserID) - : null; - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); - - if (memberUserData != null) - { - // Bucket member has blocked the joining user - if (memberUserData.GetSocialContainer().Blocked.Contains(joining_user)) - return true; - - // Joining user has blocked the bucket member - if (joiningUserData?.GetSocialContainer().Blocked.Contains(memberSession.m_UserID) == true) - return true; - } - } - } - - return false; - } - - public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // stale buckets must never accept new members - if (IsMergedAway() || IsPendingDeletion()) - { - return false; - } - - // if we're already counting down, dont let people join, just pretend we are full - if (IsLockedForMatchStart()) - { - return false; - } - - // crcs must match too - if (exe_crc != ExeCRC || ini_crc != IniCRC) - { - return false; - } - - // must be running same AC - if (anticheatID != AnticheatID) - { - return false; - } - - return numUsers <= (DesiredPlayers - m_lstMembers.Count); - } - - public bool IsAvgEloWithinThreshold(int playerElo, int eloThreshold) - { - // calculate an average elo if it's a team game, othewrise average will just be the other player - int avgElo = GetAvgElo(); - - int lowerEloBound = playerElo - eloThreshold; - int upperEloBound = playerElo + eloThreshold; - - return (avgElo >= lowerEloBound && avgElo <= upperEloBound); - } - - public int GetAvgElo() - { - int numMembers = m_lstMembers.Count; - - if (numMembers == 0) - { - return EloConfig.BaseRating; - } - - int avgElo = 0; - int numContributingMembers = 0; - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); - - if (memberUserData?.GameStats != null) - { - avgElo += MatchmakingManager.GetMatchmakingElo(memberUserData.GameStats); - ++numContributingMembers; - } - } - } - - // only divide by the members we actually got an elo for, otherwise dead/unknown sessions drag the average down - if (numContributingMembers == 0) - { - return EloConfig.BaseRating; - } - - avgElo /= numContributingMembers; - - return avgElo; - } - - - public async Task Join(UserSession playerSession) - { - // already in here? nothing to do (and never add a duplicate member) - if (HasPlayer(playerSession)) - { - return true; - } - - // cant be blocked by others in this bucket - if (!IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(playerSession, playerSession.m_UserID)) - { - if (HasSpaceForUsers(1, playerSession.ExeCRC, playerSession.IniCRC, playerSession.AnticheatID)) - { - m_lstMembers.Add(new MatchmakingBucketMember(playerSession)); - - // tell everyone - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, String.Format("Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); - } - } - - return true; - } - } - - return false; - } - - public MatchmakingBucket(UInt16 playlistID, UserSession owningSession, int minPlayers, int desiredPlayers, ConcurrentList mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - PlaylistID = playlistID; - MinPlayers = minPlayers; - DesiredPlayers = desiredPlayers; - lstMapIndices = mapIndices; - ExeCRC = exe_crc; - IniCRC = ini_crc; - AnticheatID = anticheatID; - - m_lstMembers.Add(new MatchmakingBucketMember(owningSession)); - } - - public Int64 GetLobbyID() - { - return m_LobbyID; - } - - Int64 m_LobbyID = -1; - Int64 m_StartTime = -1; - Int64 m_timeStartedWaitingOnLobbyJoins = -1; - - // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers - private const Int64 c_LobbyJoinTimeoutMSec = 45000; - private const int c_GameStartCountdownMSec = 5000; - private const int c_SetupClientTimeoutMarginMSec = 2000; - - // older clients treat any timeout_ms below this as the start countdown - private const int c_LegacyClientCountdownThresholdMSec = 10000; - - private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) - { - Console.WriteLine("START GAME"); - - WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); - startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.IsRegisteredForMatchmaking = false; - memberSession.QueueWebsocketSend(bytesJSON); - } - } - - await lobby.UpdateState(ELobbyState.INGAME); - MatchmakingManager.DestroyBucket(this); - } - - private void QueueSetupProgress(int timeoutMSec, int countdownMSec = 0) - { - WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); - setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; - setupProgress.timeout_ms = timeoutMSec; - setupProgress.countdown_ms = countdownMSec; - byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(setupProgressJSON); - } - } - } - - private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) - { - await lobby.StartFullMeshConnectivityCheck(); - lock (m_StateLock) - { - // Publish the state transition before any notification work. If a send fails, the - // regular lobby timeout still completes or aborts the setup instead of stranding it. - m_bWaitingOnMeshConnectivityChecks = true; - } - - // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients - QueueSetupProgress(Math.Max(FullMeshCheckSettings.MaxDurationMS + c_SetupClientTimeoutMarginMSec, c_LegacyClientCountdownThresholdMSec)); - - lobby.SendFullMeshConnectivityCheckRequestToMembers(); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, "Preparing match..."); - } - } - } - - private async Task AbortQuickMatchAutoStart(string reason) - { - List sessionsToRequeue = new(); - lock (m_StateLock) - { - if (m_bAbortInProgress || m_bStartCommitted) - { - return; - } - - m_bAbortInProgress = true; - m_bPendingDeletion = true; - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - sessionsToRequeue.Add(memberSession); - } - } - - m_StartTime = -1; - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = false; - m_bWaitingOnMeshConnectivityChecks = false; - } - - LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); - Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); - if (quickMatchLobby != null) - { - foreach (UserSession memberSession in sessionsToRequeue) - { - LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); - if (lobbyMember != null) - { - // TODO: Remove this fallback once all supported clients handle MATCHMAKING_ACTION_REQUEUE. - // Legacy clients do not understand the requeue action, but they can still tear down - // peer and anti-cheat connections before joining the next temporary lobby. - quickMatchLobby.SendPeerTeardownToDepartingMember(lobbyMember); - await quickMatchLobby.RemoveMember(lobbyMember); - } - } - - await lobbyManager.DeleteLobby(quickMatchLobby); - } - - WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); - requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; - byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); - - foreach (UserSession memberSession in sessionsToRequeue) - { - if (await TryRequeueRegisteredPlayer(memberSession, requeueActionJSON)) - { - await SendMatchmakingMessage(memberSession, reason); - await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); - } - } - - m_lstMembers.Clear(); - m_LobbyID = -1; - - MatchmakingManager.DestroyBucket(this); - } - - public async Task Tick() - { - bool bPendingDeletion; - bool bAutoStartInvalidated; - bool bWaitingOnMeshConnectivityChecks; - bool bWaitingOnLobbyJoins; - bool bHasStartedCountdown; - lock (m_StateLock) - { - bPendingDeletion = m_bPendingDeletion; - bAutoStartInvalidated = m_bAutoStartInvalidated; - bWaitingOnMeshConnectivityChecks = m_bWaitingOnMeshConnectivityChecks; - bWaitingOnLobbyJoins = m_bWaitingOnLobbyJoins; - bHasStartedCountdown = m_bHasStartedCountdown; - } - - // merged/deleted buckets must not create another lobby or continue a committed setup - if (IsMergedAway() || bPendingDeletion) - { - return; - } - - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - - // drop any members whose session has gone away, otherwise they are counted forever and the bucket can - // never reach the "everyone is in the lobby" condition - if (PruneDeadMembers()) - { - bAutoStartInvalidated = true; - } - - // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket - if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) - { - // nobody left? clean ourselves up rather than lingering as a ghost bucket - if (CurrentMemberCount() == 0 && !bWaitingOnLobbyJoins && !bHasStartedCountdown) - { - MatchmakingManager.DestroyBucket(this); - return; - } - - if (bAutoStartInvalidated) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); - return; - } - - if (bWaitingOnMeshConnectivityChecks) - { - Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); - if (lobbyDuringMeshCheck == null) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); - return; - } - - await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); - - if (!lobbyDuringMeshCheck.PendingFullMeshConnectivityChecks) - { - bool bStartCountdown; - bool bAbortStart; - bool bInvalidatedAtDecision; - lock (m_StateLock) - { - bInvalidatedAtDecision = m_bAutoStartInvalidated; - if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) - { - bStartCountdown = false; - bAbortStart = false; - } - else - { - m_bWaitingOnMeshConnectivityChecks = false; - bStartCountdown = !bInvalidatedAtDecision && lobbyDuringMeshCheck.LastFullMeshConnectivityCheckOutcome == true; - bAbortStart = !bStartCountdown; - if (bStartCountdown) - { - m_StartTime = Environment.TickCount64 + c_GameStartCountdownMSec; - m_bHasStartedCountdown = true; - } - } - } - - if (bStartCountdown) - { - QueueSetupProgress(c_GameStartCountdownMSec + c_SetupClientTimeoutMarginMSec, c_GameStartCountdownMSec); - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, $"Starting game in {c_GameStartCountdownMSec / 1000} seconds."); - } - } - } - else if (bAbortStart) - { - string reason = bInvalidatedAtDecision - ? "QuickMatch auto-start was aborted because a player left during match setup." - : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; - await AbortQuickMatchAutoStart(reason); - } - } - - return; - } - - if (bHasStartedCountdown) - { - if (Environment.TickCount64 < m_StartTime) - { - return; - } - - Lobby? lobbyAfterCountdown = lobbyManager.GetLobby(m_LobbyID); - if (lobbyAfterCountdown == null) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); - return; - } - - bool bStartGame; - bool bAbortStart; - lock (m_StateLock) - { - if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) - { - bStartGame = false; - bAbortStart = false; - } - else - { - m_bHasStartedCountdown = false; - m_StartTime = -1; - bStartGame = !m_bAutoStartInvalidated; - bAbortStart = !bStartGame; - if (bStartGame) - { - m_bStartCommitted = true; - m_bPendingDeletion = true; - } - } - } - - if (bStartGame) - { - await StartGameAfterSuccessfulMeshCheck(lobbyAfterCountdown); - } - else if (bAbortStart) - { - await AbortQuickMatchAutoStart("QuickMatch auto-start was aborted because a player left during match setup."); - } - - return; - } - - // do we need to start? - // TODO_MATCHMAKING: Add a timeout at which > min players starts - if (!m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) - { - // must have a min player count - if (MinPlayers != DesiredPlayers) - { - // have we hit the min player count? start a timer - // NOTE: >= not ==, a merge (or several joins in one tick) can jump straight past MinPlayers - if (!m_bReachedMinPlayers && CurrentMemberCount() >= MinPlayers) - { - m_bReachedMinPlayers = true; - m_timeReachedMinPlayers = Environment.TickCount64; - - // tell players we reached min and will start in - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, - String.Format("Matchmaker has reached the minimum number of players required. Starting in {0} seconds (more players can still join in the mean time)", playlist.GracePeriodAtMinPlayersMSec / 1000)); - } - } - } - else if (m_bReachedMinPlayers && CurrentMemberCount() < MinPlayers) // did we go back under min players? stop countdown - { - m_bReachedMinPlayers = false; - m_timeReachedMinPlayers = -1; - - // tell players we stopped - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, "One or more players have left which resulted in the matchmaker dropping below the minimum players required to start - the countdown to start has been cancelled"); - } - } - } - } - - // do we need an expansion of elo criteria? - if (TimeSinceLastEloExpansion().TotalSeconds >= EloConfig.SecondsBetweenEloExpansionsInMatchmaking) - { - // expand - ExpandElo(); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, "Expanding search criteria to find more players..."); - } - } - } - - // did we hit the timer OR have enough players to start? - bool bMinPlayersCountdownExpired = m_bReachedMinPlayers && (Environment.TickCount64 - m_timeReachedMinPlayers) > playlist.GracePeriodAtMinPlayersMSec; - if (bMinPlayersCountdownExpired || CurrentMemberCount() >= DesiredPlayers) - { - // reset min player countdown - m_bReachedMinPlayers = false; - m_timeReachedMinPlayers = -1; - - lock (m_StateLock) - { - m_bWaitingOnLobbyJoins = true; - m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; - } - - // tell everyone - UserSession? dummyHostUser = null; - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - // create lb data if necessary - using var scope = ServiceLocator.Services.CreateScope(); - var factory = scope.ServiceProvider.GetRequiredService>(); - await using var db = await factory.CreateDbContextAsync(); - - await Database.Leaderboards.CreateUserEntriesIfNotExists(db, memberSession.m_UserID); - - if (dummyHostUser == null) - { - dummyHostUser = memberSession; - } - await SendMatchmakingMessage(memberSession, "Creating a QuickMatch lobby for everyone..."); - } - } - - // should have a user by now - if (dummyHostUser != null) - { - SharedUserData? dummyHostUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(dummyHostUser.m_UserID); - - if (dummyHostUserData != null) - { - // make a lobby - DetermineMap(out string strMapName, out string strMapPath); - - using var scope = ServiceLocator.Services.CreateScope(); - var factory = scope.ServiceProvider.GetRequiredService>(); - await using var db = await factory.CreateDbContextAsync(); - - m_LobbyID = await lobbyManager.CreateLobby(db, dummyHostUser, dummyHostUserData.m_strDisplayName, "Quickmatch Lobby", strMapName, strMapPath + ".map", - true, playlist.DesiredPlayers, "", 12345, false, true, 10000, false, String.Empty, -5, false, Constants.g_DefaultCameraMaxHeight, dummyHostUser.ExeCRC, dummyHostUser.IniCRC, ELobbyType.QuickMatch, - dummyHostUser.AnticheatID); - - Lobby? quickMatchLobby = lobbyManager.GetLobby(m_LobbyID); - if (quickMatchLobby != null) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - quickMatchLobby.MatchedUserIDs.Add(memberSession.m_UserID); - } - } - } - - // tell both to join our lobby - WebSocketMessage_MatchmakerJoinLobby joinAction = new WebSocketMessage_MatchmakerJoinLobby(); - joinAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_JOIN_PREARRANGED_LOBBY; - joinAction.lobby_id = m_LobbyID; - byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(joinAction)); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.QueueWebsocketSend(bytesJSON); - } - } - } - } - } - } - else - { - // do we need to cancel? people might have left - // TODO_QUICKMATCH: Re-enable - /* - if (m_StartTime != -1 || m_bWaitingOnLobbyJoins) - { - if (CurrentMemberCount() < MinPlayers) - { - m_bWaitingOnLobbyJoins = false; - m_bHasStartedCountdown = false; - m_StartTime = -1; - - // destroy lobby - Lobby? lobby = LobbyManager.GetLobby(m_LobbyID); - if (lobby != null) - { - await LobbyManager.DeleteLobby(lobby); - } - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - ActiveUserDataCache? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - await SendMatchmakingMessage(memberSession, String.Format("A player has left and the starting countdown has been cancelled. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); - } - } - - return; - } - } - */ - - // waiting on lobby joins? - if (m_bWaitingOnLobbyJoins) - { - // done? start time etc - Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); - if (lobby == null) - { - // the lobby went away underneath us (deleted/failed) - don't leave everyone stuck waiting forever - Console.WriteLine("Matchmaking bucket lost its QuickMatch lobby {0} while waiting on joins, abandoning bucket", m_LobbyID); - - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.IsRegisteredForMatchmaking = false; - await SendMatchmakingMessage(memberSession, "The QuickMatch lobby could not be created. Please try matchmaking again."); - } - } - - m_bWaitingOnLobbyJoins = false; - m_timeStartedWaitingOnLobbyJoins = -1; - MatchmakingManager.DestroyBucket(this); - return; - } - - // has someone failed to connect? drop them so the rest of the players aren't stuck here forever - bool bJoinTimeoutExpired = m_timeStartedWaitingOnLobbyJoins != -1 - && (Environment.TickCount64 - m_timeStartedWaitingOnLobbyJoins) > c_LobbyJoinTimeoutMSec; - - if (bJoinTimeoutExpired && lobby.NumCurrentPlayers != CurrentMemberCount()) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession == null || lobby.GetMemberFromUserID(memberSession.m_UserID) == null) - { - if (memberSession != null) - { - Console.WriteLine("User {0} failed to join QuickMatch lobby {1} in time, dropping from bucket", memberSession.m_UserID, m_LobbyID); - memberSession.IsRegisteredForMatchmaking = false; - await SendMatchmakingMessage(memberSession, "You failed to join the QuickMatch lobby in time and have been removed from matchmaking."); - } - - m_lstMembers.Remove(member); - } - } - - // not enough players survived? tear the whole thing down and let people re-queue - if (CurrentMemberCount() < MinPlayers) - { - foreach (MatchmakingBucketMember member in m_lstMembers) - { - UserSession? memberSession = member.GetAssociatedSession(); - if (memberSession != null) - { - memberSession.IsRegisteredForMatchmaking = false; - await SendMatchmakingMessage(memberSession, "Not enough players joined the QuickMatch lobby. Please try matchmaking again."); - } - } - - m_bWaitingOnLobbyJoins = false; - m_timeStartedWaitingOnLobbyJoins = -1; - - await lobbyManager.DeleteLobby(lobby); - MatchmakingManager.DestroyBucket(this); - return; - } - } - - { - if (lobby.NumCurrentPlayers == CurrentMemberCount()) // everyone is in, lets start for real - { - m_timeStartedWaitingOnLobbyJoins = -1; - - lock (m_StateLock) - { - m_bWaitingOnLobbyJoins = false; - } - - // finalize the teams - const int playlistMaxPlayerPerTeam = 2; - bool bIsFFA = true; - int numTeams = lobby.NumCurrentPlayers / playlistMaxPlayerPerTeam; - - int teamID = 0; - foreach (LobbyMember member in lobby.Members) - { - if (bIsFFA) - { - member.UpdateTeam(-1); - } - else - { - member.UpdateTeam(teamID); - - teamID++; - - if (teamID >= numTeams) - { - teamID = 0; - } - } - } - - await TriggerFullMeshConnectivityChecks(lobby); - } - } - - } - - } - } - } - - // TODO_MATCHMAKING: Delete buckets if participants becomes 0 - } - // Using ConcurrentBag instead of ConcurrentList for lock-free bucket management - private static ConcurrentDictionary> m_dictMatchmakingBuckets = new(); - - // TODO_QUICKMATCH: Read from db or file - private static Dictionary g_Playlists = new() - { - { 0, new Playlist(0, "1v1 (Random Armies)", 2, 2, 8, false, -1, false, 0, new List() - { - new PlaylistMap("[RANK] AKAs Magic ZH v1", "[RANK] AKAs Magic ZH v1", true, 2), - new PlaylistMap("[RANK] Arctic Arena ZH v1", "[RANK] Arctic Arena ZH v1", true, 2), -// new PlaylistMap("[RANK] Arctic Lagoon ZH v2 (2)", "[RANK] Arctic Lagoon ZH v2", true, 2), -// new PlaylistMap("[RANK] Barren Badlands Balanced ZH v2 (2)", "[RANK] Barren Badlands Balanced ZH v2", true, 2), -// new PlaylistMap("[RANK] Black Hell ZH v1", "[RANK] Black Hell ZH v1", true, 2), -// new PlaylistMap("[RANK] Blossoming Valley ZH v1 (2)", "[RANK] Blossoming Valley ZH v1", true, 2), -// new PlaylistMap("[RANK] Blue Hole ZH v1", "[RANK] Blue Hole ZH v1", true, 2), -// new PlaylistMap("[RANK] Canyon of the Dead ZH v2 (2)", "[RANK] Canyon of the Dead ZH v2", true, 2), -// new PlaylistMap("[RANK] Dammed Scorpion ZH v1", "[RANK] Dammed Scorpion ZH v1", true, 2), -// new PlaylistMap("[RANK] Desolated District ZH v1 (2)", "[RANK] Desolated District ZH v1", true, 2), - new PlaylistMap("[RANK] Drallim Desert ZH v2", "[RANK] Drallim Desert ZH v2", true, 2), -// new PlaylistMap("[RANK] Egyptian Oasis ZH v1 (2)", "[RANK] Egyptian Oasis ZH v1", true, 2), -// new PlaylistMap("[RANK] Farmlands of the Fallen ZH v1", "[RANK] Farmlands of the Fallen ZH v1", true, 2), -// new PlaylistMap("[RANK] Hanamura Temple ZH v1 (2)", "[RANK] Hanamura Temple ZH v1", true, 2), -// new PlaylistMap("[RANK] Imminent Victory ZH v2 (2)", "[RANK] Imminent Victory ZH v2", true, 2), -// new PlaylistMap("[RANK] Liquid Gold ZH v2 (2)", "[RANK] Liquid Gold ZH v2", true, 2), - new PlaylistMap("[RANK] Mountain Mayhem v2 (2)", "[RANK] Mountain Mayhem v2", true, 2), - new PlaylistMap("[RANK] Natural Threats ZH v4 (2)", "[RANK] Natural Threats ZH v4", true, 2), -// new PlaylistMap("[RANK] Sakura Forest II ZH v1", "[RANK] Sakura Forest II ZH v1", true, 2), -// new PlaylistMap("[RANK] Sand Scorpion (2)", "[RANK] Sand Scorpion", true, 2), -// new PlaylistMap("[RANK] Snowy Drought ZH v5 (2)", "[RANK] Snowy Drought ZH v5", true, 2), -// new PlaylistMap("[RANK] Sovereignty ZH v1", "[RANK] Sovereignty ZH v1", true, 2), - new PlaylistMap("[RANK] TD NoBugsCars ZH v1 (2)", "[RANK] TD NoBugsCars ZH v1", true, 2), - new PlaylistMap("[RANK] Vendetta ZH v1 (2)", "[RANK] Vendetta ZH v1", true, 2), -// new PlaylistMap("[RANK] ZH Carrier is Over v2 (2)", "[RANK] ZH Carrier is Over v2", true, 2), - new PlaylistMap("Arabia v2 (2)", "Arabia v2", true, 2), -// new PlaylistMap("Battle Plan ZH v3", "Battle Plan ZH v3", true, 2), - new PlaylistMap("Canyon Frost ZH v1", "Canyon Frost ZH v1", true, 2), - new PlaylistMap("Forest of Camelot ZH v4 (2)", "Forest of Camelot ZH v4", true, 2), - new PlaylistMap("Koujou Okawa v3", "Koujou Okawa v3", true, 2), - new PlaylistMap("Mirmulnir v4 (2)", "Mirmulnir v4", true, 2), - new PlaylistMap("Oxygen 1", "Oxygen 1", true, 2), -// new PlaylistMap("SandForest Domination v5 (2)", "SandForest Domination v5", true, 2), -// new PlaylistMap("Seasonal Conflict Summer V2 (2)", "Seasonal Conflict Summer V2", true, 2), -// new PlaylistMap("Shivas Paradise v4", "Shivas Paradise v4", true, 2), - new PlaylistMap("Terraform v2 (2)", "Terraform v2", true, 2), -// new PlaylistMap("Thermopylae v5", "Thermopylae v5", true, 2), -// new PlaylistMap("Tiny Tactics ZH v2", "Tiny Tactics ZH v2", true, 2), - new PlaylistMap("Toxic Lake v5 (2)", "Toxic Lake v5", true, 2), - new PlaylistMap("Tremble v2 (2)", "Tremble v2", true, 2), -// new PlaylistMap("yota nation arena v3", "yota nation arena v3", true, 2), - } - ) }, - - /* PRE-WS list: - * { 0, new Playlist(0, "1v1 (Random Armies)", 2, 2, 1, false, -1, false, 0, new List() - { - new PlaylistMap("[RANK] Snowy Drought ZH v5 (2)", "[RANK] Snowy Drought ZH v5", true, 2), - new PlaylistMap("[RANK] Natural Threats ZH v4 (2)", "[RANK] Natural Threats ZH v4", true, 2), - new PlaylistMap("[RANK] Arctic Lagoon ZH v2 (2)", "[RANK] Arctic Lagoon ZH v2", true, 2), - new PlaylistMap("[RANK] ZH Carrier is Over v2 (2)", "[RANK] ZH Carrier is Over v2", true, 2), - new PlaylistMap("[RANK] Vendetta ZH v1 (2)", "[RANK] Vendetta ZH v1", true, 2), - new PlaylistMap("[RANK] TD NoBugsCars ZH v1 (2)", "[RANK] TD NoBugsCars ZH v1", true, 2), - new PlaylistMap("[RANK] Sand Scorpion (2)", "[RANK] Sand Scorpion", true, 2), - new PlaylistMap("[RANK] Mountain Mayhem v2 (2)", "[RANK] Mountain Mayhem v2", true, 2), - new PlaylistMap("[RANK] Liquid Gold ZH v2 (2)", "[RANK] Liquid Gold ZH v2", true, 2), - new PlaylistMap("[RANK] Imminent Victory ZH v2 (2)", "[RANK] Imminent Victory ZH v2", true, 2), - new PlaylistMap("[RANK] Egyptian Oasis ZH v1 (2)", "[RANK] Egyptian Oasis ZH v1", true, 2), - new PlaylistMap("[RANK] Desolated District ZH v1 (2)", "[RANK] Desolated District ZH v1", true, 2), - new PlaylistMap("[RANK] Canyon of the Dead ZH v2 (2)", "[RANK] Canyon of the Dead ZH v2", true, 2), - new PlaylistMap("[RANK] Blossoming Valley ZH v1 (2)", "[RANK] Blossoming Valley ZH v1", true, 2), - //new PlaylistMap("[RANK] Battle Plan ZH v1 (2)", "[RANK] Battle Plan ZH v1", true, 2), // NOTE: Disabled. OldA say it is bugged according to map creator. - new PlaylistMap("[RANK] Barren Badlands Balanced ZH v2 (2)", "[RANK] Barren Badlands Balanced ZH v2", true, 2), - - // new maps added in 12_05_25 update - new PlaylistMap("Battle Plan ZH v3", "Battle Plan ZH v3", true, 2), - new PlaylistMap("Canyon Frost ZH v1", "Canyon Frost ZH v1", true, 2), - new PlaylistMap("Koujou Okawa v3", "Koujou Okawa v3", true, 2), - new PlaylistMap("Oxygen 1", "Oxygen 1", true, 2), - new PlaylistMap("Shivas Paradise v4", "Shivas Paradise v4", true, 2), - new PlaylistMap("Thermopylae v5", "Thermopylae v5", true, 2), - new PlaylistMap("Tiny Tactics ZH v2", "Tiny Tactics ZH v2", true, 2), - new PlaylistMap("yota nation arena v3", "yota nation arena v3", true, 2), - new PlaylistMap("[RANK] AKAs Magic ZH v1", "[RANK] AKAs Magic ZH v1", true, 2), - new PlaylistMap("[RANK] Arctic Arena ZH v1", "[RANK] Arctic Arena ZH v1", true, 2), - new PlaylistMap("[RANK] Black Hell ZH v1", "[RANK] Black Hell ZH v1", true, 2), - new PlaylistMap("[RANK] Blue Hole ZH v1", "[RANK] Blue Hole ZH v1", true, 2), - new PlaylistMap("[RANK] Dammed Scorpion ZH v1", "[RANK] Dammed Scorpion ZH v1", true, 2), - new PlaylistMap("[RANK] Drallim Desert ZH v2", "[RANK] Drallim Desert ZH v2", true, 2), - new PlaylistMap("[RANK] Farmlands of the Fallen ZH v1", "[RANK] Farmlands of the Fallen ZH v1", true, 2), - new PlaylistMap("[RANK] Sakura Forest II ZH v1", "[RANK] Sakura Forest II ZH v1", true, 2), - new PlaylistMap("[RANK] Sovereignty ZH v1", "[RANK] Sovereignty ZH v1", true, 2), - - // new maps added in 4_28_26 EAC update (for WS / from GR) - new PlaylistMap("Arabia v2 (2)", "Arabia v2", true, 2), - new PlaylistMap("SandForest Domination v5 (2)", "SandForest Domination v5", true, 2), - new PlaylistMap("Toxic Lake v4 (2)", "Toxic Lake v4", true, 2), - new PlaylistMap("Tremble v2 (2)", "Tremble v2", true, 2), - new PlaylistMap("[RANK] Hanamura Temple ZH v1 (2)", "[RANK] Hanamura Temple ZH v1", true, 2), - } - ) }, - */ - - { 1, new Playlist(1, "6-8P FFA (Random Armies)", 6, 8, 1, false, -1, false, 30000, new List() - { - new PlaylistMap("Beijing Uprise v4 (6)", "Beijing Uprise v4", true, 6), - new PlaylistMap("Defcon6 (6)", "Defcon6", false, 6), - new PlaylistMap("Taiga Terror (6)", "Taiga Terror", true, 6), - new PlaylistMap("Swamp Assault v3 [WBC2021] (7)", "Swamp Assault v3 [WBC2021]", true, 7), - new PlaylistMap("[RANK] Muddy Madness ZH v1 (8)", "[RANK] Muddy Madness ZH v1", true, 8), - new PlaylistMap("[RANK] Wastelands Dust ZH v1 (8)", "[RANK] Wastelands Dust ZH v1", true, 8) - } - ) } - }; - - public static Dictionary GetPlaylists() { return g_Playlists; } - - public static int GetTotalQueuedPlayersInPlaylist(UInt16 playlistID) - { - int totalPlayers = 0; - - if (m_dictMatchmakingBuckets.ContainsKey(playlistID)) - { - foreach (MatchmakingBucket bucket in m_dictMatchmakingBuckets[playlistID]) - { - totalPlayers += bucket.CurrentMemberCount(); - } - } - - return totalPlayers; - } - - public static async Task Tick() - { - // TODO_QUICKMATCH: Move to init func, maybe dont use static for matchmakingmanager - if (m_dictMatchmakingBuckets.Count == 0) - { - foreach (var kvPair in g_Playlists) - { - m_dictMatchmakingBuckets.TryAdd(kvPair.Key, new ConcurrentBag()); - } - } - - // tick mm buckets - // TODO_QUICKMATCH: This is slow - List lstBucketsMergedNeedingDeleted = new(); - foreach (var kvPair in m_dictMatchmakingBuckets) - { - foreach (MatchmakingBucket mmBucket in kvPair.Value) - { - // if we've already been merged and are awaiting delayed deletion, dont process it anymore - if (!lstBucketsMergedNeedingDeleted.Contains(mmBucket) && !mmBucket.IsMergedAway()) - { - await mmBucket.Tick(); - - // try to merge with any other bucket within this playlist - foreach (MatchmakingBucket mmBucketMergeCandidate in kvPair.Value) - { - if (mmBucketMergeCandidate != mmBucket) - { - // if either bucket has already been merged and is awaiting delayed deletion, dont process it anymore - if (lstBucketsMergedNeedingDeleted.Contains(mmBucket) || mmBucket.IsMergedAway()) - { - break; - } - - if (!lstBucketsMergedNeedingDeleted.Contains(mmBucketMergeCandidate) && !mmBucketMergeCandidate.IsMergedAway()) - { - if (mmBucket.CanMergeWithOtherBucket(mmBucketMergeCandidate)) - { - await mmBucket.MergeWithOtherBucket(mmBucketMergeCandidate); - - lstBucketsMergedNeedingDeleted.Add(mmBucketMergeCandidate); - } - } - } - } - } - } - } - - // queue for deletion - foreach (MatchmakingBucket bucket in lstBucketsMergedNeedingDeleted) - { - m_bucketsPendingDeletion.Enqueue(bucket); - } - - // Drain the queue rather than enumerating and clearing a shared list. A concurrent cancellation can - // enqueue a bucket while cleanup is running, and clearing the list would otherwise lose that request. - while (m_bucketsPendingDeletion.TryDequeue(out MatchmakingBucket? bucket)) - { - if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) - { - // ConcurrentBag doesn't support Remove, so we filter and rebuild - var remainingBuckets = bucketBag.Where(b => b != bucket).ToList(); - m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); - } - } - - List> lstDestroy = new(); - foreach (WeakReference wrSession in lstSessions) - { - if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null) - { - lstDestroy.Add(wrSession); - } - else - { - await thisSession.MatchmakingStateLock.WaitAsync(); - try - { - // A cancellation can remove the weak reference while this tick is iterating a snapshot. - // Re-check registration while holding the per-session gate before assigning any bucket. - if (!thisSession.IsRegisteredForMatchmaking || !IsPendingSession(thisSession)) - { - lstDestroy.Add(wrSession); - continue; - } - - SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); - - if (thisSessionUserData == null) - { - thisSession.IsRegisteredForMatchmaking = false; - lstDestroy.Add(wrSession); - } - else - { - PlayerStats? thisSessionStats = thisSessionUserData.GameStats; - if (thisSessionStats == null) - { - thisSession.IsRegisteredForMatchmaking = false; - lstDestroy.Add(wrSession); - await SendMatchmakingMessage(thisSession, "Matchmaking could not start because your player statistics are unavailable. Please try again."); - continue; - } - - if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) - { - - // TODO_MATCHAMAKING: Better way of tracking this, we need to know who is already in a bucket - // Was the user in a bucket? if so theres nothing to do in terms of bucket management - bool bUseInBucket = false; - MatchmakingBucket? mmBucketUserIsIn = null; - foreach (var kvBucketPair in m_dictMatchmakingBuckets) - { - foreach (MatchmakingBucket mmBucket in kvBucketPair.Value) - { - if (mmBucket.HasPlayer(thisSession)) - { - bUseInBucket = true; - mmBucketUserIsIn = mmBucket; - break; - } - } - - if (bUseInBucket) - { - break; - } - } - - if (bUseInBucket) - { - // already sorted into a bucket - stop tracking this session here, otherwise a stale entry - // can sort the same player into a second bucket (and therefore a second lobby) later on - lstDestroy.Add(wrSession); - } - else - { - // is there a suitable bucket for us - // TODO_MATCHMAKING: Optimize lookup - if (m_dictMatchmakingBuckets.ContainsKey(thisSession.MatchmakingPlaylistID)) - { - MatchmakingBucket? bucketInUse = null; - foreach (MatchmakingBucket mmBucket in m_dictMatchmakingBuckets[thisSession.MatchmakingPlaylistID]) - { - if (mmBucket.IsMergedAway()) - { - continue; - } - - // must be within initial elo threshold for a join, otherwise we'll make a bucket and try to merge buckets using the elo iteration expansion algorithm - int matchmakingElo = MatchmakingManager.GetMatchmakingElo(thisSessionUserData.GameStats); - int eloExpansionToUse = (mmBucket.GetAvgElo() >= EloConfig.HighEloThreshold || matchmakingElo >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; - if (mmBucket.IsAvgEloWithinThreshold(matchmakingElo, eloExpansionToUse)) - { - // TODO_MATCHMAKING: Squads - if (mmBucket.HasSpaceForUsers(1, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID)) - { - // do the maps overlap? if so we can join - if (mmBucket.DoMapSelectionsIntersect(thisSession.MatchmakingMapIndicies)) - { - bool bJoined = await mmBucket.Join(thisSession); - - if (bJoined) - { - // stop looking - joining more than one bucket would matchmake this player into multiple lobbies - bucketInUse = mmBucket; - break; - } - } - } - } - } - - // didnt find a bucket? make one - if (bucketInUse == null) - { - MatchmakingBucket newBucket = new MatchmakingBucket(playlist.PlaylistID, thisSession, playlist.MinPlayers, playlist.DesiredPlayers, thisSession.MatchmakingMapIndicies, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID); - m_dictMatchmakingBuckets[thisSession.MatchmakingPlaylistID].Add(newBucket); - bucketInUse = newBucket; - } - - // send status to use - await SendMatchmakingMessage(thisSession, String.Format("You are now matchmaking in playlist \"{0}\". There are currently {1} player(s) searching for a match in this playlist", playlist.Name, GetTotalQueuedPlayersInPlaylist(playlist.PlaylistID))); - await SendMatchmakingMessage(thisSession, String.Format("Status: {0}/{1} players. ({2} required to start)", bucketInUse.CurrentMemberCount(), bucketInUse.DesiredPlayers, bucketInUse.MinPlayers)); - - // now remove us from lstSessions, this list is essentially people who need sorted into a bucket - lstDestroy.Add(wrSession); - } - } - } - else - { - // invalid playlist somehow - thisSession.IsRegisteredForMatchmaking = false; - lstDestroy.Add(wrSession); - } - } - } - finally - { - thisSession.MatchmakingStateLock.Release(); - } - } - } - - // now remove - foreach (WeakReference wrSession in lstDestroy) - { - if (wrSession != null) - { - lstSessions.Remove(wrSession); - } - } - } - - // TODO_MATCHMAKING: Deregister player if they disconnect or leave quickmatch - private static ConcurrentList> lstSessions = new(); - - private static bool IsPendingSession(UserSession session) - { - foreach (WeakReference wrSession in lstSessions) - { - if (wrSession.TryGetTarget(out UserSession? pendingSession) && ReferenceEquals(pendingSession, session)) - { - return true; - } - } - - return false; - } - - private static void RemovePendingSession(UserSession session) - { - foreach (WeakReference wrSession in lstSessions.ToList()) - { - if (!wrSession.TryGetTarget(out UserSession? pendingSession) || ReferenceEquals(pendingSession, session)) - { - lstSessions.Remove(wrSession); - } - } - } - - private static async Task TryRequeueRegisteredPlayer(UserSession session, byte[] requeueActionJSON) - { - await session.MatchmakingStateLock.WaitAsync(); - try - { - if (!session.IsRegisteredForMatchmaking) - { - return false; - } - - if (!IsPendingSession(session)) - { - lstSessions.Add(new WeakReference(session)); - } - - session.UpdateSessionLobbyID(-1); - session.QueueWebsocketSend(requeueActionJSON); - return true; - } - finally - { - session.MatchmakingStateLock.Release(); - } - } - - private static ConcurrentQueue m_bucketsPendingDeletion = new(); - public static void DestroyBucket(MatchmakingBucket bucket) - { - bucket.MarkPendingDeletion(); - m_bucketsPendingDeletion.Enqueue(bucket); - } - - public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) - { - // validate the request - a bad playlist or out of range map index from a client must never reach a bucket - if (!g_Playlists.TryGetValue(playlistID, out Playlist? playlist)) - { - await SendMatchmakingMessage(plr, "That playlist is not available. Matchmaking was not started."); - return; - } - - List validatedMapIndices = mapIndices - .Where(mapIndex => mapIndex >= 0 && mapIndex < playlist.Maps.Count) - .Distinct() - .ToList(); - - int minSelectedMaps = Math.Max(1, playlist.MinSelectedMaps); - if (validatedMapIndices.Count < minSelectedMaps) - { - await SendMatchmakingMessage(plr, String.Format("You must select at least {0} valid map(s) to matchmake in this playlist.", minSelectedMaps)); - return; - } - - bool bCancellationRejected; - await plr.MatchmakingStateLock.WaitAsync(); - try - { - // A duplicate registration must not leave the player queued twice or in two buckets. - plr.IsRegisteredForMatchmaking = false; - RemovePendingSession(plr); - bCancellationRejected = await RemovePlayerFromAllBuckets(plr); - - if (!bCancellationRejected) - { - plr.MatchmakingPlaylistID = playlistID; - plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); - plr.ExeCRC = exe_crc; - plr.IniCRC = ini_crc; - plr.AnticheatID = anticheatID; - plr.IsRegisteredForMatchmaking = true; - lstSessions.Add(new WeakReference(plr)); - } - } - finally - { - plr.MatchmakingStateLock.Release(); - } - - if (bCancellationRejected) - { - await SendMatchmakingMessage(plr, "Matchmaking cannot be restarted because your game is already starting."); - return; - } - - await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); - } - - private static async Task RemovePlayerFromAllBuckets(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - bool bCancellationRejected = false; - - // also remove from any bucket we are in to avoid ghost buckets - foreach (var kvPair in m_dictMatchmakingBuckets) - { - foreach (MatchmakingBucket mmBucket in kvPair.Value) - { - bool bRemoved = mmBucket.RemovePlayer(plr, out bool bBucketCancellationRejected); - bCancellationRejected |= bBucketCancellationRejected; - - if (bRemoved) - { - // remove from QM lobby too - Lobby? lobby = lobbyManager.GetLobby(mmBucket.GetLobbyID()); - if (lobby != null) - { - LobbyMember? lobbyMember = lobby.GetMemberFromUserID(plr.m_UserID); - if (lobbyMember != null) - { - Console.WriteLine("User {0} Leave MM Lobby", plr.m_UserID); - await lobby.RemoveMember(lobbyMember); - } - } - - // if we're the last player, destroy the bucket - if (mmBucket.CurrentMemberCount() == 0) - { - DestroyBucket(mmBucket); - } - } - } - } - - return bCancellationRejected; - } - - public static async Task DeregisterPlayer(UserSession plr) - { - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - bool bCancellationRejected; - await plr.MatchmakingStateLock.WaitAsync(); - try - { - plr.IsRegisteredForMatchmaking = false; - RemovePendingSession(plr); - bCancellationRejected = await RemovePlayerFromAllBuckets(plr); - } - finally - { - plr.MatchmakingStateLock.Release(); - } - - // leave QM lobby too - if (!bCancellationRejected) - { - Console.WriteLine("[Source 4] User {0} Leave Any Lobby", plr.m_UserID); - await lobbyManager.LeaveAnyLobby(plr.m_UserID); - } - } +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU Affero General Public License for more details. +** +** You should have received a copy of the GNU Affero General Public License +** along with this program. If not, see . +*/ + +using Database; +using Discord.Rest; +using GenOnlineService; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using System; +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.WebSockets; +using System.Security.Claims; +using System.Text; +using System.Text.Json; + +public class PlaylistMap +{ + public string Name { get; private set; } + public string Path { get; private set; } + public bool Custom { get; private set; } + public int MaxPlayers { get; private set; } + + public PlaylistMap(string strName, string strPath, bool bCustom, int maxPlayers) + { + Name = strName; + Path = strPath; + Custom = bCustom; + MaxPlayers = maxPlayers; + } +} + +public class ConcurrentList +{ + private List m_internalList; + private readonly object m_lockObj = new object(); + + public ConcurrentList() + { + m_internalList = new List(); + } + + public ConcurrentList(IEnumerable collection) + { + if (collection != null) + { + m_internalList = new List(collection); + } + else + { + m_internalList = new List(); + } + } + + public void AddRange(IEnumerable collection) + { + lock (m_lockObj) + { + m_internalList.AddRange(collection); + } + } + + public void Add(T item) + { + lock (m_lockObj) + { + m_internalList.Add(item); + } + } + + public void Clear() + { + lock (m_lockObj) + { + m_internalList.Clear(); + } + } + + public T this[int index] + { + get + { + lock (m_lockObj) + { + if (index < 0 || index >= m_internalList.Count) + throw new IndexOutOfRangeException("Index out of range."); + return m_internalList[index]; + } + } + } + + public int Count + { + get + { + lock (m_lockObj) + { + return m_internalList.Count; + } + } + } + + public bool Contains(T item) + { + lock (m_lockObj) + { + return m_internalList.Contains(item); + } + } + public bool Remove(T item) + { + lock (m_lockObj) + { + return m_internalList.Remove(item); + } + } + public List ToList() + { + lock (m_lockObj) + { + return m_internalList.ToList(); + } + } + public IEnumerator GetEnumerator() + { + lock (m_lockObj) + { + return m_internalList.ToList().GetEnumerator(); + } + } +} + +public class Playlist +{ + public UInt16 PlaylistID { get; private set; } + public string Name { get; private set; } + public int MinPlayers { get; private set; } + public int DesiredPlayers { get; private set; } + public int MinSelectedMaps { get; private set; } + + public bool AllowTeams { get; private set; } + public int TeamSize { get; private set; } + public bool AllowArmySelection { get; private set; } + public UInt16 GracePeriodAtMinPlayersMSec { get; private set; } + public List Maps { get; private set; } + + public Playlist(UInt16 a_PlaylistID, string a_strName, + int a_MinPlayers, int a_DesiredPlayers, int a_MinSelectedMaps, bool a_bAllowTeams, int a_TeamSize, bool a_bAllowArmySelection, UInt16 a_gracePeriodAtMinPlayersMSec, List allowedMaps) + { + PlaylistID = a_PlaylistID; + Name = a_strName; + MinPlayers = a_MinPlayers; + MinSelectedMaps = a_MinSelectedMaps; + DesiredPlayers = a_DesiredPlayers; + AllowTeams = a_bAllowTeams; + TeamSize = a_TeamSize; + AllowArmySelection = a_bAllowArmySelection; + GracePeriodAtMinPlayersMSec = a_gracePeriodAtMinPlayersMSec; + Maps = allowedMaps; + } +} + +static class MatchmakingManager +{ + private static readonly ILogger s_log = AppLog.For(typeof(MatchmakingManager)); + // World Series 2026 Qualification in September requires the matchmaking to be + // based off the monthly ELO. + internal static int GetMatchmakingElo(PlayerStats stats) + { + DateTimeOffset now = DateTimeOffset.UtcNow; + DateTimeOffset septemberStart = new DateTimeOffset(2026, 9, 1, 0, 0, 0, TimeSpan.Zero); + DateTimeOffset octoberStart = new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero); + + return now >= septemberStart && now < octoberStart + ? stats.MonthlyEloRating + : stats.EloRating; + } + + public static void PlayerWidenSearch(UserSession playerSession) + { + // NOTE: we dont check the state of the bucket here, but it doesn't really matter since expanding the maps after it started won't do anything anyway + + // remove the map limitation + foreach (var kvPair in m_dictMatchmakingBuckets) + { + foreach (MatchmakingBucket mmBucket in kvPair.Value) + { + if (mmBucket.HasPlayer(playerSession)) + { + // get the maximum map set for this playlist + if (MatchmakingManager.g_Playlists.TryGetValue(mmBucket.PlaylistID, out Playlist? playlist)) + { + List lstAllMaps = new(); + for (int mapIndex = 0; mapIndex < playlist.Maps.Count; ++mapIndex) + { + lstAllMaps.Add(mapIndex); + } + + // are we the owner of the bucket? update the original list too + + MatchmakingBucketMember? bucketOwner = mmBucket.GetOwner(); + if (bucketOwner != null) + { + if (mmBucket.lstMapIndices.Count > 0 && bucketOwner.GetAssociatedSession() == playerSession) + { + mmBucket.lstMapIndices = new ConcurrentList(lstAllMaps); + } + } + + // update our player too + playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); + + // can't be in multiple buckets + return; + } + } + } + } + + // not in a bucket yet (still queued waiting to be sorted) - widen their selection anyway, otherwise the + // widen request is silently dropped and they get bucketed with their original map list + if (MatchmakingManager.g_Playlists.TryGetValue(playerSession.MatchmakingPlaylistID, out Playlist? queuedPlaylist)) + { + List lstAllMaps = new(); + for (int mapIndex = 0; mapIndex < queuedPlaylist.Maps.Count; ++mapIndex) + { + lstAllMaps.Add(mapIndex); + } + + playerSession.MatchmakingMapIndicies = new ConcurrentList(lstAllMaps); + } + } + + private static async Task SendMatchmakingMessage(UserSession cache, string message) + { + UserSession? sess = GenOnlineService.WebSocketManager.GetSessionFromUser(cache.m_UserID, cache.GetSessionType()); + if (sess != null) + { + WebSocketMessage_MatchmakingMessage msg = new WebSocketMessage_MatchmakingMessage(); + msg.msg_id = (int)EWebSocketMessageID.MATCHMAKING_MESSAGE; + msg.message = message; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(msg)); + + sess.QueueWebsocketSend(bytesJSON); + } + } + + public class MatchmakingBucketMember + { + WeakReference? m_SessionRef = null; + + public MatchmakingBucketMember(UserSession owningSession) + { + m_SessionRef = new WeakReference(owningSession); + } + + public bool Is(UserSession playerSession) + { + if (m_SessionRef != null) + { + if (m_SessionRef.TryGetTarget(out UserSession? thisSession)) + { + if (thisSession != null && thisSession == playerSession) + { + return true; + } + } + } + + return false; + } + + public UserSession? GetAssociatedSession() + { + if (m_SessionRef != null) + { + if (m_SessionRef.TryGetTarget(out UserSession? thisSession)) + { + return thisSession; + } + } + + return null; + } + } + + public class MatchmakingBucket + { + private static readonly ILogger s_log = AppLog.For(typeof(MatchmakingBucket)); + private ConcurrentList m_lstMembers = new(); + public ConcurrentList lstMapIndices { get; set; } = new(); + + private Int64 m_timeReachedMinPlayers = -1; + private bool m_bReachedMinPlayers = false; + private bool m_bWaitingOnLobbyJoins = false; + private bool m_bHasStartedCountdown = false; + private bool m_bWaitingOnMeshConnectivityChecks = false; + private bool m_bAutoStartInvalidated = false; + private bool m_bPendingDeletion = false; + private bool m_bMergedAway = false; + private bool m_bAbortInProgress = false; + private bool m_bStartCommitted = false; + private readonly object m_StateLock = new(); + + // a bucket that has been merged into another bucket (or that has already been handed a lobby) must never + // accept or donate members again, otherwise a player ends up in two buckets and gets sent to two lobbies + public bool IsMergedAway() + { + lock (m_StateLock) + { + return m_bMergedAway; + } + } + + public bool IsLockedForMatchStart() + { + lock (m_StateLock) + { + return m_bCreatingLobby || m_bHasStartedCountdown || m_bWaitingOnLobbyJoins || m_bWaitingOnMeshConnectivityChecks || m_bPendingDeletion; + } + } + + private bool IsPendingDeletion() + { + lock (m_StateLock) + { + return m_bPendingDeletion; + } + } + + public UInt16 PlaylistID { get; private set; } + public int MinPlayers { get; private set; } + public int DesiredPlayers { get; private set; } + + public UInt32 ExeCRC { get; private set; } + public UInt32 IniCRC { get; private set; } + + public EKnownAnticheatID AnticheatID { get; private set; } + + public int eloExpansionIteration { get; private set; } = 1; // 1 * EloExpansionValue, matches the initial value + + public void ExpandElo() + { + ++eloExpansionIteration; + m_LastELOExpansionTime = DateTime.Now; + } + + DateTime m_CreationTime = DateTime.Now; + DateTime m_LastELOExpansionTime = DateTime.Now; + + private TimeSpan TimeSinceLastEloExpansion() + { + TimeSpan timeDifference = DateTime.Now - m_LastELOExpansionTime; + return timeDifference; + } + + public MatchmakingBucketMember? GetOwner() + { + if (m_lstMembers.Count > 0) + { + return m_lstMembers[0]; + } + + return null; + } + + private sealed record Formation(ImmutableArray Players, PlaylistMap Map, bool UsedFallbackMap); + private Formation? m_Formation; + private bool m_bCreatingLobby; + private static long s_NextBucketID; + private readonly long m_BucketID = Interlocked.Increment(ref s_NextBucketID); + + private Formation? TryBeginFormation(Playlist playlist, bool graceExpired) + { + lock (m_StateLock) + { + if (IsLockedForMatchStart() || m_bMergedAway) return null; + var players = m_lstMembers.ToList().Select(member => member.GetAssociatedSession()).ToArray(); + if (players.Length < MinPlayers || players.Length > DesiredPlayers + || (!graceExpired && players.Length < DesiredPlayers) + || players.Any(player => player == null || player.IsRetired || !player.IsRegisteredForMatchmaking)) return null; + + var selectedPlayers = players.Cast().ToArray(); + var sharedMaps = new HashSet(lstMapIndices.ToList()); + foreach (var player in selectedPlayers) + sharedMaps.IntersectWith(player.MatchmakingMapIndicies.ToList()); + var exactMaps = playlist.Maps.Where(map => map.MaxPlayers == selectedPlayers.Length).ToArray(); + var preferredMaps = sharedMaps.Where(index => index >= 0 && index < playlist.Maps.Count) + .Select(index => playlist.Maps[index]).Where(map => map.MaxPlayers == selectedPlayers.Length).ToArray(); + var choices = preferredMaps.Length > 0 ? preferredMaps : exactMaps; + if (choices.Length == 0) return null; + + m_Formation = new Formation(selectedPlayers.ToImmutableArray(), choices[Random.Shared.Next(choices.Length)], preferredMaps.Length == 0); + m_bCreatingLobby = true; + m_bReachedMinPlayers = false; + m_timeReachedMinPlayers = -1; + return m_Formation; + } + } + + private bool IsFormationCurrent(Lobby? lobby = null) + { + lock (m_StateLock) + { + return m_Formation != null && !m_bAutoStartInvalidated && !m_bPendingDeletion + && CurrentMemberCount() == m_Formation.Players.Length + && m_Formation.Players.All(player => !player.IsRetired && player.IsRegisteredForMatchmaking && HasPlayer(player)) + && (lobby == null || (lobby.NumCurrentPlayers == m_Formation.Players.Length + && m_Formation.Players.All(player => lobby.GetMemberFromUserID(player.m_UserID)?.GetSession() + .TryGetTarget(out var memberSession) == true && ReferenceEquals(memberSession, player)))); + } + } + + private async Task CreateFormationLobbyAsync(Playlist playlist, Formation formation) + { + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + try + { + using var scope = ServiceLocator.Services.CreateScope(); + var factory = scope.ServiceProvider.GetRequiredService>(); + await using var db = await factory.CreateDbContextAsync(); + foreach (var player in formation.Players) + { + await Database.Leaderboards.CreateUserEntriesIfNotExists(db, player.m_UserID); + await SendMatchmakingMessage(player, $"Creating a {formation.Players.Length}-player QuickMatch on {formation.Map.Name}." + + (formation.Players.Length < playlist.DesiredPlayers ? " Starting below the playlist maximum after the grace period." : "") + + (formation.UsedFallbackMap ? " No shared map preference fits this size; using an exact-size playlist map." : "")); + } + var host = formation.Players[0]; + var hostData = GenOnlineService.WebSocketManager.GetSharedDataForUser(host.m_UserID); + if (hostData == null) throw new InvalidOperationException("QuickMatch host session disappeared."); + if (!IsFormationCurrent()) throw new InvalidOperationException("QuickMatch formation changed."); + m_LobbyID = await lobbyManager.CreateLobby(db, host, hostData.m_strDisplayName, "Quickmatch Lobby", formation.Map.Name, formation.Map.Path + ".map", + true, formation.Players.Length, "", 12345, false, true, 10000, false, String.Empty, -5, false, Constants.g_DefaultCameraMaxHeight, + ExeCRC, IniCRC, ELobbyType.QuickMatch, AnticheatID); + var lobby = lobbyManager.GetLobby(m_LobbyID) ?? throw new InvalidOperationException("QuickMatch lobby creation failed."); + foreach (var player in formation.Players) lobby.MatchedUserIDs.Add(player.m_UserID); + var action = new WebSocketMessage_MatchmakerJoinLobby + { + msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_JOIN_PREARRANGED_LOBBY, + lobby_id = m_LobbyID + }; + var payload = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(action)); + lock (m_StateLock) + { + if (!IsFormationCurrent()) throw new InvalidOperationException("QuickMatch formation changed."); + m_bWaitingOnLobbyJoins = true; + m_timeStartedWaitingOnLobbyJoins = Environment.TickCount64; + foreach (var player in formation.Players) player.QueueWebsocketSend(payload); + } + } + catch (Exception ex) + { + s_log.LogError(ex, "QuickMatch formation failed for bucket {BucketId}", m_BucketID); + await AbortQuickMatchAutoStartAsync("QuickMatch setup failed or a player left during formation."); + } + finally + { + lock (m_StateLock) m_bCreatingLobby = false; + } + } + + public bool DoMapSelectionsIntersect(ConcurrentList lstRhs) + { + var mapSet = new HashSet(lstMapIndices.ToList()); + var mapSetRhs = new HashSet(lstRhs.ToList()); + foreach (int mapIndex in mapSetRhs) + { + if (mapSet.Contains(mapIndex)) + { + return true; + } + } + return false; + } + + public bool CanMergeWithOtherBucket(MatchmakingBucket bucketToMerge) + { + if (IsPendingDeletion() || bucketToMerge.IsPendingDeletion()) + { + return false; + } + + // playlist must match + if (bucketToMerge.PlaylistID != this.PlaylistID) + { + return false; + } + + // either bucket already merged away this tick? it is stale, never touch it again + if (IsMergedAway() || bucketToMerge.IsMergedAway()) + { + return false; + } + + // if we're already counting down, dont let people join, just pretend we are full + if (IsLockedForMatchStart()) + { + return false; + } + + // the other bucket may have already been given a lobby this tick - its members are on their way into + // that lobby, so pulling them in here would matchmake them into a second lobby + if (bucketToMerge.IsLockedForMatchStart()) + { + return false; + } + + // must have space for all users in the rhs bucket, and CRCs must match + if (!HasSpaceForUsers(bucketToMerge.CurrentMemberCount(), bucketToMerge.ExeCRC, bucketToMerge.IniCRC, bucketToMerge.AnticheatID)) + { + return false; + } + + // must be within the eloThreshold + int eloExpansionToUse = (bucketToMerge.GetAvgElo() >= EloConfig.HighEloThreshold || GetAvgElo() >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; + if (!IsAvgEloWithinThreshold(bucketToMerge.GetAvgElo(), eloExpansionIteration * eloExpansionToUse)) + { + return false; + } + + // cant be blocked by any participant (or have any participant blocked) + // TODO_OPTIMIZE: This is O(n^2) + foreach (MatchmakingBucketMember rhsMember in bucketToMerge.m_lstMembers) + { + UserSession? rhsSession = rhsMember.GetAssociatedSession(); + + if (rhsSession != null) + { + if (IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(rhsSession, rhsSession.m_UserID)) + { + return false; + } + } + } + + + // must have overlap in our map selections + if (!bucketToMerge.DoMapSelectionsIntersect(this.lstMapIndices)) + { + return false; + } + + return true; + } + + public async Task MergeWithOtherBucketAsync(MatchmakingBucket bucketToMerge) + { + if (ReferenceEquals(this, bucketToMerge)) return false; + var first = m_BucketID < bucketToMerge.m_BucketID ? this : bucketToMerge; + var second = ReferenceEquals(first, this) ? bucketToMerge : this; + lock (first.m_StateLock) + lock (second.m_StateLock) + { + if (!CanMergeWithOtherBucket(bucketToMerge)) return false; + foreach (var member in bucketToMerge.m_lstMembers) + { + var session = member.GetAssociatedSession(); + if (session != null && !session.IsRetired && !HasPlayer(session)) m_lstMembers.Add(member); + } + bucketToMerge.m_bMergedAway = true; + bucketToMerge.m_lstMembers.Clear(); + } + + // tell all players + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? session = member.GetAssociatedSession(); + if (session != null) + { + await SendMatchmakingMessage(session, String.Format("Your matchmaking bucket was merged with another bucket. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); + } + } + return true; + } + + public bool HasPlayer(UserSession playerSession) + { + foreach (MatchmakingBucketMember member in m_lstMembers) + { + if (member.Is(playerSession)) + { + return true; + } + } + + return false; + } + + public bool RemovePlayer(UserSession playerSession, out bool bCancellationRejected) + { + bCancellationRejected = false; + lock (m_StateLock) + { + foreach (MatchmakingBucketMember member in m_lstMembers) + { + if (member.Is(playerSession)) + { + if (m_bStartCommitted) + { + bCancellationRejected = true; + return false; + } + + if (m_bCreatingLobby || m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + } + + m_lstMembers.Remove(member); + return true; + } + } + } + + return false; + } + + public int CurrentMemberCount() + { + return m_lstMembers.Count; + } + + // members whose UserSession has been collected/disconnected are dead weight - they inflate the member count, + // which both blocks the "everyone joined the lobby" check and skews the average elo + public bool PruneDeadMembers() + { + bool bInvalidatedSetup = false; + lock (m_StateLock) + { + foreach (MatchmakingBucketMember member in m_lstMembers) + { + if (member.GetAssociatedSession() == null) + { + if (m_bCreatingLobby || m_bWaitingOnLobbyJoins || m_bHasStartedCountdown || m_bWaitingOnMeshConnectivityChecks) + { + m_bAutoStartInvalidated = true; + bInvalidatedSetup = true; + } + + m_lstMembers.Remove(member); + } + } + } + + return bInvalidatedSetup; + } + + internal void MarkPendingDeletion() + { + lock (m_StateLock) + { + m_bPendingDeletion = true; + } + } + + // TODO_EFCORE: Shared User data, and session<->websocket could be weakrefs + public bool IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(UserSession? joiningUserSession, Int64 joining_user) + { + // NOTE: We check blocking in both directions, joiner blocked them, or joiner is blocked by a player + SharedUserData? joiningUserData = joiningUserSession != null + ? GenOnlineService.WebSocketManager.GetSharedDataForUser(joiningUserSession.m_UserID) + : null; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); + + if (memberUserData != null) + { + // Bucket member has blocked the joining user + if (memberUserData.GetSocialContainer().Blocked.Contains(joining_user)) + return true; + + // Joining user has blocked the bucket member + if (joiningUserData?.GetSocialContainer().Blocked.Contains(memberSession.m_UserID) == true) + return true; + } + } + } + + return false; + } + + public bool HasSpaceForUsers(int numUsers, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + { + // stale buckets must never accept new members + if (IsMergedAway() || IsPendingDeletion()) + { + return false; + } + + // if we're already counting down, dont let people join, just pretend we are full + if (IsLockedForMatchStart()) + { + return false; + } + + // crcs must match too + if (exe_crc != ExeCRC || ini_crc != IniCRC) + { + return false; + } + + // must be running same AC + if (anticheatID != AnticheatID) + { + return false; + } + + return numUsers <= (DesiredPlayers - m_lstMembers.Count); + } + + public bool IsAvgEloWithinThreshold(int playerElo, int eloThreshold) + { + // calculate an average elo if it's a team game, othewrise average will just be the other player + int avgElo = GetAvgElo(); + + int lowerEloBound = playerElo - eloThreshold; + int upperEloBound = playerElo + eloThreshold; + + return (avgElo >= lowerEloBound && avgElo <= upperEloBound); + } + + public int GetAvgElo() + { + int numMembers = m_lstMembers.Count; + + if (numMembers == 0) + { + return EloConfig.BaseRating; + } + + int avgElo = 0; + int numContributingMembers = 0; + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + SharedUserData? memberUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(memberSession.m_UserID); + + if (memberUserData?.GameStats != null) + { + avgElo += MatchmakingManager.GetMatchmakingElo(memberUserData.GameStats); + ++numContributingMembers; + } + } + } + + // only divide by the members we actually got an elo for, otherwise dead/unknown sessions drag the average down + if (numContributingMembers == 0) + { + return EloConfig.BaseRating; + } + + avgElo /= numContributingMembers; + + return avgElo; + } + + + public async Task JoinAsync(UserSession playerSession) + { + lock (m_StateLock) + { + if (HasPlayer(playerSession)) return true; + if (playerSession.IsRetired || IsJoiningUserBlockedByOrHasBlockedAnyBucketMember(playerSession, playerSession.m_UserID) + || !HasSpaceForUsers(1, playerSession.ExeCRC, playerSession.IniCRC, playerSession.AnticheatID)) return false; + m_lstMembers.Add(new MatchmakingBucketMember(playerSession)); + } + foreach (var member in m_lstMembers) + { + var session = member.GetAssociatedSession(); + if (session != null) + await SendMatchmakingMessage(session, $"Status: {CurrentMemberCount()}/{DesiredPlayers} players. ({MinPlayers} required to start)"); + } + return true; + } + + public MatchmakingBucket(UInt16 playlistID, UserSession owningSession, int minPlayers, int desiredPlayers, ConcurrentList mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + { + PlaylistID = playlistID; + MinPlayers = minPlayers; + DesiredPlayers = desiredPlayers; + lstMapIndices = mapIndices; + ExeCRC = exe_crc; + IniCRC = ini_crc; + AnticheatID = anticheatID; + + m_lstMembers.Add(new MatchmakingBucketMember(owningSession)); + } + + public Int64 GetLobbyID() + { + return m_LobbyID; + } + + Int64 m_LobbyID = -1; + Int64 m_StartTime = -1; + Int64 m_timeStartedWaitingOnLobbyJoins = -1; + + // how long we give everyone to actually connect to the QuickMatch lobby before we give up on the stragglers + private const Int64 LobbyJoinTimeoutMs = 45000; + private const int GameStartCountdownMs = 5000; + private const int SetupClientTimeoutMarginMs = 2000; + + // older clients treat any timeout_ms below this as the start countdown + private const int LegacyClientCountdownThresholdMs = 10000; + + private async Task StartGameAfterSuccessfulMeshCheck(Lobby lobby) + { + s_log.LogInformation("Starting QuickMatch game for lobby {LobbyId}", lobby.LobbyID); + AppMetrics.RecordMatchmakingMatch(PlaylistID.ToString(CultureInfo.InvariantCulture), "started"); + + WebSocketMessage_MatchmakerStartGame startGameAction = new WebSocketMessage_MatchmakerStartGame(); + startGameAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_START_GAME; + byte[] bytesJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(startGameAction)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + MatchmakingManager.RecordQueueExit(memberSession, "matched"); + memberSession.IsRegisteredForMatchmaking = false; + memberSession.QueueWebsocketSend(bytesJSON); + } + } + + await lobby.UpdateState(ELobbyState.INGAME); + MatchmakingManager.DestroyBucket(this); + } + + private void QueueSetupProgress(int timeoutMSec, int countdownMSec = 0) + { + WebSocketMessage_MatchmakerSetupProgress setupProgress = new WebSocketMessage_MatchmakerSetupProgress(); + setupProgress.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_SETUP_PROGRESS; + setupProgress.timeout_ms = timeoutMSec; + setupProgress.countdown_ms = countdownMSec; + byte[] setupProgressJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(setupProgress)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + memberSession.QueueWebsocketSend(setupProgressJSON); + } + } + } + + private long m_MeshCheckID; + + private async Task TriggerFullMeshConnectivityChecks(Lobby lobby) + { + m_MeshCheckID = await lobby.StartMeshCheckAsync(); + lock (m_StateLock) + { + // Publish the state transition before any notification work. If a send fails, the + // regular lobby timeout still completes or aborts the setup instead of stranding it. + m_bWaitingOnMeshConnectivityChecks = true; + } + + // kept at or above the legacy threshold so short tuned checks don't show a start countdown on older clients + QueueSetupProgress(Math.Max(FullMeshCheckSettings.MaxDurationMs + SetupClientTimeoutMarginMs, LegacyClientCountdownThresholdMs)); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Preparing match..."); + } + } + } + + private async Task AbortQuickMatchAutoStartAsync(string reason, bool dropNonJoiners = false) + { + List sessionsToRequeue = new(); + long abortedLobbyID; + lock (m_StateLock) + { + if (m_bAbortInProgress || m_bStartCommitted) + { + return; + } + + m_bAbortInProgress = true; + abortedLobbyID = m_LobbyID; + m_bPendingDeletion = true; + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + sessionsToRequeue.Add(memberSession); + } + } + + m_StartTime = -1; + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_bWaitingOnMeshConnectivityChecks = false; + } + AppMetrics.RecordMatchmakingMatch(PlaylistID.ToString(CultureInfo.InvariantCulture), "setup_aborted"); + + LobbyManager lobbyManager = ServiceLocator.Services.GetRequiredService(); + Lobby? quickMatchLobby = lobbyManager.GetLobby(abortedLobbyID); + if (quickMatchLobby != null) + { + if (dropNonJoiners) + { + // Close admission before classifying late arrivals; never change a newer registration. + await quickMatchLobby.UpdateState(ELobbyState.COMPLETE); + foreach (var session in sessionsToRequeue) + { + if (quickMatchLobby.GetMemberFromUserID(session.m_UserID) != null) continue; + bool dropped = false; + await session.MatchmakingStateLock.WaitAsync(); + try + { + lock (m_StateLock) + { + if (HasPlayer(session)) + { + MatchmakingManager.RecordQueueExit(session, "join_timeout"); + session.IsRegisteredForMatchmaking = false; + dropped = true; + } + } + } + finally { session.MatchmakingStateLock.Release(); } + if (dropped) await SendMatchmakingMessage(session, "You failed to join the QuickMatch lobby in time and have been removed from matchmaking."); + } + } + foreach (UserSession memberSession in sessionsToRequeue) + { + LobbyMember? lobbyMember = quickMatchLobby.GetMemberFromUserID(memberSession.m_UserID); + if (lobbyMember != null) + { + // TODO: Remove this fallback once all supported clients handle MATCHMAKING_ACTION_REQUEUE. + // Legacy clients do not understand the requeue action, but they can still tear down + // peer and anti-cheat connections before joining the next temporary lobby. + await lobbyManager.RemoveMatchmakingMemberAsync(quickMatchLobby, lobbyMember, sendPeerTeardown: true); + } + } + + await lobbyManager.DeleteLobby(quickMatchLobby); + } + + WebSocketMessage_Simple requeueAction = new WebSocketMessage_Simple(); + requeueAction.msg_id = (int)EWebSocketMessageID.MATCHMAKING_ACTION_REQUEUE; + byte[] requeueActionJSON = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(requeueAction)); + + foreach (UserSession memberSession in sessionsToRequeue) + { + if (await lobbyManager.TryRequeueMatchmakingAsync(memberSession, abortedLobbyID, requeueActionJSON)) + { + await SendMatchmakingMessage(memberSession, reason); + await SendMatchmakingMessage(memberSession, "Re-queueing you into matchmaking..."); + } + } + + m_lstMembers.Clear(); + m_LobbyID = -1; + + MatchmakingManager.DestroyBucket(this); + } + + public async Task Tick() + { + bool bPendingDeletion; + bool bAutoStartInvalidated; + bool bWaitingOnMeshConnectivityChecks; + bool bWaitingOnLobbyJoins; + bool bHasStartedCountdown; + lock (m_StateLock) + { + if (m_bCreatingLobby) return; + bPendingDeletion = m_bPendingDeletion; + bAutoStartInvalidated = m_bAutoStartInvalidated; + bWaitingOnMeshConnectivityChecks = m_bWaitingOnMeshConnectivityChecks; + bWaitingOnLobbyJoins = m_bWaitingOnLobbyJoins; + bHasStartedCountdown = m_bHasStartedCountdown; + } + + // merged/deleted buckets must not create another lobby or continue a committed setup + if (IsMergedAway() || bPendingDeletion) + { + return; + } + + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + + // drop any members whose session has gone away, otherwise they are counted forever and the bucket can + // never reach the "everyone is in the lobby" condition + if (PruneDeadMembers()) + { + bAutoStartInvalidated = true; + } + + // TODO_QUICKMATCH: What if the playlist is null? is this even possible since we validated before creating the bucket + if (g_Playlists.TryGetValue(PlaylistID, out Playlist? playlist)) + { + // nobody left? clean ourselves up rather than lingering as a ghost bucket + if (CurrentMemberCount() == 0 && !bWaitingOnLobbyJoins && !bHasStartedCountdown) + { + MatchmakingManager.DestroyBucket(this); + return; + } + + if (bAutoStartInvalidated) + { + await AbortQuickMatchAutoStartAsync("QuickMatch auto-start was aborted because a player left during match setup."); + return; + } + + if (bWaitingOnMeshConnectivityChecks) + { + Lobby? lobbyDuringMeshCheck = lobbyManager.GetLobby(m_LobbyID); + if (lobbyDuringMeshCheck == null) + { + await AbortQuickMatchAutoStartAsync("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + await lobbyDuringMeshCheck.ProcessPendingFullMeshConnectivityChecks(); + + if (await lobbyDuringMeshCheck.GetValidatedMeshCheckResultAsync(m_MeshCheckID) is bool meshComplete) + { + bool bStartCountdown; + bool bAbortStart; + bool bInvalidatedAtDecision; + lock (m_StateLock) + { + bInvalidatedAtDecision = m_bAutoStartInvalidated; + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartCountdown = false; + bAbortStart = false; + } + else + { + m_bWaitingOnMeshConnectivityChecks = false; + bStartCountdown = IsFormationCurrent(lobbyDuringMeshCheck) && meshComplete; + bAbortStart = !bStartCountdown; + if (bStartCountdown) + { + m_StartTime = Environment.TickCount64 + GameStartCountdownMs; + m_bHasStartedCountdown = true; + } + } + } + + if (bStartCountdown) + { + QueueSetupProgress(GameStartCountdownMs + SetupClientTimeoutMarginMs, GameStartCountdownMs); + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, $"Starting game in {GameStartCountdownMs / 1000} seconds."); + } + } + } + else if (bAbortStart) + { + string reason = bInvalidatedAtDecision + ? "QuickMatch auto-start was aborted because a player left during match setup." + : "QuickMatch auto-start was aborted because not all players were fully mesh-connected."; + await AbortQuickMatchAutoStartAsync(reason); + } + } + + return; + } + + if (bHasStartedCountdown) + { + if (Environment.TickCount64 < m_StartTime) + { + return; + } + + Lobby? lobbyAfterCountdown = lobbyManager.GetLobby(m_LobbyID); + if (lobbyAfterCountdown == null) + { + await AbortQuickMatchAutoStartAsync("QuickMatch auto-start was aborted because the temporary lobby no longer exists."); + return; + } + + bool meshStillValid = await lobbyAfterCountdown.GetValidatedMeshCheckResultAsync(m_MeshCheckID) == true; + bool bStartGame; + bool bAbortStart; + lock (m_StateLock) + { + if (m_bStartCommitted || m_bAbortInProgress || m_bPendingDeletion) + { + bStartGame = false; + bAbortStart = false; + } + else + { + m_bHasStartedCountdown = false; + m_StartTime = -1; + bStartGame = meshStillValid && IsFormationCurrent(lobbyAfterCountdown); + bAbortStart = !bStartGame; + if (bStartGame) + { + m_bStartCommitted = true; + m_bPendingDeletion = true; + } + } + } + + if (bStartGame) + { + await StartGameAfterSuccessfulMeshCheck(lobbyAfterCountdown); + } + else if (bAbortStart) + { + await AbortQuickMatchAutoStartAsync("QuickMatch auto-start was aborted because a player left during match setup."); + } + + return; + } + + // do we need to start? + // TODO_MATCHMAKING: Add a timeout at which > min players starts + if (!m_bWaitingOnLobbyJoins && !m_bHasStartedCountdown) + { + // must have a min player count + if (MinPlayers != DesiredPlayers) + { + // have we hit the min player count? start a timer + // NOTE: >= not ==, a merge (or several joins in one tick) can jump straight past MinPlayers + if (!m_bReachedMinPlayers && CurrentMemberCount() >= MinPlayers) + { + m_bReachedMinPlayers = true; + m_timeReachedMinPlayers = Environment.TickCount64; + + // tell players we reached min and will start in + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, + String.Format("Matchmaker has reached the minimum number of players required. Starting in {0} seconds (more players can still join in the mean time)", playlist.GracePeriodAtMinPlayersMSec / 1000)); + } + } + } + else if (m_bReachedMinPlayers && CurrentMemberCount() < MinPlayers) // did we go back under min players? stop countdown + { + m_bReachedMinPlayers = false; + m_timeReachedMinPlayers = -1; + + // tell players we stopped + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "One or more players have left which resulted in the matchmaker dropping below the minimum players required to start - the countdown to start has been cancelled"); + } + } + } + } + + // do we need an expansion of elo criteria? + if (TimeSinceLastEloExpansion().TotalSeconds >= EloConfig.SecondsBetweenEloExpansionsInMatchmaking) + { + // expand + ExpandElo(); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, "Expanding search criteria to find more players..."); + } + } + } + + // did we hit the timer OR have enough players to start? + bool bMinPlayersCountdownExpired = m_bReachedMinPlayers && (Environment.TickCount64 - m_timeReachedMinPlayers) > playlist.GracePeriodAtMinPlayersMSec; + var formation = TryBeginFormation(playlist, bMinPlayersCountdownExpired); + if (formation != null) await CreateFormationLobbyAsync(playlist, formation); + } + else + { + // do we need to cancel? people might have left + // TODO_QUICKMATCH: Re-enable + /* + if (m_StartTime != -1 || m_bWaitingOnLobbyJoins) + { + if (CurrentMemberCount() < MinPlayers) + { + m_bWaitingOnLobbyJoins = false; + m_bHasStartedCountdown = false; + m_StartTime = -1; + + // destroy lobby + Lobby? lobby = LobbyManager.GetLobby(m_LobbyID); + if (lobby != null) + { + await LobbyManager.DeleteLobby(lobby); + } + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + ActiveUserDataCache? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + await SendMatchmakingMessage(memberSession, String.Format("A player has left and the starting countdown has been cancelled. Status: {0}/{1} players. ({2} required to start)", CurrentMemberCount(), DesiredPlayers, MinPlayers)); + } + } + + return; + } + } + */ + + // waiting on lobby joins? + if (m_bWaitingOnLobbyJoins) + { + // done? start time etc + Lobby? lobby = lobbyManager.GetLobby(m_LobbyID); + if (lobby == null) + { + // the lobby went away underneath us (deleted/failed) - don't leave everyone stuck waiting forever + s_log.LogWarning("Matchmaking bucket lost QuickMatch lobby {LobbyId} while waiting on joins; abandoning bucket", m_LobbyID); + + foreach (MatchmakingBucketMember member in m_lstMembers) + { + UserSession? memberSession = member.GetAssociatedSession(); + if (memberSession != null) + { + MatchmakingManager.RecordQueueExit(memberSession, "setup_failed"); + memberSession.IsRegisteredForMatchmaking = false; + await SendMatchmakingMessage(memberSession, "The QuickMatch lobby could not be created. Please try matchmaking again."); + } + } + + m_bWaitingOnLobbyJoins = false; + m_timeStartedWaitingOnLobbyJoins = -1; + MatchmakingManager.DestroyBucket(this); + return; + } + + // has someone failed to connect? drop them so the rest of the players aren't stuck here forever + bool bJoinTimeoutExpired = m_timeStartedWaitingOnLobbyJoins != -1 + && (Environment.TickCount64 - m_timeStartedWaitingOnLobbyJoins) > LobbyJoinTimeoutMs; + + if (bJoinTimeoutExpired && !IsFormationCurrent(lobby)) + { + await AbortQuickMatchAutoStartAsync("Not all selected players joined the QuickMatch lobby in time.", dropNonJoiners: true); + return; + } + + { + if (IsFormationCurrent(lobby)) + { + m_timeStartedWaitingOnLobbyJoins = -1; + + lock (m_StateLock) + { + m_bWaitingOnLobbyJoins = false; + } + + // finalize the teams + const int playlistMaxPlayerPerTeam = 2; + bool bIsFFA = true; + int numTeams = lobby.NumCurrentPlayers / playlistMaxPlayerPerTeam; + + int teamID = 0; + foreach (LobbyMember member in lobby.Members) + { + if (bIsFFA) + { + member.UpdateTeam(-1); + } + else + { + member.UpdateTeam(teamID); + + teamID++; + + if (teamID >= numTeams) + { + teamID = 0; + } + } + } + + await TriggerFullMeshConnectivityChecks(lobby); + } + } + + } + + } + } + } + + // TODO_MATCHMAKING: Delete buckets if participants becomes 0 + } + // Using ConcurrentBag instead of ConcurrentList for lock-free bucket management + private static ConcurrentDictionary> m_dictMatchmakingBuckets = new(); + + public static void RemoveInvalidPlaylistsAtStartup() + { + foreach (var (id, playlist) in g_Playlists.ToArray()) + { + string? error = null; + if (playlist.MinPlayers < 2 || playlist.DesiredPlayers > Lobby.MaxLobbySize || playlist.MinPlayers > playlist.DesiredPlayers) + error = "player bounds exceed supported lobby capacity"; + else if (playlist.MinSelectedMaps < 1 || playlist.MinSelectedMaps > playlist.Maps.Count) + error = "minimum map selection is outside the playlist map count"; + else if (playlist.Maps.Any(map => map.MaxPlayers < playlist.MinPlayers || map.MaxPlayers > playlist.DesiredPlayers)) + error = "a map capacity is outside playlist player bounds"; + else if (Enumerable.Range(playlist.MinPlayers, playlist.DesiredPlayers - playlist.MinPlayers + 1) + .Any(size => !playlist.Maps.Any(map => map.MaxPlayers == size))) + error = "an allowed formation size has no exact-size map"; + if (error == null) continue; + s_log.LogError("Disabling matchmaking playlist {PlaylistId} ({PlaylistName}): {Reason}", id, playlist.Name, error); + g_Playlists.Remove(id); + } + } + + // TODO_QUICKMATCH: Read from db or file + private static Dictionary g_Playlists = new() + { + { 0, new Playlist(0, "1v1 (Random Armies)", 2, 2, 8, false, -1, false, 0, new List() + { + new PlaylistMap("[RANK] AKAs Magic ZH v1", "[RANK] AKAs Magic ZH v1", true, 2), + new PlaylistMap("[RANK] Arctic Arena ZH v1", "[RANK] Arctic Arena ZH v1", true, 2), +// new PlaylistMap("[RANK] Arctic Lagoon ZH v2 (2)", "[RANK] Arctic Lagoon ZH v2", true, 2), +// new PlaylistMap("[RANK] Barren Badlands Balanced ZH v2 (2)", "[RANK] Barren Badlands Balanced ZH v2", true, 2), +// new PlaylistMap("[RANK] Black Hell ZH v1", "[RANK] Black Hell ZH v1", true, 2), +// new PlaylistMap("[RANK] Blossoming Valley ZH v1 (2)", "[RANK] Blossoming Valley ZH v1", true, 2), +// new PlaylistMap("[RANK] Blue Hole ZH v1", "[RANK] Blue Hole ZH v1", true, 2), +// new PlaylistMap("[RANK] Canyon of the Dead ZH v2 (2)", "[RANK] Canyon of the Dead ZH v2", true, 2), +// new PlaylistMap("[RANK] Dammed Scorpion ZH v1", "[RANK] Dammed Scorpion ZH v1", true, 2), +// new PlaylistMap("[RANK] Desolated District ZH v1 (2)", "[RANK] Desolated District ZH v1", true, 2), + new PlaylistMap("[RANK] Drallim Desert ZH v2", "[RANK] Drallim Desert ZH v2", true, 2), +// new PlaylistMap("[RANK] Egyptian Oasis ZH v1 (2)", "[RANK] Egyptian Oasis ZH v1", true, 2), +// new PlaylistMap("[RANK] Farmlands of the Fallen ZH v1", "[RANK] Farmlands of the Fallen ZH v1", true, 2), +// new PlaylistMap("[RANK] Hanamura Temple ZH v1 (2)", "[RANK] Hanamura Temple ZH v1", true, 2), +// new PlaylistMap("[RANK] Imminent Victory ZH v2 (2)", "[RANK] Imminent Victory ZH v2", true, 2), +// new PlaylistMap("[RANK] Liquid Gold ZH v2 (2)", "[RANK] Liquid Gold ZH v2", true, 2), + new PlaylistMap("[RANK] Mountain Mayhem v2 (2)", "[RANK] Mountain Mayhem v2", true, 2), + new PlaylistMap("[RANK] Natural Threats ZH v4 (2)", "[RANK] Natural Threats ZH v4", true, 2), +// new PlaylistMap("[RANK] Sakura Forest II ZH v1", "[RANK] Sakura Forest II ZH v1", true, 2), +// new PlaylistMap("[RANK] Sand Scorpion (2)", "[RANK] Sand Scorpion", true, 2), +// new PlaylistMap("[RANK] Snowy Drought ZH v5 (2)", "[RANK] Snowy Drought ZH v5", true, 2), +// new PlaylistMap("[RANK] Sovereignty ZH v1", "[RANK] Sovereignty ZH v1", true, 2), + new PlaylistMap("[RANK] TD NoBugsCars ZH v1 (2)", "[RANK] TD NoBugsCars ZH v1", true, 2), + new PlaylistMap("[RANK] Vendetta ZH v1 (2)", "[RANK] Vendetta ZH v1", true, 2), +// new PlaylistMap("[RANK] ZH Carrier is Over v2 (2)", "[RANK] ZH Carrier is Over v2", true, 2), + new PlaylistMap("Arabia v2 (2)", "Arabia v2", true, 2), +// new PlaylistMap("Battle Plan ZH v3", "Battle Plan ZH v3", true, 2), + new PlaylistMap("Canyon Frost ZH v1", "Canyon Frost ZH v1", true, 2), + new PlaylistMap("Forest of Camelot ZH v4 (2)", "Forest of Camelot ZH v4", true, 2), + new PlaylistMap("Koujou Okawa v3", "Koujou Okawa v3", true, 2), + new PlaylistMap("Mirmulnir v4 (2)", "Mirmulnir v4", true, 2), + new PlaylistMap("Oxygen 1", "Oxygen 1", true, 2), +// new PlaylistMap("SandForest Domination v5 (2)", "SandForest Domination v5", true, 2), +// new PlaylistMap("Seasonal Conflict Summer V2 (2)", "Seasonal Conflict Summer V2", true, 2), +// new PlaylistMap("Shivas Paradise v4", "Shivas Paradise v4", true, 2), + new PlaylistMap("Terraform v2 (2)", "Terraform v2", true, 2), +// new PlaylistMap("Thermopylae v5", "Thermopylae v5", true, 2), +// new PlaylistMap("Tiny Tactics ZH v2", "Tiny Tactics ZH v2", true, 2), + new PlaylistMap("Toxic Lake v5 (2)", "Toxic Lake v5", true, 2), + new PlaylistMap("Tremble v2 (2)", "Tremble v2", true, 2), +// new PlaylistMap("yota nation arena v3", "yota nation arena v3", true, 2), + } + ) }, + + /* PRE-WS list: + * { 0, new Playlist(0, "1v1 (Random Armies)", 2, 2, 1, false, -1, false, 0, new List() + { + new PlaylistMap("[RANK] Snowy Drought ZH v5 (2)", "[RANK] Snowy Drought ZH v5", true, 2), + new PlaylistMap("[RANK] Natural Threats ZH v4 (2)", "[RANK] Natural Threats ZH v4", true, 2), + new PlaylistMap("[RANK] Arctic Lagoon ZH v2 (2)", "[RANK] Arctic Lagoon ZH v2", true, 2), + new PlaylistMap("[RANK] ZH Carrier is Over v2 (2)", "[RANK] ZH Carrier is Over v2", true, 2), + new PlaylistMap("[RANK] Vendetta ZH v1 (2)", "[RANK] Vendetta ZH v1", true, 2), + new PlaylistMap("[RANK] TD NoBugsCars ZH v1 (2)", "[RANK] TD NoBugsCars ZH v1", true, 2), + new PlaylistMap("[RANK] Sand Scorpion (2)", "[RANK] Sand Scorpion", true, 2), + new PlaylistMap("[RANK] Mountain Mayhem v2 (2)", "[RANK] Mountain Mayhem v2", true, 2), + new PlaylistMap("[RANK] Liquid Gold ZH v2 (2)", "[RANK] Liquid Gold ZH v2", true, 2), + new PlaylistMap("[RANK] Imminent Victory ZH v2 (2)", "[RANK] Imminent Victory ZH v2", true, 2), + new PlaylistMap("[RANK] Egyptian Oasis ZH v1 (2)", "[RANK] Egyptian Oasis ZH v1", true, 2), + new PlaylistMap("[RANK] Desolated District ZH v1 (2)", "[RANK] Desolated District ZH v1", true, 2), + new PlaylistMap("[RANK] Canyon of the Dead ZH v2 (2)", "[RANK] Canyon of the Dead ZH v2", true, 2), + new PlaylistMap("[RANK] Blossoming Valley ZH v1 (2)", "[RANK] Blossoming Valley ZH v1", true, 2), + //new PlaylistMap("[RANK] Battle Plan ZH v1 (2)", "[RANK] Battle Plan ZH v1", true, 2), // NOTE: Disabled. OldA say it is bugged according to map creator. + new PlaylistMap("[RANK] Barren Badlands Balanced ZH v2 (2)", "[RANK] Barren Badlands Balanced ZH v2", true, 2), + + // new maps added in 12_05_25 update + new PlaylistMap("Battle Plan ZH v3", "Battle Plan ZH v3", true, 2), + new PlaylistMap("Canyon Frost ZH v1", "Canyon Frost ZH v1", true, 2), + new PlaylistMap("Koujou Okawa v3", "Koujou Okawa v3", true, 2), + new PlaylistMap("Oxygen 1", "Oxygen 1", true, 2), + new PlaylistMap("Shivas Paradise v4", "Shivas Paradise v4", true, 2), + new PlaylistMap("Thermopylae v5", "Thermopylae v5", true, 2), + new PlaylistMap("Tiny Tactics ZH v2", "Tiny Tactics ZH v2", true, 2), + new PlaylistMap("yota nation arena v3", "yota nation arena v3", true, 2), + new PlaylistMap("[RANK] AKAs Magic ZH v1", "[RANK] AKAs Magic ZH v1", true, 2), + new PlaylistMap("[RANK] Arctic Arena ZH v1", "[RANK] Arctic Arena ZH v1", true, 2), + new PlaylistMap("[RANK] Black Hell ZH v1", "[RANK] Black Hell ZH v1", true, 2), + new PlaylistMap("[RANK] Blue Hole ZH v1", "[RANK] Blue Hole ZH v1", true, 2), + new PlaylistMap("[RANK] Dammed Scorpion ZH v1", "[RANK] Dammed Scorpion ZH v1", true, 2), + new PlaylistMap("[RANK] Drallim Desert ZH v2", "[RANK] Drallim Desert ZH v2", true, 2), + new PlaylistMap("[RANK] Farmlands of the Fallen ZH v1", "[RANK] Farmlands of the Fallen ZH v1", true, 2), + new PlaylistMap("[RANK] Sakura Forest II ZH v1", "[RANK] Sakura Forest II ZH v1", true, 2), + new PlaylistMap("[RANK] Sovereignty ZH v1", "[RANK] Sovereignty ZH v1", true, 2), + + // new maps added in 4_28_26 EAC update (for WS / from GR) + new PlaylistMap("Arabia v2 (2)", "Arabia v2", true, 2), + new PlaylistMap("SandForest Domination v5 (2)", "SandForest Domination v5", true, 2), + new PlaylistMap("Toxic Lake v4 (2)", "Toxic Lake v4", true, 2), + new PlaylistMap("Tremble v2 (2)", "Tremble v2", true, 2), + new PlaylistMap("[RANK] Hanamura Temple ZH v1 (2)", "[RANK] Hanamura Temple ZH v1", true, 2), + } + ) }, + */ + + { 1, new Playlist(1, "6-8P FFA (Random Armies)", 6, 8, 1, false, -1, false, 30000, new List() + { + new PlaylistMap("Beijing Uprise v4 (6)", "Beijing Uprise v4", true, 6), + new PlaylistMap("Defcon6 (6)", "Defcon6", false, 6), + new PlaylistMap("Taiga Terror (6)", "Taiga Terror", true, 6), + new PlaylistMap("Swamp Assault v3 [WBC2021] (7)", "Swamp Assault v3 [WBC2021]", true, 7), + new PlaylistMap("[RANK] Muddy Madness ZH v1 (8)", "[RANK] Muddy Madness ZH v1", true, 8), + new PlaylistMap("[RANK] Wastelands Dust ZH v1 (8)", "[RANK] Wastelands Dust ZH v1", true, 8) + } + ) } + }; + + public static Dictionary GetPlaylists() { return g_Playlists; } + + public static int GetTotalQueuedPlayersInPlaylist(UInt16 playlistID) + { + int totalPlayers = 0; + + if (m_dictMatchmakingBuckets.ContainsKey(playlistID)) + { + foreach (MatchmakingBucket bucket in m_dictMatchmakingBuckets[playlistID]) + { + totalPlayers += bucket.CurrentMemberCount(); + } + } + + return totalPlayers; + } + + public static async Task Tick() + { + // TODO_QUICKMATCH: Move to init func, maybe dont use static for matchmakingmanager + if (m_dictMatchmakingBuckets.Count == 0) + { + foreach (var kvPair in g_Playlists) + { + m_dictMatchmakingBuckets.TryAdd(kvPair.Key, new ConcurrentBag()); + } + } + + // tick mm buckets + // TODO_QUICKMATCH: This is slow + List lstBucketsMergedNeedingDeleted = new(); + foreach (var kvPair in m_dictMatchmakingBuckets) + { + foreach (MatchmakingBucket mmBucket in kvPair.Value) + { + // if we've already been merged and are awaiting delayed deletion, dont process it anymore + if (!lstBucketsMergedNeedingDeleted.Contains(mmBucket) && !mmBucket.IsMergedAway()) + { + await mmBucket.Tick(); + + // try to merge with any other bucket within this playlist + foreach (MatchmakingBucket mmBucketMergeCandidate in kvPair.Value) + { + if (mmBucketMergeCandidate != mmBucket) + { + // if either bucket has already been merged and is awaiting delayed deletion, dont process it anymore + if (lstBucketsMergedNeedingDeleted.Contains(mmBucket) || mmBucket.IsMergedAway()) + { + break; + } + + if (!lstBucketsMergedNeedingDeleted.Contains(mmBucketMergeCandidate) && !mmBucketMergeCandidate.IsMergedAway()) + { + if (mmBucket.CanMergeWithOtherBucket(mmBucketMergeCandidate)) + { + if (await mmBucket.MergeWithOtherBucketAsync(mmBucketMergeCandidate)) + lstBucketsMergedNeedingDeleted.Add(mmBucketMergeCandidate); + } + } + } + } + } + } + } + + // queue for deletion + foreach (MatchmakingBucket bucket in lstBucketsMergedNeedingDeleted) + { + m_bucketsPendingDeletion.Enqueue(bucket); + } + + // Drain the queue rather than enumerating and clearing a shared list. A concurrent cancellation can + // enqueue a bucket while cleanup is running, and clearing the list would otherwise lose that request. + while (m_bucketsPendingDeletion.TryDequeue(out MatchmakingBucket? bucket)) + { + if (m_dictMatchmakingBuckets.TryGetValue(bucket.PlaylistID, out var bucketBag)) + { + // ConcurrentBag doesn't support Remove, so we filter and rebuild + var remainingBuckets = bucketBag.Where(b => b != bucket).ToList(); + m_dictMatchmakingBuckets[bucket.PlaylistID] = new ConcurrentBag(remainingBuckets); + } + } + + List> lstDestroy = new(); + foreach (WeakReference wrSession in lstSessions) + { + if (!wrSession.TryGetTarget(out UserSession? thisSession) || thisSession == null) + { + lstDestroy.Add(wrSession); + } + else + { + await thisSession.MatchmakingStateLock.WaitAsync(); + try + { + // A cancellation can remove the weak reference while this tick is iterating a snapshot. + // Re-check registration while holding the per-session gate before assigning any bucket. + if (thisSession.IsRetired || !thisSession.IsRegisteredForMatchmaking || !IsPendingSession(thisSession)) + { + lstDestroy.Add(wrSession); + continue; + } + + SharedUserData? thisSessionUserData = GenOnlineService.WebSocketManager.GetSharedDataForUser(thisSession.m_UserID); + + if (thisSessionUserData == null) + { + RecordQueueExit(thisSession, "session_unavailable"); + thisSession.IsRegisteredForMatchmaking = false; + lstDestroy.Add(wrSession); + } + else + { + PlayerStats? thisSessionStats = thisSessionUserData.GameStats; + if (thisSessionStats == null) + { + thisSession.IsRegisteredForMatchmaking = false; + RecordQueueExit(thisSession, "stats_unavailable"); + lstDestroy.Add(wrSession); + await SendMatchmakingMessage(thisSession, "Matchmaking could not start because your player statistics are unavailable. Please try again."); + continue; + } + + if (g_Playlists.TryGetValue(thisSession.MatchmakingPlaylistID, out Playlist? playlist)) + { + + // TODO_MATCHAMAKING: Better way of tracking this, we need to know who is already in a bucket + // Was the user in a bucket? if so theres nothing to do in terms of bucket management + bool bUseInBucket = false; + MatchmakingBucket? mmBucketUserIsIn = null; + foreach (var kvBucketPair in m_dictMatchmakingBuckets) + { + foreach (MatchmakingBucket mmBucket in kvBucketPair.Value) + { + if (mmBucket.HasPlayer(thisSession)) + { + bUseInBucket = true; + mmBucketUserIsIn = mmBucket; + break; + } + } + + if (bUseInBucket) + { + break; + } + } + + if (bUseInBucket) + { + // already sorted into a bucket - stop tracking this session here, otherwise a stale entry + // can sort the same player into a second bucket (and therefore a second lobby) later on + lstDestroy.Add(wrSession); + } + else + { + // is there a suitable bucket for us + // TODO_MATCHMAKING: Optimize lookup + if (m_dictMatchmakingBuckets.ContainsKey(thisSession.MatchmakingPlaylistID)) + { + MatchmakingBucket? bucketInUse = null; + foreach (MatchmakingBucket mmBucket in m_dictMatchmakingBuckets[thisSession.MatchmakingPlaylistID]) + { + if (mmBucket.IsMergedAway()) + { + continue; + } + + // must be within initial elo threshold for a join, otherwise we'll make a bucket and try to merge buckets using the elo iteration expansion algorithm + int matchmakingElo = MatchmakingManager.GetMatchmakingElo(thisSessionUserData.GameStats); + int eloExpansionToUse = (mmBucket.GetAvgElo() >= EloConfig.HighEloThreshold || matchmakingElo >= EloConfig.HighEloThreshold) ? EloConfig.EloExpansionValue_HighELO : EloConfig.EloExpansionValue_Standard; + if (mmBucket.IsAvgEloWithinThreshold(matchmakingElo, eloExpansionToUse)) + { + // TODO_MATCHMAKING: Squads + if (mmBucket.HasSpaceForUsers(1, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID)) + { + // do the maps overlap? if so we can join + if (mmBucket.DoMapSelectionsIntersect(thisSession.MatchmakingMapIndicies)) + { + bool bJoined = await mmBucket.JoinAsync(thisSession); + + if (bJoined) + { + // stop looking - joining more than one bucket would matchmake this player into multiple lobbies + bucketInUse = mmBucket; + break; + } + } + } + } + } + + // didnt find a bucket? make one + if (bucketInUse == null) + { + MatchmakingBucket newBucket = new MatchmakingBucket(playlist.PlaylistID, thisSession, playlist.MinPlayers, playlist.DesiredPlayers, thisSession.MatchmakingMapIndicies, thisSession.ExeCRC, thisSession.IniCRC, thisSession.AnticheatID); + m_dictMatchmakingBuckets[thisSession.MatchmakingPlaylistID].Add(newBucket); + bucketInUse = newBucket; + } + + // send status to use + await SendMatchmakingMessage(thisSession, String.Format("You are now matchmaking in playlist \"{0}\". There are currently {1} player(s) searching for a match in this playlist", playlist.Name, GetTotalQueuedPlayersInPlaylist(playlist.PlaylistID))); + await SendMatchmakingMessage(thisSession, String.Format("Status: {0}/{1} players. ({2} required to start)", bucketInUse.CurrentMemberCount(), bucketInUse.DesiredPlayers, bucketInUse.MinPlayers)); + + // now remove us from lstSessions, this list is essentially people who need sorted into a bucket + lstDestroy.Add(wrSession); + } + } + } + else + { + // invalid playlist somehow + RecordQueueExit(thisSession, "playlist_unavailable"); + thisSession.IsRegisteredForMatchmaking = false; + lstDestroy.Add(wrSession); + } + } + } + finally + { + thisSession.MatchmakingStateLock.Release(); + } + } + } + + // now remove + foreach (WeakReference wrSession in lstDestroy) + { + if (wrSession != null) + { + lstSessions.Remove(wrSession); + } + } + + foreach (UInt16 playlistID in g_Playlists.Keys) + { + AppMetrics.RecordMatchmakingQueue(playlistID.ToString(CultureInfo.InvariantCulture), GetTotalQueuedPlayersInPlaylist(playlistID)); + } + } + + // TODO_MATCHMAKING: Deregister player if they disconnect or leave quickmatch + private static ConcurrentList> lstSessions = new(); + + private static bool IsPendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions) + { + if (wrSession.TryGetTarget(out UserSession? pendingSession) && ReferenceEquals(pendingSession, session)) + { + return true; + } + } + + return false; + } + + private static void RemovePendingSession(UserSession session) + { + foreach (WeakReference wrSession in lstSessions.ToList()) + { + if (!wrSession.TryGetTarget(out UserSession? pendingSession) || ReferenceEquals(pendingSession, session)) + { + lstSessions.Remove(wrSession); + } + } + } + + // records the wait of a queued player once; players that were not queued are ignored + internal static void RecordQueueExit(UserSession session, string outcome) + { + long startedAt = Interlocked.Exchange(ref session.MatchmakingStartedTimestamp, 0); + if (startedAt == 0) return; + AppMetrics.RecordMatchmakingQueueExit( + session.MatchmakingPlaylistID.ToString(CultureInfo.InvariantCulture), + outcome, + Stopwatch.GetElapsedTime(startedAt)); + } + + internal static async Task TryRequeueRegisteredPlayer(UserSession session, byte[] requeueActionJSON) + { + await session.MatchmakingStateLock.WaitAsync(); + try + { + if (session.IsRetired || !session.IsRegisteredForMatchmaking) + { + return false; + } + + if (!IsPendingSession(session)) + { + lstSessions.Add(new WeakReference(session)); + } + + session.QueueWebsocketSend(requeueActionJSON); + return true; + } + finally + { + session.MatchmakingStateLock.Release(); + } + } + + private static ConcurrentQueue m_bucketsPendingDeletion = new(); + public static void DestroyBucket(MatchmakingBucket bucket) + { + bucket.MarkPendingDeletion(); + m_bucketsPendingDeletion.Enqueue(bucket); + } + + public static async Task RegisterPlayer(UserSession plr, UInt16 playlistID, List mapIndices, UInt32 exe_crc, UInt32 ini_crc, EKnownAnticheatID anticheatID) + { + using Activity? activity = AppMetrics.StartActivity("matchmaking.register"); + activity?.SetTag("user.id", plr.m_UserID); + activity?.SetTag("matchmaking.playlist_id", playlistID); + // validate the request - a bad playlist or out of range map index from a client must never reach a bucket + if (!g_Playlists.TryGetValue(playlistID, out Playlist? playlist)) + { + activity?.SetTag("matchmaking.outcome", "unavailable_playlist"); + s_log.LogWarning("User {UserId} requested unavailable matchmaking playlist {PlaylistId}", plr.m_UserID, playlistID); + await SendMatchmakingMessage(plr, "That playlist is not available. Matchmaking was not started."); + return; + } + + List validatedMapIndices = mapIndices + .Where(mapIndex => mapIndex >= 0 && mapIndex < playlist.Maps.Count) + .Distinct() + .ToList(); + + int minSelectedMaps = Math.Max(1, playlist.MinSelectedMaps); + if (validatedMapIndices.Count < minSelectedMaps) + { + activity?.SetTag("matchmaking.outcome", "insufficient_maps"); + s_log.LogWarning("User {UserId} selected {SelectedMapCount} valid maps for playlist {PlaylistId}; minimum is {MinimumMapCount}", plr.m_UserID, validatedMapIndices.Count, playlistID, minSelectedMaps); + await SendMatchmakingMessage(plr, String.Format("You must select at least {0} valid map(s) to matchmake in this playlist.", minSelectedMaps)); + return; + } + + MatchmakingDetachment detached; + bool wasQueued; + await plr.MatchmakingStateLock.WaitAsync(); + try + { + if (plr.IsRetired) return; + wasQueued = plr.MatchmakingStartedTimestamp != 0; + plr.IsRegisteredForMatchmaking = false; + RemovePendingSession(plr); + detached = RemovePlayerFromAllBuckets(plr); + + if (!detached.CancellationRejected) + { + if (wasQueued) + { + RecordQueueExit(plr, "restarted"); + } + plr.MatchmakingPlaylistID = playlistID; + plr.MatchmakingMapIndicies = new ConcurrentList(validatedMapIndices); + plr.ExeCRC = exe_crc; + plr.IniCRC = ini_crc; + plr.AnticheatID = anticheatID; + plr.IsRegisteredForMatchmaking = true; + plr.MatchmakingStartedTimestamp = Stopwatch.GetTimestamp(); + lstSessions.Add(new WeakReference(plr)); + } + } + finally + { + plr.MatchmakingStateLock.Release(); + } + + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + foreach (var entry in detached.Members) + await lobbyManager.RemoveMatchmakingMemberAsync(entry.Lobby, entry.Member); + + if (detached.CancellationRejected) + { + activity?.SetTag("matchmaking.outcome", "start_in_progress"); + s_log.LogWarning("User {UserId} could not restart matchmaking because the game is starting", plr.m_UserID); + await SendMatchmakingMessage(plr, "Matchmaking cannot be restarted because your game is already starting."); + return; + } + + activity?.SetTag("matchmaking.outcome", "accepted"); + s_log.LogInformation("User {UserId} started matchmaking in playlist {PlaylistId} with {SelectedMapCount} maps", plr.m_UserID, playlistID, validatedMapIndices.Count); + await SendMatchmakingMessage(plr, "Started matchmaking... Searching for players..."); + } + + internal sealed record MatchmakingDetachment(bool CancellationRejected, List<(Lobby Lobby, LobbyMember Member)> Members); + + private static MatchmakingDetachment RemovePlayerFromAllBuckets(UserSession player) + { + var lobbies = ServiceLocator.Services.GetRequiredService(); + bool rejected = false; + List<(Lobby, LobbyMember)> members = new(); + foreach (var buckets in m_dictMatchmakingBuckets.Values) + { + foreach (var bucket in buckets) + { + bool removed = bucket.RemovePlayer(player, out bool cancellationRejected); + rejected |= cancellationRejected; + if (!removed) continue; + var lobby = lobbies.GetLobby(bucket.GetLobbyID()); + var member = lobby?.GetMemberFromUserID(player.m_UserID); + if (member != null && member.GetSession().TryGetTarget(out var memberSession) && ReferenceEquals(memberSession, player)) + members.Add((lobby!, member)); + if (bucket.CurrentMemberCount() == 0) DestroyBucket(bucket); + } + } + return new(rejected, members); + } + + // Only matchmaking state is changed here. Membership effects run after releasing + // this lock, so lifecycle operations can acquire user gate -> matchmaking lock. + internal static async Task DetachSessionAsync(UserSession player) + { + await player.MatchmakingStateLock.WaitAsync(); + try + { + player.IsRegisteredForMatchmaking = false; + RemovePendingSession(player); + var detached = RemovePlayerFromAllBuckets(player); + if (!detached.CancellationRejected) RecordQueueExit(player, "withdrawn"); + return detached; + } + finally + { + player.MatchmakingStateLock.Release(); + } + } + + public static Task DeregisterPlayerAsync(UserSession player) => + ServiceLocator.Services.GetRequiredService().CancelMatchmakingAsync(player); } diff --git a/GenOnlineService/Moderation.cs b/GenOnlineService/Moderation.cs index 7d0b9b7..7c034d9 100644 --- a/GenOnlineService/Moderation.cs +++ b/GenOnlineService/Moderation.cs @@ -34,18 +34,21 @@ public sealed class ModerationResult public static class ModerationManager { + private static readonly ILogger s_log = AppLog.For(typeof(ModerationManager)); public const int MaximumReasonLength = 256; public static async Task KickUser(Int64 targetUserID, string reason) { if (reason.Length > MaximumReasonLength) { + AppMetrics.RecordSecurityAction(EModerationAction.Kick.ToMetricLabel(), "reason_too_long"); return new ModerationResult { Result = EModerationResult.ReasonTooLong }; } SharedUserData? target = WebSocketManager.GetSharedDataForUser(targetUserID); if (target == null) { + AppMetrics.RecordSecurityAction(EModerationAction.Kick.ToMetricLabel(), "target_offline"); return new ModerationResult { Result = EModerationResult.TargetNotOnline }; } @@ -59,6 +62,8 @@ public static async Task KickUser(Int64 targetUserID, string r public static async Task DisconnectUser(Int64 userID, EModerationAction action, string? reason) { + AppMetrics.RecordSecurityAction(action.ToMetricLabel(), "applied"); + s_log.LogInformation("Applying moderation action {Action} to user {UserId}", action, userID); WebSocketMessage_ModerationNotice notice = new WebSocketMessage_ModerationNotice { msg_id = (int)EWebSocketMessageID.MODERATION_NOTICE, diff --git a/GenOnlineService/Program.cs b/GenOnlineService/Program.cs index 7de732d..a688ac6 100644 --- a/GenOnlineService/Program.cs +++ b/GenOnlineService/Program.cs @@ -27,11 +27,19 @@ using Microsoft.AspNetCore.RateLimiting; using Microsoft.AspNetCore.WebSockets; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Options; using Microsoft.IdentityModel.Tokens; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using Sentry; +using Sentry.OpenTelemetry; using System.Collections.Concurrent; using System.Configuration; +using System.Diagnostics; using System.Drawing; using System.IdentityModel.Tokens.Jwt; using System.Net; @@ -275,6 +283,7 @@ public static class StatsTracker { public static async Task Update(int numLobbies, int numPlayers) { + AppMetrics.RecordServiceSnapshot(numLobbies, numPlayers); int hourOfDay = DateTime.Now.Hour; // store stats @@ -710,7 +719,7 @@ private static async Task InitializeDatabase(WebApplicationBuilder builder) // TODO_EFCORE: Consider use of ExecuteDeleteAsync and options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); // TODO_EFCORE: Move to AddPooledDbContextFactory instead and use private readonly IDbContextFactory _factory; - builder.Services.AddPooledDbContextFactory(options => + builder.Services.AddPooledDbContextFactory((serviceProvider, options) => { options.UseMySql( csb.ConnectionString, @@ -718,8 +727,15 @@ private static async Task InitializeDatabase(WebApplicationBuilder builder) options.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); + // command events carry SQL text; ignoring them here holds whatever the logging configuration says + options.ConfigureWarnings(warnings => warnings.Ignore( + RelationalEventId.CommandExecuting, + RelationalEventId.CommandExecuted, + RelationalEventId.CommandError, + RelationalEventId.CommandCanceled)); + options.AddInterceptors(serviceProvider.GetRequiredService()); + #if RELEASE - options.UseLoggerFactory(LoggerFactory.Create(builder => { })); // Empty logger options.EnableSensitiveDataLogging(false); // Ensure sensitive data is not logged options.EnableDetailedErrors(false); // Disable detailed error messages #endif @@ -1158,6 +1174,31 @@ public static string BuildWebSocketUrl(HttpRequest request, string environment, new PathString($"/env/{Uri.EscapeDataString(environment)}/contract/{Uri.EscapeDataString(contractVersion)}/ws")); } + private static async Task RunObservedBackgroundJobAsync( + string jobName, + ILogger logger, + Func operation) + { + bool succeeded = true; + using Activity? activity = AppMetrics.StartActivity( + $"background.{jobName}", + ActivityKind.Internal, + new KeyValuePair("job.name", jobName)); + + try + { + await operation(); + } + catch (Exception ex) + { + succeeded = false; + AppMetrics.RecordException(activity, ex); + logger.LogError(ex, "Background job {JobName} failed", jobName); + } + + AppMetrics.RecordBackgroundJob(jobName, succeeded); + } + public static async Task Main(string[] args) { #if !DEBUG @@ -1168,7 +1209,63 @@ public static async Task Main(string[] args) ThreadPool.SetMinThreads(200, 200); var builder = WebApplication.CreateBuilder(args); + builder.Services.AddSingleton(); + bool enableOpenTelemetry = builder.Configuration.GetValue("OpenTelemetry:Enabled"); + bool enableSentry = builder.Configuration.GetValue("Sentry:enabled"); + bool enableSentryOpenTelemetry = enableOpenTelemetry && enableSentry; + if (enableOpenTelemetry) + { + string? openTelemetryEndpoint = builder.Configuration["OpenTelemetry:OtlpEndpoint"]; + string openTelemetryServiceName = builder.Configuration["OpenTelemetry:ServiceName"] ?? AppMetrics.MeterName; + double traceSampleRatio = builder.Configuration.GetValue("OpenTelemetry:TraceSampleRatio") ?? 1.0; + if (traceSampleRatio < 0 || traceSampleRatio > 1) + { + throw new InvalidOperationException("OpenTelemetry:TraceSampleRatio must be between 0 and 1."); + } + + void ConfigureOtlpExporter(OtlpExporterOptions options) + { + if (!String.IsNullOrWhiteSpace(openTelemetryEndpoint)) + { + options.Endpoint = new Uri(openTelemetryEndpoint); + } + } + + builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService(openTelemetryServiceName)) + .WithMetrics(metrics => + { + metrics.AddMeter(AppMetrics.MeterName); + metrics.AddMeter(AppMetrics.MySqlMeterName); + metrics.AddAspNetCoreInstrumentation(); + metrics.AddHttpClientInstrumentation(); + metrics.AddRuntimeInstrumentation(); + metrics.AddOtlpExporter(ConfigureOtlpExporter); + }) + .WithTracing(tracing => + { + tracing.SetSampler(new ParentBasedSampler(new TraceIdRatioBasedSampler(traceSampleRatio))); + tracing.AddAspNetCoreInstrumentation(options => options.RecordException = true); + tracing.AddHttpClientInstrumentation(options => options.RecordException = true); + tracing.AddSource(AppMetrics.ActivitySourceName); + tracing.AddSource(AppMetrics.MySqlActivitySourceName); + // before Sentry and the exporter, so neither sees unredacted spans + tracing.AddProcessor(new TelemetryRedactionProcessor()); + if (enableSentryOpenTelemetry) + { + tracing.AddSentry(); + } + tracing.AddOtlpExporter(ConfigureOtlpExporter); + }) + .WithLogging(logging => logging.AddOtlpExporter(ConfigureOtlpExporter), options => + { + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + options.ParseStateValues = true; + }); + } RoomCatalog.Initialize(Path.Combine(builder.Environment.ContentRootPath, "data", "rooms.json")); + MatchmakingManager.RemoveInvalidPlaylistsAtStartup(); // Add services to the container. @@ -1208,26 +1305,35 @@ public static async Task Main(string[] args) if ((bool)sentry_enabled) { - // init sentry + double sentryTracesSampleRate = sentrySettings.GetValue("traces_sample_rate") ?? 0.1; + bool sentryEnableLogs = sentrySettings.GetValue("enable_logs") ?? false; + SentrySdk.Init(options => { - // A Sentry Data Source Name (DSN) is required. - // See https://docs.sentry.io/product/sentry-basics/dsn-explainer/ - // You can set it in the SENTRY_DSN environment variable, or you can set it in code here. options.Dsn = sentry_dsn; - // When debug is enabled, the Sentry client will emit detailed debugging information to the console. - // This might be helpful, or might interfere with the normal operation of your application. - // We enable it here for demonstration purposes when first trying Sentry. - // You shouldn't do this in your applications unless you're troubleshooting issues with Sentry. options.Debug = false; - // This option is recommended. It enables Sentry's "Release Health" feature. options.AutoSessionTracking = true; options.Environment = sentry_env; options.Release = "generalsonline-services@082826"; + options.EnableLogs = sentryEnableLogs; + // Sentry's own EF/SqlClient listener would put SQL into span descriptions + options.DisableDiagnosticSourceIntegration(); + if (enableSentryOpenTelemetry) + { + options.TracesSampleRate = sentryTracesSampleRate; + options.UseOpenTelemetry(disableSentryTracing: true); + } + }); + + builder.Logging.AddSentry(options => + { + options.InitializeSdk = false; + options.MinimumBreadcrumbLevel = LogLevel.Warning; + options.MinimumEventLevel = LogLevel.Error; }); } @@ -1244,6 +1350,7 @@ public static async Task Main(string[] args) builder.Services.AddHostedService(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); var rateLimitingSettings = Program.g_Config.GetSection("RateLimiting"); bool bUseBuiltinRateLimiter = rateLimitingSettings.GetValue("use_builtin_ratelimiter"); // use built in Kestrel/dotnet rate limiting if you do not have a reverse proxy or other rate limiter in front of service @@ -1252,6 +1359,11 @@ public static async Task Main(string[] args) { builder.Services.AddRateLimiter(options => { + options.OnRejected = (context, cancellationToken) => + { + AppMetrics.RecordSecurityAction("http_rate_limit", "rejected"); + return ValueTask.CompletedTask; + }; options.GlobalLimiter = PartitionedRateLimiter.Create(httpContext => { // Use authenticated user ID or fallback to IP address @@ -1603,9 +1715,7 @@ public static async Task Main(string[] args) } else { - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("*** WARNING: Core:enforce_https is disabled. Bearer tokens will be sent in clear text over any plain-HTTP listener. ***"); - Console.ForegroundColor = ConsoleColor.Gray; + app.Logger.LogWarning("Core:enforce_https is disabled. Bearer tokens will be sent in clear text over any plain-HTTP listener."); } app.UseCors(); @@ -1621,21 +1731,22 @@ public static async Task Main(string[] args) timerCleanup.Elapsed += async (sender, e) => { try - { - await WebSocketManager.CheckForTimeouts(); - - var lobbyManager = ServiceLocator.Services.GetRequiredService(); - - int numLobbies = lobbyManager.GetNumLobbies(); - await StatsTracker.Update(numLobbies, WebSocketManager.GetNumberOfUsersOnline()); - - await lobbyManager.Cleanup(); - - PendingLoginManager.CleanupExpiredLogins(); - } - catch (Exception ex) - { - Console.WriteLine($"[timerCleanup] Exception: {ex}"); + { + await RunObservedBackgroundJobAsync("service_cleanup", app.Logger, async () => + { + await WebSocketManager.CheckForTimeouts(); + + var lobbyManager = ServiceLocator.Services.GetRequiredService(); + int numLobbies = lobbyManager.GetNumLobbies(); + await StatsTracker.Update(numLobbies, WebSocketManager.GetNumberOfUsersOnline()); + await lobbyManager.Cleanup(); + + int expiredLoginCount = PendingLoginManager.CleanupExpiredLogins(); + if (expiredLoginCount > 0) + { + app.Logger.LogDebug("Removed {ExpiredLoginCount} expired pending logins", expiredLoginCount); + } + }); } finally { @@ -1658,7 +1769,7 @@ public static async Task Main(string[] args) } catch (Exception ex) { - Console.WriteLine($"[timerTick lobby] Exception: {ex}"); + app.Logger.LogError(ex, "Lobby tick timer failed"); } finally { @@ -1682,7 +1793,7 @@ public static async Task Main(string[] args) } catch (Exception ex) { - Console.WriteLine($"[cleanupTick lobby] Exception: {ex}"); + app.Logger.LogError(ex, "Lobby cleanup timer failed"); } finally { @@ -1704,7 +1815,7 @@ public static async Task Main(string[] args) } catch (Exception ex) { - Console.WriteLine($"[timerTick matchmaking] Exception: {ex}"); + app.Logger.LogError(ex, "Matchmaking tick timer failed"); } finally { @@ -1726,7 +1837,7 @@ public static async Task Main(string[] args) } catch (Exception ex) { - Console.WriteLine($"[timerTick rooms] Exception: {ex}"); + app.Logger.LogError(ex, "Network-room tick timer failed"); } finally { @@ -1744,16 +1855,13 @@ public static async Task Main(string[] args) { try { - using (var scope = app.Services.CreateScope()) + await RunObservedBackgroundJobAsync("daily_stats_save", app.Logger, async () => { + using var scope = app.Services.CreateScope(); var factory = scope.ServiceProvider.GetRequiredService>(); await using var db = await factory.CreateDbContextAsync(); await DailyStatsManager.SaveToDB(db); - } - } - catch (Exception ex) - { - Console.WriteLine($"[timerTick dailystats] Exception: {ex}"); + }); } finally { @@ -1771,16 +1879,13 @@ public static async Task Main(string[] args) { try { - using (var scope = app.Services.CreateScope()) + await RunObservedBackgroundJobAsync("token_revocation_reconcile", app.Logger, async () => { + using var scope = app.Services.CreateScope(); var factory = scope.ServiceProvider.GetRequiredService>(); await using var db = await factory.CreateDbContextAsync(); await TokenRevocationManager.ReconcileBans(db); - } - } - catch (Exception ex) - { - Console.WriteLine($"[timerTick tokenrevocation] Exception: {ex}"); + }); } finally { @@ -1792,8 +1897,7 @@ public static async Task Main(string[] args) AppDomain.CurrentDomain.ProcessExit += (_, _) => { - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("EXIT REQUESTED!"); + app.Logger.LogInformation("Exit requested"); }; // create a token diff --git a/GenOnlineService/SlowDatabaseCommandInterceptor.cs b/GenOnlineService/SlowDatabaseCommandInterceptor.cs new file mode 100644 index 0000000..f016c23 --- /dev/null +++ b/GenOnlineService/SlowDatabaseCommandInterceptor.cs @@ -0,0 +1,84 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +*/ + +using System.Data.Common; +using Microsoft.EntityFrameworkCore.Diagnostics; + +namespace GenOnlineService +{ + // Logs slow commands with operation, duration and outcome only, never SQL or parameters. + public sealed class SlowDatabaseCommandInterceptor : DbCommandInterceptor + { + private readonly ILogger _logger; + private readonly TimeSpan _threshold; + + public SlowDatabaseCommandInterceptor(IConfiguration configuration, ILogger logger) + { + _logger = logger; + _threshold = TimeSpan.FromMilliseconds(Math.Max(1, configuration.GetValue("OpenTelemetry:SlowDatabaseCommandMilliseconds") ?? 500)); + } + + public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return result; + } + + public override ValueTask ReaderExecutedAsync(DbCommand command, CommandExecutedEventData eventData, DbDataReader result, CancellationToken cancellationToken = default) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return ValueTask.FromResult(result); + } + + public override object? ScalarExecuted(DbCommand command, CommandExecutedEventData eventData, object? result) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return result; + } + + public override ValueTask ScalarExecutedAsync(DbCommand command, CommandExecutedEventData eventData, object? result, CancellationToken cancellationToken = default) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return ValueTask.FromResult(result); + } + + public override int NonQueryExecuted(DbCommand command, CommandExecutedEventData eventData, int result) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return result; + } + + public override ValueTask NonQueryExecutedAsync(DbCommand command, CommandExecutedEventData eventData, int result, CancellationToken cancellationToken = default) + { + Check(eventData.ExecuteMethod, "success", eventData.Duration); + return ValueTask.FromResult(result); + } + + public override void CommandFailed(DbCommand command, CommandErrorEventData eventData) + { + Check(eventData.ExecuteMethod, "error", eventData.Duration); + } + + public override Task CommandFailedAsync(DbCommand command, CommandErrorEventData eventData, CancellationToken cancellationToken = default) + { + Check(eventData.ExecuteMethod, "error", eventData.Duration); + return Task.CompletedTask; + } + + private void Check(DbCommandMethod method, string outcome, TimeSpan duration) + { + if (duration >= _threshold) + { + _logger.LogWarning("Slow database {DatabaseOperation} command took {ElapsedMilliseconds} ms with outcome {Outcome}", + method, duration.TotalMilliseconds, outcome); + } + } + } +} diff --git a/GenOnlineService/TelemetryRedactionProcessor.cs b/GenOnlineService/TelemetryRedactionProcessor.cs new file mode 100644 index 0000000..a3ab84f --- /dev/null +++ b/GenOnlineService/TelemetryRedactionProcessor.cs @@ -0,0 +1,56 @@ +/* +** GeneralsOnline Game Services - Backend Services for Command & Conquer Generals Online: Zero Hour +** Copyright (C) 2025 GeneralsOnline Development Team +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU Affero General Public License as +** published by the Free Software Foundation, either version 3 of the +** License, or (at your option) any later version. +*/ + +using System.Diagnostics; +using OpenTelemetry; + +namespace GenOnlineService +{ + // Removes SQL, connection details and outbound URL paths (the TURN key is in the path) from spans before export. + public sealed class TelemetryRedactionProcessor : BaseProcessor + { + private static readonly string[] s_databaseTags = + [ + "db.statement", + "db.query.text", + "db.connection_string", + "db.user" + ]; + + public override void OnEnd(Activity activity) + { + if (activity.Source.Name == AppMetrics.MySqlActivitySourceName) + { + foreach (string tag in s_databaseTags) + { + activity.SetTag(tag, null); + } + } + + if (activity.Kind == ActivityKind.Client) + { + RedactUrl(activity, "url.full"); + RedactUrl(activity, "http.url"); + activity.SetTag("url.path", null); + activity.SetTag("url.query", null); + } + } + + private static void RedactUrl(Activity activity, string tag) + { + if (activity.GetTagItem(tag) is not string url) + { + return; + } + + activity.SetTag(tag, Uri.TryCreate(url, UriKind.Absolute, out Uri? uri) ? uri.GetLeftPart(UriPartial.Authority) : null); + } + } +} diff --git a/GenOnlineService/TokenRevocation.cs b/GenOnlineService/TokenRevocation.cs index 7370808..e16a008 100644 --- a/GenOnlineService/TokenRevocation.cs +++ b/GenOnlineService/TokenRevocation.cs @@ -64,6 +64,8 @@ namespace Database { public static class UserTokens { + private static readonly ILogger s_log = AppLog.For(typeof(UserTokens)); + public static async Task> GetAll(AppDbContext db) { try @@ -72,8 +74,7 @@ public static async Task> GetAll(AppDbContext db) } catch (Exception ex) { - Console.WriteLine($"[ERROR] UserTokens.GetAll failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UserTokens.GetAll failed"); return new List(); } } @@ -96,8 +97,7 @@ ON DUPLICATE KEY UPDATE } catch (Exception ex) { - Console.WriteLine($"[ERROR] UserTokens.Upsert failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UserTokens.Upsert failed"); } } @@ -109,8 +109,7 @@ public static async Task> GetBannedUserIDs(AppDbContext db) } catch (Exception ex) { - Console.WriteLine($"[ERROR] UserTokens.GetBannedUserIDs failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "UserTokens.GetBannedUserIDs failed"); throw; } } @@ -124,6 +123,8 @@ namespace GenOnlineService // survives a restart. public static class TokenRevocationManager { + private static readonly ILogger s_log = AppLog.For(typeof(TokenRevocationManager)); + private sealed class CachedState { public int Generation; @@ -176,7 +177,7 @@ public static async Task Initialize(IDbContextFactory dbFactory, A s_bannedUsers = new HashSet(banned); } - Console.WriteLine($"[TokenRevocation] Loaded {s_state.Count} token state entries, {banned.Count} banned users."); + s_log.LogInformation("Loaded {StateEntryCount} token state entries, {BannedUserCount} banned users", s_state.Count, banned.Count); } public static int GetGeneration(Int64 userID, EUserSessionType sessionType) @@ -247,7 +248,7 @@ public static async Task OnTokensIssued(Int64 userID, EUserSessionType sessionTy // Invalidates every token previously issued to this user across all session types. public static async Task RevokeAllTokensForUser(Int64 userID, string reason) { - Console.WriteLine($"[TokenRevocation] Revoking all tokens for user {userID} ({reason})."); + s_log.LogInformation("Revoking all tokens for user {UserId} ({Reason})", userID, reason); foreach (EUserSessionType sessionType in s_allSessionTypes) { @@ -309,8 +310,7 @@ private static async Task Persist(Int64 userID, EUserSessionType sessionType, Ca } catch (Exception ex) { - Console.WriteLine($"[ERROR] TokenRevocation.Persist failed: {ex.Message}"); - SentrySdk.CaptureException(ex); + s_log.LogError(ex, "TokenRevocation.Persist failed"); } } diff --git a/GenOnlineService/appsettings.json b/GenOnlineService/appsettings.json index 3e791a3..f0d844f 100644 --- a/GenOnlineService/appsettings.json +++ b/GenOnlineService/appsettings.json @@ -16,9 +16,18 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore": "Warning", + "MySqlConnector": "Warning" } }, + "OpenTelemetry": { + "Enabled": false, + "OtlpEndpoint": "http://localhost:4317", + "ServiceName": "GenOnlineService", + "TraceSampleRatio": 1.0, + "SlowDatabaseCommandMilliseconds": 500 + }, "AllowedHosts": "*", "AllowedOrigins": [], "TrustedProxies": [], @@ -77,7 +86,9 @@ }, "Sentry": { "enabled": false, - "dsn": "" + "dsn": "", + "enable_logs": false, + "traces_sample_rate": 0.1 }, "Middleware": { "jwks_endpoint": null, diff --git a/README.md b/README.md index 0588860..3206b22 100644 --- a/README.md +++ b/README.md @@ -41,3 +41,7 @@ GeneralsOnline Game Services Code provides RESTful web services which act as a r - Import the SQL structure to your database (GenOnlineService\Database_Structure\structure.sql) - When upgrading an existing database, apply new `GenOnlineService\Database_Structure\upgrade_*.sql` files in date order before starting the updated service - Run GenOnlineService.exe + +# Observability + +The service can export OpenTelemetry logs, metrics, and traces over OTLP, with Sentry as an optional destination for errors, logs, and traces. See [docs/observability.md](docs/observability.md) for configuration, coverage, metric names, privacy controls, and an Alloy example. diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..5efa7f2 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,145 @@ +# Observability + +GenOnlineService sends logs, metrics, and traces with OpenTelemetry. + +Send OTLP to a collector such as Grafana Alloy. The collector can route metrics to Prometheus, logs to Loki, and traces to Tempo. Sentry is optional and receives errors, and optionally logs and traces. Metrics only go over OTLP. + +## Configuration + +Remote export is off by default: + +```json +{ + "OpenTelemetry": { + "Enabled": false, + "OtlpEndpoint": "http://localhost:4317", + "ServiceName": "GenOnlineService", + "TraceSampleRatio": 1.0, + "SlowDatabaseCommandMilliseconds": 500 + }, + "Sentry": { + "enabled": false, + "dsn": "", + "environment": "production", + "enable_logs": false, + "traces_sample_rate": 0.1 + } +} +``` + +Production values can use environment variables: + +```text +OpenTelemetry__Enabled=true +OpenTelemetry__OtlpEndpoint=http://alloy:4317 +OpenTelemetry__TraceSampleRatio=0.25 +Sentry__enabled=true +Sentry__dsn=https://... +Sentry__environment=production +``` + +`TraceSampleRatio` controls how many traces OpenTelemetry keeps. Sentry can sample those traces again with `traces_sample_rate`. Sentry keeps 10% of those traces by default (`0.1`). Raise it while validating the setup, then lower it if volume or cost is a concern. + +## Sentry + +- Errors: `ILogger` entries at Error or above become Sentry events, Warning and above become breadcrumbs. This works without OpenTelemetry. +- Logs: `enable_logs` sends `ILogger` entries to Sentry Logs. Off by default. +- Traces: only when `OpenTelemetry:Enabled` is also true, because Sentry receives the OpenTelemetry spans. `traces_sample_rate` applies to those. +- Metrics: not sent. Sentry 6.8 cannot read .NET meters, so metrics go to the OTLP collector only. + +Sentry's own EF Core/SqlClient listener is disabled so SQL never ends up in Sentry span descriptions. + +## What is collected + +- Logs from `ILogger`, with trace IDs when available. +- HTTP requests and outgoing HTTP calls. +- MySQL commands and connection-pool health. +- WebSocket message spans, connections, and failures. +- Authentication, matchmaking, external publication, background jobs, and security actions. +- .NET runtime health. + +Fast tick loops are not traced on every pass. That would create too much data. + +## Metrics + +Database, HTTP, and runtime metrics come from the built-in MySqlConnector, ASP.NET Core, HttpClient, and runtime meters. Application metrics start with `genonline.`: + +| Metric | Type | Labels | +| --- | --- | --- | +| `genonline.players.online` | gauge | | +| `genonline.lobbies.active` | gauge | | +| `genonline.authentication.attempts` | counter | `flow`, `source`, `outcome` | +| `genonline.websocket.connections.active` | gauge | `session.type` | +| `genonline.websocket.failures` | counter | `direction`, `reason` | +| `genonline.websocket.send_queue.depth` | gauge | | +| `genonline.matchmaking.queue.players` | gauge | `playlist` | +| `genonline.matchmaking.wait.duration` | histogram (s) | `playlist`, `outcome` | +| `genonline.matchmaking.matches` | counter | `playlist`, `outcome` | +| `genonline.background_job.failures` | counter | `job` | +| `genonline.background_job.last_success.time` | gauge (unix s) | `job` | +| `genonline.external_publication.failures` | counter | `outcome` | +| `genonline.external_publication.backlog.age` | gauge (s) | | +| `genonline.security.actions` | counter | `action`, `outcome` | + +`genonline.security.actions` covers HTTP and chat rate-limit rejections, moderation actions, and anti-cheat reviews and bans. + +Metric labels use small fixed sets. They do not contain player, lobby, or match IDs. + +## Database data + +Spans never carry SQL text, connection strings, or database users: a span processor removes them before Sentry or the exporter sees the span. + +Slow-command logs contain only the operation, duration, and result. They never contain SQL or parameters. + +EF Core's command events (executing, executed, error, canceled) are ignored in the context options, so EF never logs SQL text whatever the logging configuration says. The `MySqlConnector` log category defaults to Warning. + +## Outbound URLs + +The TURN key is part of the Cloudflare API path, and configured URLs may carry tokens. Outbound HTTP spans therefore keep only the scheme, host, and port of the URL. Incoming request spans keep their path, and the ASP.NET Core instrumentation redacts query string values. + +## Identifiers in logs and traces + +Metrics never contain player, lobby, or match IDs. Logs and traces do include user, lobby, and match IDs (pseudonymous) so incidents can be traced. Display names appear only in Debug logs. Treat the collector and Sentry as systems that hold player identifiers. + +Per-message WebSocket failures are logged at Error at most once per user per minute; the rest are Debug. Metrics and spans still record every failure. + +## Alloy example + +This receives all three OTLP signals and prints them for testing: + +```alloy +otelcol.receiver.otlp "genonline" { + grpc { + endpoint = "0.0.0.0:4317" + } + + output { + metrics = [otelcol.processor.batch.genonline.input] + logs = [otelcol.processor.batch.genonline.input] + traces = [otelcol.processor.batch.genonline.input] + } +} + +otelcol.processor.batch "genonline" { + output { + metrics = [otelcol.exporter.debug.genonline.input] + logs = [otelcol.exporter.debug.genonline.input] + traces = [otelcol.exporter.debug.genonline.input] + } +} + +otelcol.exporter.debug "genonline" {} +``` + +Replace the debug exporter with the real Prometheus, Loki, and Tempo outputs. + +## Useful alerts + +- HTTP errors and slow requests. +- Database errors, slow commands, and pool pressure. +- Background job failures, or a `last_success.time` that stops moving. +- External publication failures and a growing backlog age. +- Long matchmaking waits and setup failures. +- WebSocket failures and a growing send queue. +- Spikes in rate-limit rejections or security actions. +- New Sentry error groups.