diff --git a/Parallel.Cli/Commands/RestoreCommand.cs b/Parallel.Cli/Commands/RestoreCommand.cs index c2415b4..e481121 100644 --- a/Parallel.Cli/Commands/RestoreCommand.cs +++ b/Parallel.Cli/Commands/RestoreCommand.cs @@ -44,17 +44,6 @@ public RestoreCommand() : base("restore", "Pulls files a vault.") this.AddOption(_dryRunOpt); this.AddOption(_verboseOpt); this.SetHandler(HandlePullAsync, _sourceOpt, _configOpt, _beforeOpt, _destOpt, _archiveOpt, _forceOpt, _verboseOpt, _dryRunOpt); - - this.AddCommand(addCmd); - addCmd.AddArgument(_sourceArg); - addCmd.AddOption(_configOpt); - addCmd.AddOption(_destOpt); - addCmd.SetHandler(HandleAddAsync, _sourceArg, _configOpt, _destOpt); - - this.AddCommand(removeCmd); - removeCmd.AddArgument(_sourceArg); - removeCmd.AddOption(_configOpt); - removeCmd.SetHandler(HandleRemoveAsync, _sourceArg, _configOpt); } private async Task HandlePullAsync(string? path, string? config, DateTime before, string? destination, bool archive, bool force, bool verbose, bool dryRun) @@ -95,10 +84,10 @@ private async Task PullSystemAsync(LocalVaultConfig vault, DateTime timestamp, s return; } - foreach (PullRecord record in syncManager.RemoteVault.PullDirectories.Where(r => r.Machine == Environment.MachineName)) + /*foreach (PullRecord record in syncManager.RemoteVault.PullDirectories.Where(r => r.Machine == Environment.MachineName)) { await PullInternalAsync(syncManager, record.Source, timestamp, (output ?? record.Destination), archive, force, verbose, dryRun); - } + }*/ } private async Task PullPathAsync(LocalVaultConfig vault, string path, DateTime timestamp, string? destination, bool archive, bool force, bool verbose, bool dryRun) @@ -148,85 +137,11 @@ private async Task PullInternalAsync(ISyncManager syncManager, string path, Date { CommandLine.WriteLine(syncManager.RemoteVault, $"Pulling {restoreFiles.Count:N0} files...", ConsoleColor.DarkGray); IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, restoreFiles.Count) : new LoggingProgressReporter(syncManager.RemoteVault); - int pulledFiles = await syncManager.PullFilesAsync(restoreFiles.ToArray(), progressReporter); + int pulledFiles = await syncManager.RestoreFilesAsync(restoreFiles.ToArray(), progressReporter); CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pulled {pulledFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); await syncManager.DisconnectAsync(); } } - - #region Add - - private async Task HandleAddAsync(string path, string? config, string? destination) - { - LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config); - if (!string.IsNullOrEmpty(config) && localVault != null) - { - await AddPathAsync(localVault, path, destination); - } - else - { - await Program.Settings.ForEachVaultAsync(vault => AddPathAsync(vault, path, destination)); - } - } - - private async Task AddPathAsync(LocalVaultConfig vault, string path, string? destination) - { - ISyncManager? syncManager = SyncManager.CreateNew(vault); - if (syncManager == null || !await syncManager.ConnectAsync()) - { - CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red); - return; - } - - PullRecord record = new(path, destination); - if (!syncManager.RemoteVault.PullDirectories.Add(record)) - { - CommandLine.WriteLine(vault, $"Unable to add path: '{path}'", ConsoleColor.Yellow); - return; - } - - CommandLine.WriteLine(vault, $"Successfully added '{path}'", ConsoleColor.Green); - await syncManager.DisconnectAsync(); - } - - #endregion - - #region Remove - - private async Task HandleRemoveAsync(string path, string? config) - { - LocalVaultConfig? localVault = string.IsNullOrEmpty(config) ? ParallelConfig.Load().Vaults.FirstOrDefault(v => v.Enabled) : ParallelConfig.GetVault(config); - if (!string.IsNullOrEmpty(config) && localVault != null) - { - await RemovePathAsync(localVault, path); - } - else - { - await Program.Settings.ForEachVaultAsync(vault => RemovePathAsync(vault, path)); - } - } - - private async Task RemovePathAsync(LocalVaultConfig vault, string path) - { - ISyncManager? syncManager = SyncManager.CreateNew(vault); - if (syncManager == null || !await syncManager.ConnectAsync()) - { - CommandLine.WriteLine(vault, "Failed to connect to vault!", ConsoleColor.Red); - return; - } - - IEnumerable records = syncManager.RemoteVault.PullDirectories.Where(r => r.Machine == Environment.MachineName && r.Source == path); - foreach (PullRecord record in records) - { - if (syncManager.RemoteVault.PullDirectories.Remove(record)) continue; - CommandLine.WriteLine(vault, $"Unable to remove path: '{path}'", ConsoleColor.Yellow); - } - - CommandLine.WriteLine(vault, $"Successfully removed '{path}'", ConsoleColor.Green); - await syncManager.DisconnectAsync(); - } - - #endregion } } \ No newline at end of file diff --git a/Parallel.Cli/Commands/ScrubCommand.cs b/Parallel.Cli/Commands/ScrubCommand.cs index 3ee375d..0bb8904 100644 --- a/Parallel.Cli/Commands/ScrubCommand.cs +++ b/Parallel.Cli/Commands/ScrubCommand.cs @@ -64,7 +64,7 @@ private async Task ScrubSystemAsync(LocalVaultConfig vault, bool verbose) return; } - foreach (string path in syncManager.RemoteVault.PushDirectories) + foreach (string path in syncManager.RemoteVault.SyncDirectories) { await ScrubInternalAsync(syncManager, path, verbose); } @@ -88,15 +88,24 @@ private async Task ScrubInternalAsync(ISyncManager syncManager, string path, boo IReadOnlyList files = await (syncManager.Database?.GetFilesAsync(path, DateTime.Now) ?? Task.FromResult>([])); if (files.Count == 0) { - CommandLine.WriteLine($"No prunable files were found!", ConsoleColor.Yellow); + CommandLine.WriteLine($"No files were found!", ConsoleColor.Yellow); return; } CommandLine.WriteLine(syncManager.RemoteVault, $"Scrubbing {files.Count:N0} files...", ConsoleColor.DarkGray); IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, files.Count) : new LoggingProgressReporter(syncManager.RemoteVault); - int scrubbedFiles = await syncManager.ScrubFilesAsync(files, progressReporter); + int validFiles = await syncManager.ScrubFilesAsync(files, progressReporter); + if (validFiles == files.Count) + { + CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully scrubbed {files.Count:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + } + else + { + int corrupted = files.Count - validFiles; + CommandLine.WriteLine(syncManager.RemoteVault, $"Removed {corrupted:N0} corrupted or missing files in {_sw.Elapsed}.", ConsoleColor.Yellow); + } - CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully scrubbed {scrubbedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + await syncManager.DisconnectAsync(); } } diff --git a/Parallel.Cli/Commands/SnapshotsCommand.cs b/Parallel.Cli/Commands/SnapshotsCommand.cs index 1abe2c6..e566311 100644 --- a/Parallel.Cli/Commands/SnapshotsCommand.cs +++ b/Parallel.Cli/Commands/SnapshotsCommand.cs @@ -109,7 +109,7 @@ private async Task CreateSnapshotAsync(LocalVaultConfig vault) } ConcurrentBag snapshots = new(); - await System.Threading.Tasks.Parallel.ForEachAsync(syncManager.RemoteVault.PushDirectories, ParallelConfig.Options, async (path, ct) => + await System.Threading.Tasks.Parallel.ForEachAsync(syncManager.RemoteVault.SyncDirectories, ParallelConfig.Options, async (path, ct) => { IReadOnlyList files = await (syncManager.Database?.GetLatestFilesAsync(path, DateTime.UtcNow, false) ?? Task.FromResult>([])); foreach (LocalFile file in files) snapshots.Add(new SnapshotRecord(file)); diff --git a/Parallel.Cli/Commands/SyncCommand.cs b/Parallel.Cli/Commands/SyncCommand.cs index 1084fcf..9ea4115 100644 --- a/Parallel.Cli/Commands/SyncCommand.cs +++ b/Parallel.Cli/Commands/SyncCommand.cs @@ -9,6 +9,7 @@ using Parallel.Core.IO.Syncing; using Parallel.Core.Models; using Parallel.Core.Settings; +using Parallel.Core.Utils; namespace Parallel.Cli.Commands { @@ -33,7 +34,8 @@ public SyncCommand() : base("sync", "Syncs system files with vaults.") this.AddOption(_configOpt); this.AddOption(_forceOpt); this.AddOption(_verboseOpt); - this.SetHandler(HandlePushAsync, _sourceOpt, _configOpt, _forceOpt, _verboseOpt); + this.AddOption(_dryRunOpt); + this.SetHandler(HandleSyncAsync, _sourceOpt, _configOpt, _forceOpt, _verboseOpt, _dryRunOpt); this.AddCommand(addCmd); addCmd.AddArgument(_sourceArg); @@ -46,9 +48,9 @@ public SyncCommand() : base("sync", "Syncs system files with vaults.") removeCmd.SetHandler(HandleRemoveAsync, _sourceArg, _configOpt); } - #region Push + #region Sync - private async Task HandlePushAsync(string? path, string? config, bool force, bool verbose) + private async Task HandleSyncAsync(string? path, string? config, bool force, bool verbose, bool dryRun) { _sw = Stopwatch.StartNew(); LocalVaultConfig? localVault = ParallelConfig.GetVault(config); @@ -56,27 +58,27 @@ private async Task HandlePushAsync(string? path, string? config, bool force, boo { if (!string.IsNullOrEmpty(path)) { - await PushPathAsync(localVault, path, force, verbose); + await SyncPathAsync(localVault, path, force, verbose, dryRun); } else { - await PushSystemAsync(localVault, force, verbose); + await SyncSystemAsync(localVault, force, verbose, dryRun); } } else { if (!string.IsNullOrEmpty(path)) { - await Program.Settings.ForEachVaultAsync(vault => PushPathAsync(vault, path, force, verbose)); + await Program.Settings.ForEachVaultAsync(vault => SyncPathAsync(vault, path, force, verbose, dryRun)); } else { - await Program.Settings.ForEachVaultAsync(vault => PushSystemAsync(vault, force, verbose)); + await Program.Settings.ForEachVaultAsync(vault => SyncSystemAsync(vault, force, verbose, dryRun)); } } } - private async Task PushSystemAsync(LocalVaultConfig vault, bool force, bool verbose) + private async Task SyncSystemAsync(LocalVaultConfig vault, bool force, bool verbose, bool dryRun) { ISyncManager? syncManager = SyncManager.CreateNew(vault); if (syncManager == null || !await syncManager.ConnectAsync()) @@ -85,15 +87,15 @@ private async Task PushSystemAsync(LocalVaultConfig vault, bool force, bool verb return; } - foreach (string path in syncManager.RemoteVault.PushDirectories) + foreach (string path in syncManager.RemoteVault.SyncDirectories) { - await PushInternalAsync(syncManager, path, force, verbose); + await SyncInternalAsync(syncManager, path, force, verbose, dryRun); } await syncManager.DisconnectAsync(); } - private async Task PushPathAsync(LocalVaultConfig vault, string path, bool force, bool verbose) + private async Task SyncPathAsync(LocalVaultConfig vault, string path, bool force, bool verbose, bool dryRun) { ISyncManager? syncManager = SyncManager.CreateNew(vault); if (syncManager == null || !await syncManager.ConnectAsync()) @@ -102,14 +104,14 @@ private async Task PushPathAsync(LocalVaultConfig vault, string path, bool force return; } - await PushInternalAsync(syncManager, path, force, verbose); + await SyncInternalAsync(syncManager, path, force, verbose, dryRun); await syncManager.DisconnectAsync(); } - private async Task PushInternalAsync(ISyncManager syncManager, string path, bool force, bool verbose) + private async Task SyncInternalAsync(ISyncManager syncManager, string path, bool force, bool verbose, bool dryRun) { // Normalize paths for safe comparison - string[] backupFolders = syncManager.RemoteVault.PushDirectories.ToArray(); + string[] backupFolders = syncManager.RemoteVault.SyncDirectories.ToArray(); string[] ignoredFolders = syncManager.RemoteVault.IgnoreDirectories.ToArray(); bool isFile = PathBuilder.IsFile(path); @@ -134,12 +136,22 @@ private async Task PushInternalAsync(ISyncManager syncManager, string path, bool CommandLine.WriteLine(syncManager.RemoteVault, $"The provided {(isFile ? "file" : "folder")} is already up to date.", ConsoleColor.Green); return; } + + if (dryRun) + { + string fileName = PathBuilder.TempFile; + await File.WriteAllLinesAsync(fileName, files.Select(f => f.Fullname).OrderBy(f => f)); + CommandLine.WriteLine($"This operation will sync {files.Length:N0} files ({Formatter.FromBytes(files.Sum(f => f.LocalSize))})", ConsoleColor.Green); + CommandLine.WriteLine($"A detailed list can be found here: {fileName}", ConsoleColor.DarkGray); + } + else + { + CommandLine.WriteLine(syncManager.RemoteVault, $"Syncing {files.Length:N0} files...", ConsoleColor.DarkGray); + IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, successFiles) : new LoggingProgressReporter(syncManager.RemoteVault); + int pushedFiles = await syncManager.SyncFilesAsync(files, progressReporter, force); - CommandLine.WriteLine(syncManager.RemoteVault, $"Pushing {files.Length:N0} files...", ConsoleColor.DarkGray); - IProgressReporter progressReporter = verbose ? new ProgressReporter(syncManager.RemoteVault, successFiles) : new LoggingProgressReporter(syncManager.RemoteVault); - int pushedFiles = await syncManager.PushFilesAsync(files, progressReporter, force); - - CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pushed {pushedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + CommandLine.WriteLine(syncManager.RemoteVault, $"Successfully pushed {pushedFiles:N0} files in {_sw.Elapsed}.", ConsoleColor.Green); + } } #endregion @@ -168,7 +180,7 @@ private async Task AddPathAsync(LocalVaultConfig vault, string path) return; } - if (!syncManager.RemoteVault.PushDirectories.Add(path)) + if (!syncManager.RemoteVault.SyncDirectories.Add(path)) { CommandLine.WriteLine(vault, $"Unable to add path: '{path}'", ConsoleColor.Yellow); return; @@ -204,7 +216,7 @@ private async Task RemovePathAsync(LocalVaultConfig vault, string path) return; } - if (!syncManager.RemoteVault.PushDirectories.Remove(path)) + if (!syncManager.RemoteVault.SyncDirectories.Remove(path)) { CommandLine.WriteLine(vault, $"Unable to remove path: '{path}'", ConsoleColor.Yellow); return; diff --git a/Parallel.Cli/Commands/VaultsCommand.cs b/Parallel.Cli/Commands/VaultsCommand.cs index aeb584e..696d540 100644 --- a/Parallel.Cli/Commands/VaultsCommand.cs +++ b/Parallel.Cli/Commands/VaultsCommand.cs @@ -55,7 +55,7 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") string bucketName = string.IsNullOrEmpty(bucketInput) ? "parallel" : bucketInput; spc.RootDirectory = bucketName; - string? regionInput = CommandLine.ReadString("Region Name (Leave empty for default)"); + string? regionInput = CommandLine.ReadString("Region (Leave empty for default)"); string regionName = string.IsNullOrEmpty(regionInput) ? "us-east-1" : regionInput; spc.Region = regionName; @@ -113,8 +113,7 @@ public VaultsCommand() : base("vaults", "View or edit the vaults.") RemoteVaultConfig remoteVault = syncManager.RemoteVault; CommandLine.WriteLine($"'{remoteVault.Name}' ({remoteVault.Id}):"); - CommandLine.WriteArray("Push Directories", remoteVault.PushDirectories); - CommandLine.WriteArray("Pull Directories", remoteVault.PullDirectories.Select(d => d.Source)); + CommandLine.WriteArray("Push Directories", remoteVault.SyncDirectories); CommandLine.WriteArray("Ignore Directories", remoteVault.IgnoreDirectories); CommandLine.WriteArray("Prune Directories", remoteVault.PruneDirectories); CommandLine.WriteLine($"Prune Period: {remoteVault.PrunePeriod} days"); diff --git a/Parallel.Cli/Services/RunCommand.cs b/Parallel.Cli/Services/RunCommand.cs index 85a06ad..56a5973 100644 --- a/Parallel.Cli/Services/RunCommand.cs +++ b/Parallel.Cli/Services/RunCommand.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Parallel.Core.Tasks; namespace Parallel.Cli.Services { @@ -15,22 +16,23 @@ public class RunCommand : Command public RunCommand() : base("run", "Starts the background services.") { - this.AddOption(_cleanOpt); - this.AddOption(_syncOpt); - this.AddOption(_pruneOpt); - this.SetHandler(HandleCommandAsync, _cleanOpt, _pruneOpt, _syncOpt); + this.SetHandler(HandleCommandAsync); } - private async Task HandleCommandAsync(bool clean, bool prune, bool sync) + private async Task HandleCommandAsync() { HostApplicationBuilder builder = Host.CreateApplicationBuilder(); if (OperatingSystem.IsWindows()) builder.Services.AddWindowsService(); if (OperatingSystem.IsLinux()) builder.Services.AddSystemd(); - if (clean) builder.Services.AddHostedService(); - if (prune) builder.Services.AddHostedService(); - if (sync) builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + + builder.Services.AddHostedService(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Logging.ClearProviders(); builder.Logging.AddSerilog(); diff --git a/Parallel.Cli/Services/SyncService.cs b/Parallel.Cli/Services/SyncService.cs index 9aab986..a3ed9a2 100644 --- a/Parallel.Cli/Services/SyncService.cs +++ b/Parallel.Cli/Services/SyncService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Parallel.Core.IO.Syncing; using Parallel.Core.Settings; +using Parallel.Core.Tasks; namespace Parallel.Cli.Services { @@ -24,11 +25,12 @@ public class SyncService : BackgroundService { private readonly ConcurrentDictionary _vaults = new(); private readonly ILogger _logger; - //private readonly TaskQueuer _queuer; + private readonly TaskQueuer _queuer; - public SyncService(ILogger logger) + public SyncService(ILogger logger, TaskQueuer queuer) { _logger = logger; + _queuer = queuer; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -73,6 +75,12 @@ private async Task SyncVaultAsync(FileSyncManager syncManager, CancellationToken { while (!stoppingToken.IsCancellationRequested) { + _queuer.Enqueue(syncManager.Id, (ct) => + { + Console.WriteLine($"Syncing vault {syncManager.Id}"); + return Task.CompletedTask; + }); + Log.Debug($"{syncManager.Id} will sync every {syncManager.RemoteVault.SyncInterval} minutes"); //await Task.Delay(TimeSpan.FromMinutes(syncManager.RemoteVault.SyncInterval), stoppingToken); await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); diff --git a/Parallel.Cli/Services/TaskService.cs b/Parallel.Cli/Services/TaskService.cs new file mode 100644 index 0000000..7af7913 --- /dev/null +++ b/Parallel.Cli/Services/TaskService.cs @@ -0,0 +1,22 @@ +// Copyright 2026 Entex Interactive + +using Microsoft.Extensions.Hosting; +using Parallel.Core.Tasks; + +namespace Parallel.Cli.Services +{ + public class TaskService : BackgroundService + { + private readonly TaskQueueExecutor _executor; + + public TaskService(TaskQueueExecutor executor) + { + _executor = executor; + } + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + { + return _executor.RunAsync(stoppingToken); + } + } +} \ No newline at end of file diff --git a/Parallel.Core/IO/Syncing/BaseSyncManager.cs b/Parallel.Core/IO/Syncing/BaseSyncManager.cs index af8eddd..a9ed5bb 100644 --- a/Parallel.Core/IO/Syncing/BaseSyncManager.cs +++ b/Parallel.Core/IO/Syncing/BaseSyncManager.cs @@ -126,10 +126,10 @@ public async Task DisconnectAsync() } /// - public abstract Task PushFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite); + public abstract Task SyncFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite); /// - public abstract Task PullFilesAsync(IReadOnlyList files, IProgressReporter progress); + public abstract Task RestoreFilesAsync(IReadOnlyList files, IProgressReporter progress); /// public abstract Task PruneFilesAsync(IReadOnlyList files, IProgressReporter progress); diff --git a/Parallel.Core/IO/Syncing/FileSyncManager.cs b/Parallel.Core/IO/Syncing/FileSyncManager.cs index 019c6a3..163e605 100644 --- a/Parallel.Core/IO/Syncing/FileSyncManager.cs +++ b/Parallel.Core/IO/Syncing/FileSyncManager.cs @@ -24,7 +24,7 @@ public class FileSyncManager : BaseSyncManager public FileSyncManager(LocalVaultConfig localVault) : base(localVault) { } /// - public override async Task PushFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite) + public override async Task SyncFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite) { if (!files.Any()) return 0; int completed = 0; @@ -86,7 +86,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(deletedFiles, ParallelConfig. } /// - public override async Task PullFilesAsync(IReadOnlyList files, IProgressReporter progress) + public override async Task RestoreFilesAsync(IReadOnlyList files, IProgressReporter progress) { if (!files.Any()) return 0; int completed = 0; @@ -173,28 +173,38 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options public override async Task ScrubFilesAsync(IReadOnlyList files, IProgressReporter progress) { if (!files.Any()) return 0; + int corrupted = 0; int completed = 0; + int unknown = 0; ConcurrentDictionary threadPool = new ConcurrentDictionary(); await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options, async (file, ct) => { - SemaphoreSlim threadLock = threadPool.GetOrAdd(file.LocalCheckSum!, _ => new SemaphoreSlim(1, 1)); + SemaphoreSlim threadLock = threadPool.GetOrAdd(file.RemoteCheckSum!, _ => new SemaphoreSlim(1, 1)); await threadLock.WaitAsync(ct); try { - string? remoteHash = await StorageProvider.HashFileAsync(PathBuilder.GetObjectFile(RemoteVault, file.RemoteCheckSum!), 81920, ct); + string remotePath = PathBuilder.GetObjectFile(RemoteVault, file.RemoteCheckSum!); + string? remoteHash = await StorageProvider.HashFileAsync(remotePath, 81920, ct); if (string.IsNullOrEmpty(remoteHash)) { // This is true when the remote file is not in the vault. - progress.Failed(file, "File not found!"); + if (!await (Database?.AddHistoryAsync(HistoryType.Pruned, file) ?? Task.FromResult(false))) Log.Error("Failed to add history: {Fullname}", file.Fullname); + if (!await (Database?.RemoveFileAsync(file) ?? Task.FromResult(false))) return; + progress.Report(ProgressOperation.Pruned, file); + Interlocked.Increment(ref unknown); return; } if (file.RemoteCheckSum != remoteHash) { // Is it true is the file stored is corrupted - progress.Failed(file, "File corrupted!"); + if (!await (Database?.AddHistoryAsync(HistoryType.Pruned, file) ?? Task.FromResult(false))) Log.Error("Failed to add history: {Fullname}", file.Fullname); + if (!await (Database?.RemoveFileAsync(file) ?? Task.FromResult(false))) return; + await StorageProvider.DeleteFileAsync(remotePath); + progress.Report(ProgressOperation.Pruned, file); + Interlocked.Increment(ref corrupted); return; } @@ -211,6 +221,7 @@ await System.Threading.Tasks.Parallel.ForEachAsync(files, ParallelConfig.Options } }); + Console.WriteLine($"Unknown: {unknown}, Corrupted: {corrupted}, Completed: {completed}"); return completed; } } diff --git a/Parallel.Core/IO/Syncing/ISyncManager.cs b/Parallel.Core/IO/Syncing/ISyncManager.cs index 4d12f3b..cb09cc4 100644 --- a/Parallel.Core/IO/Syncing/ISyncManager.cs +++ b/Parallel.Core/IO/Syncing/ISyncManager.cs @@ -54,14 +54,14 @@ public interface ISyncManager /// /// /// - Task PushFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite); + Task SyncFilesAsync(IReadOnlyList files, IProgressReporter progress, bool overwrite); /// /// Restores an array of files from a vault. /// /// /// - Task PullFilesAsync(IReadOnlyList files, IProgressReporter progress); + Task RestoreFilesAsync(IReadOnlyList files, IProgressReporter progress); /// /// Deletes files from a vault. diff --git a/Parallel.Core/Parallel.Core.csproj b/Parallel.Core/Parallel.Core.csproj index a056241..a895c51 100644 --- a/Parallel.Core/Parallel.Core.csproj +++ b/Parallel.Core/Parallel.Core.csproj @@ -37,10 +37,11 @@ - + + - + diff --git a/Parallel.Core/Settings/RemoteVaultConfig.cs b/Parallel.Core/Settings/RemoteVaultConfig.cs index b1d03d6..a457d6a 100644 --- a/Parallel.Core/Settings/RemoteVaultConfig.cs +++ b/Parallel.Core/Settings/RemoteVaultConfig.cs @@ -33,12 +33,7 @@ public class RemoteVaultConfig : LocalVaultConfig /// /// A collection of directories to be pushed to vaults. /// - public HashSet PushDirectories { get; } = CreatePushDirectories(); - - /// - /// A collection of directories to be pulled when reconciling. - /// - public HashSet PullDirectories { get; } = []; + public HashSet SyncDirectories { get; } = CreateSyncDirectories(); /// /// A collection of directories to be ignored when archiving or cleaning. @@ -58,15 +53,14 @@ public RemoteVaultConfig(LocalVaultConfig localVault) : base(localVault.Id, loca public RemoteVaultConfig(string id, string name, StorageCredentials credentials, int prunePeriod, IEnumerable? pushDirectories, IEnumerable? pullDirectories, IEnumerable? ignoreDirectories, IEnumerable? pruneDirectories) : base(id, name, credentials) { PrunePeriod = prunePeriod; - PushDirectories = new HashSet(pushDirectories ?? []); - PullDirectories = new HashSet(pullDirectories ?? []); + SyncDirectories = new HashSet(pushDirectories ?? []); IgnoreDirectories = new HashSet(ignoreDirectories ?? []); PruneDirectories = new HashSet(pruneDirectories ?? []); } #region Privates - private static HashSet CreatePushDirectories() + private static HashSet CreateSyncDirectories() { return [ diff --git a/Parallel.Core/Storage/SshStorageProvider.cs b/Parallel.Core/Storage/SshStorageProvider.cs index 564929f..3670300 100644 --- a/Parallel.Core/Storage/SshStorageProvider.cs +++ b/Parallel.Core/Storage/SshStorageProvider.cs @@ -29,7 +29,8 @@ public class SshStorageProvider : IStorageProvider /// The credentials to log in with. public SshStorageProvider(LocalVaultConfig localVault) { - _connectionInfo = new ConnectionInfo(localVault.Credentials.Address, localVault.Credentials.Username, new PasswordAuthenticationMethod(localVault.Credentials.Username, Encryption.Decode(localVault.Credentials.Password))); + //_connectionInfo = new ConnectionInfo(localVault.Credentials.Address, localVault.Credentials.Username, [new PasswordAuthenticationMethod(localVault.Credentials.Username, Encryption.Decode(localVault.Credentials.Password))]); + _connectionInfo = new ConnectionInfo(localVault.Credentials.Address ?? string.Empty, localVault.Credentials.Username ?? string.Empty, [new PasswordAuthenticationMethod(localVault.Credentials.Username, Encryption.Decode(localVault.Credentials.Password))]); _client = new SftpClient(_connectionInfo); } diff --git a/Parallel.Core/Tasks/EnqueuedTask.cs b/Parallel.Core/Tasks/EnqueuedTask.cs new file mode 100644 index 0000000..1fd5286 --- /dev/null +++ b/Parallel.Core/Tasks/EnqueuedTask.cs @@ -0,0 +1,41 @@ +// Copyright 2026 Entex Interactive + +namespace Parallel.Core.Tasks +{ + public class EnqueuedTask + { + /// + /// The time the task was created. + /// + public DateTime EnqueuedAt { get; } + + /// + /// The actual to execute. + /// + public Func TaskFunc { get; } + + /// + /// The unique key to use deduplication. + /// + public string Key { get; } + + /// + /// The priority of the task. + /// + public int Priority; + + /// + /// Creates a new instance of the class. + /// + /// + /// + /// + public EnqueuedTask(string key, Func taskFunc, int priority = 0) + { + Key = key; + EnqueuedAt = DateTime.UtcNow; + TaskFunc = taskFunc; + Priority = priority; + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Tasks/TaskQueueExecutor.cs b/Parallel.Core/Tasks/TaskQueueExecutor.cs new file mode 100644 index 0000000..13dda90 --- /dev/null +++ b/Parallel.Core/Tasks/TaskQueueExecutor.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.Logging; + +namespace Parallel.Core.Tasks +{ + public sealed class TaskQueueExecutor + { + private readonly TaskQueuer _queuer; + private readonly ILogger _logger; + private readonly int _maxConcurrency; + + public TaskQueueExecutor(TaskQueuer queuer, ILogger logger, int maxConcurrency) + { + _queuer = queuer; + _logger = logger; + _maxConcurrency = maxConcurrency; + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + IEnumerable workers = Enumerable.Range(0, _maxConcurrency).Select(_ => WorkerAsync(cancellationToken)); + await Task.WhenAll(workers); + } + + private async Task WorkerAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + Func task = await _queuer.WaitAsync(cancellationToken); + await task(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Queued task failed."); + } + } + } + } +} \ No newline at end of file diff --git a/Parallel.Core/Tasks/TaskQueueOptions.cs b/Parallel.Core/Tasks/TaskQueueOptions.cs new file mode 100644 index 0000000..7c25084 --- /dev/null +++ b/Parallel.Core/Tasks/TaskQueueOptions.cs @@ -0,0 +1,9 @@ +// Copyright 2026 Entex Interactive + +namespace Parallel.Core.Tasks +{ + public class TaskQueueOptions + { + public int MaxConcurrency { get; set; } = Environment.ProcessorCount; + } +} \ No newline at end of file diff --git a/Parallel.Core/Tasks/TaskQueuer.cs b/Parallel.Core/Tasks/TaskQueuer.cs new file mode 100644 index 0000000..a990d1c --- /dev/null +++ b/Parallel.Core/Tasks/TaskQueuer.cs @@ -0,0 +1,84 @@ +// Copyright 2026 Entex Interactive + +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace Parallel.Core.Tasks +{ + /// + /// Represents an asynchronous queue for running tasks. + /// + public class TaskQueuer + { + private readonly ILogger _logger; + private readonly ConcurrentQueue _queue = new(); + private readonly ConcurrentDictionary _tasks = new(); + private readonly SemaphoreSlim _semaphore = new(0); + + public TaskQueuer(ILogger logger) + { + _logger = logger; + } + + /// + /// Gets the number of elements contained in the . + /// + public int Count => _queue.Count; + + /// + /// Gets a value that indicates whether the is empty. + /// + public bool IsEmpty => _queue.IsEmpty; + + /// + /// Adds a task to be executed by the . + /// + /// The unique key to prevent + /// The task to be executed. + public void Enqueue(string key, Func task) + { + ArgumentNullException.ThrowIfNull(task); + Func safeTask = async (ct) => + { + try + { + await task(ct); + } + catch (Exception ex) + { + _logger.LogError(ex, "Exception throw executing task: {key}", key); + } + }; + + _tasks.AddOrUpdate(key, k => + { + EnqueuedTask enqueue = new EnqueuedTask(key, safeTask); + _queue.Enqueue(enqueue); + _semaphore.Release(); + return enqueue; + }, (k, existing) => + { + Interlocked.Increment(ref existing.Priority); + return existing; + }); + } + + public async Task> WaitAsync(CancellationToken ct) + { + await _semaphore.WaitAsync(ct); + while (true) + { + if (!_queue.TryDequeue(out EnqueuedTask? candidate)) continue; + EnqueuedTask? next = _tasks.Values.MaxBy(t => t.Priority + (DateTime.UtcNow - t.EnqueuedAt).TotalSeconds * 0.1); + if (ReferenceEquals(candidate, next)) + { + _logger.LogDebug($"Starting task with key: '{candidate.Key}', remaining: {_queue.Count - 1}"); + _tasks.TryRemove(candidate.Key, out _); + return candidate.TaskFunc; + } + + _queue.Enqueue(candidate); + } + } + } +} \ No newline at end of file