using InnovEnergy.App.Backend.Database; using InnovEnergy.App.Backend.DataTypes.Methods; using InnovEnergy.Lib.S3Utils; using InnovEnergy.Lib.S3Utils.DataTypes; namespace InnovEnergy.App.Backend.DeleteOldData; public static class DeleteOldDataFromS3 { private static Timer? _cleanupTimer; public static void StartScheduler() { var now = DateTime.UtcNow; var next = new DateTime(now.Year, now.Month, now.Day, 3, 0, 0, DateTimeKind.Utc); if (next <= now) next = next.AddDays(1); _cleanupTimer = new Timer( _ => { try { CleanupAllInstallations().GetAwaiter().GetResult(); } catch (Exception ex) { Console.Error.WriteLine($"[S3Cleanup] Scheduler error: {ex.Message}"); } }, null, next - now, TimeSpan.FromDays(1) ); Console.WriteLine($"[S3Cleanup] Scheduled daily at 03:00 UTC, first run in {(next - now).TotalHours:F1}h"); } private static async Task CleanupAllInstallations() { var cutoffTimestamp = DateTimeOffset.UtcNow.AddYears(-1).ToUnixTimeSeconds(); var cutoffKey = cutoffTimestamp.ToString(); var installations = Db.Installations.ToList(); Console.WriteLine($"[S3Cleanup] Starting cleanup for {installations.Count} installations, cutoff: {cutoffKey}"); foreach (var installation in installations) { try { var s3Region = new S3Region( $"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials ); var bucket = s3Region.Bucket(installation.BucketName()); Console.WriteLine($"[S3Cleanup] Processing {installation.Name} (bucket: {bucket.Name})"); var deleted = await DeleteObjectsBefore(bucket, cutoffKey); Console.WriteLine($"[S3Cleanup] {installation.Name}: deleted {deleted} objects"); } catch (Exception ex) { Console.Error.WriteLine($"[S3Cleanup] Failed for {installation.Name}: {ex.Message}"); } } Console.WriteLine("[S3Cleanup] Finished cleanup for all installations"); } public static async Task DryRun(long? installationId = null) { var cutoffTimestamp = DateTimeOffset.UtcNow.AddYears(-1).ToUnixTimeSeconds(); var cutoffKey = cutoffTimestamp.ToString(); var allInstallations = Db.Installations.ToList(); var installations = installationId.HasValue ? allInstallations.Where(i => i.Id == installationId.Value).ToList() : allInstallations; var results = new List(); results.Add($"Cutoff: {cutoffKey} ({DateTimeOffset.FromUnixTimeSeconds(cutoffTimestamp):yyyy-MM-dd HH:mm:ss} UTC)"); results.Add($"Installations: {installations.Count} (of {allInstallations.Count} total)"); results.Add(""); foreach (var installation in installations) { try { var s3Region = new S3Region( $"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials ); var bucket = s3Region.Bucket(installation.BucketName()); var sampleKeys = new List(); var hasOldData = false; await foreach (var obj in bucket.ListObjects()) { if (string.Compare(obj.Path, cutoffKey, StringComparison.Ordinal) >= 0) break; hasOldData = true; if (sampleKeys.Count < 5) sampleKeys.Add(obj.Path); else break; // only need a sample, not full count } results.Add($"{installation.Name} (bucket: {bucket.Name})"); results.Add($" Has old data: {(hasOldData ? "YES" : "NO")}"); if (sampleKeys.Count > 0) results.Add($" Sample keys: {string.Join(", ", sampleKeys)}"); results.Add(""); } catch (Exception ex) { results.Add($"{installation.Name}: ERROR - {ex.Message}"); results.Add(""); } } return string.Join("\n", results); } private static async Task DeleteObjectsBefore(S3Bucket bucket, string cutoffKey) { var totalDeleted = 0; var keysToDelete = new List(); await foreach (var obj in bucket.ListObjects()) { if (string.Compare(obj.Path, cutoffKey, StringComparison.Ordinal) >= 0) break; keysToDelete.Add(obj.Path); if (keysToDelete.Count >= 1000) { if (await bucket.DeleteObjects(keysToDelete)) totalDeleted += keysToDelete.Count; else Console.Error.WriteLine($"[S3Cleanup] Failed to delete batch of {keysToDelete.Count} objects from {bucket.Name}"); keysToDelete.Clear(); } } if (keysToDelete.Count > 0) { if (await bucket.DeleteObjects(keysToDelete)) totalDeleted += keysToDelete.Count; else Console.Error.WriteLine($"[S3Cleanup] Failed to delete batch of {keysToDelete.Count} objects from {bucket.Name}"); } return totalDeleted; } }