Innovenergy
/
Innovenergy_trunk
3
1
Fork 1
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Add "Delete data older than one year from each installation" functionality in the backend

This commit is contained in:
Noe 2025-06-05 11:06:30 +02:00
parent 857f220f43
commit e1b4998d30
3 changed files with 133 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
csharp/App/Backend/DeleteOldData/DeleteOldDataFromS3.cs Normal file
View File

@ -0,0 +1,94 @@
using System.Diagnostics;
using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.DataTypes;
using InnovEnergy.Lib.Utils;
namespace InnovEnergy.App.Backend.DeleteOldData;
public class DeleteOldDataFromS3
{
public static void DeleteFrom(Installation installation, int timestamps_to_delete)
{
string configPath = "/home/ubuntu/.s3cfg";
string bucketPath = installation.Product ==(int)ProductType.Salidomo ? $"s3://{installation.S3BucketId}-c0436b6a-d276-4cd8-9c44-1eae86cf5d0e/{timestamps_to_delete}*" : $"s3://{installation.S3BucketId}-3e5b3069-214a-43ee-8d85-57d72000c19d/{timestamps_to_delete}*" ;
//Console.WriteLine($"Deleting old data from {bucketPath}");
Console.WriteLine("Deleting data for timestamp prefix: " + timestamps_to_delete);
try
{
ProcessStartInfo startInfo = new ProcessStartInfo
{
FileName = "s3cmd",
Arguments = $"--config {configPath} rm {bucketPath}",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using Process process = new Process { StartInfo = startInfo };
process.OutputDataReceived += (sender, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
Console.WriteLine("[s3cmd] " + e.Data);
};
process.ErrorDataReceived += (sender, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
Console.WriteLine("[s3cmd-ERR] " + e.Data);
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
process.WaitForExit();
}
catch (Exception ex)
{
Console.WriteLine("Exception occurred during deletion: " + ex.Message);
}
}
public static async Task DeleteOldData()
{
while (true){
var installations = Db.Installations.ToList();
foreach (var installation in installations){
Console.WriteLine("DELETE S3 DATA FOR INSTALLATION "+installation.Name);
long oneYearAgoTimestamp = DateTimeOffset.UtcNow.AddYears(-1).ToUnixTimeSeconds();
Console.WriteLine("delete data before "+oneYearAgoTimestamp);
for (int lastDigit=4;lastDigit>=0; lastDigit--)
{
int timestamps_to_delete = int.Parse(oneYearAgoTimestamp.ToString().Substring(0, lastDigit+1));
timestamps_to_delete--;
Console.WriteLine(timestamps_to_delete);
while (true)
{
if (timestamps_to_delete % 10 == 0)
{
Console.WriteLine("delete " + timestamps_to_delete + "*");
DeleteFrom(installation,timestamps_to_delete);
break;
}
Console.WriteLine("delete " + timestamps_to_delete + "*");
DeleteFrom(installation,timestamps_to_delete);
timestamps_to_delete--;
}
}
}
Console.WriteLine("FINISHED DELETING S3 DATA FOR ALL INSTALLATIONS\n");
await Task.Delay(TimeSpan.FromDays(1));
}
}
}

4
csharp/App/Backend/Program.cs
View File

@ -3,6 +3,7 @@ using Flurl.Http;
using Hellang.Middleware.ProblemDetails; using Hellang.Middleware.ProblemDetails;
using InnovEnergy.App.Backend.Database; using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.Websockets; using InnovEnergy.App.Backend.Websockets;
using InnovEnergy.App.Backend.DeleteOldData;
using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.OpenApi.Models; using Microsoft.OpenApi.Models;
@ -30,6 +31,9 @@ public static class Program
WebsocketManager.MonitorSalimaxInstallationTable().SupressAwaitWarning(); WebsocketManager.MonitorSalimaxInstallationTable().SupressAwaitWarning();
WebsocketManager.MonitorSalidomoInstallationTable().SupressAwaitWarning(); WebsocketManager.MonitorSalidomoInstallationTable().SupressAwaitWarning();
WebsocketManager.MonitorSodistoreInstallationTable().SupressAwaitWarning(); WebsocketManager.MonitorSodistoreInstallationTable().SupressAwaitWarning();
Task.Run(() => DeleteOldDataFromS3.DeleteOldData());
builder.Services.AddControllers(); builder.Services.AddControllers();
builder.Services.AddProblemDetails(setup => builder.Services.AddProblemDetails(setup =>

70
csharp/App/Backend/Websockets/WebsockerManager.cs
View File

@ -19,7 +19,7 @@ public static class WebsocketManager
{ {
while (true){ while (true){
lock (InstallationConnections){ lock (InstallationConnections){
Console.WriteLine("MONITOR SALIMAX INSTALLATIONS\n"); // Console.WriteLine("MONITOR SALIMAX INSTALLATIONS\n");
foreach (var installationConnection in InstallationConnections){ foreach (var installationConnection in InstallationConnections){
if (installationConnection.Value.Product==(int)ProductType.Salimax && (DateTime.Now - installationConnection.Value.Timestamp) > TimeSpan.FromMinutes(2)){ if (installationConnection.Value.Product==(int)ProductType.Salimax && (DateTime.Now - installationConnection.Value.Timestamp) > TimeSpan.FromMinutes(2)){
@ -35,7 +35,7 @@ public static class WebsocketManager
if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);} if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);}
} }
} }
Console.WriteLine("FINISHED MONITORING SALIMAX INSTALLATIONS\n"); // Console.WriteLine("FINISHED MONITORING SALIMAX INSTALLATIONS\n");
} }
await Task.Delay(TimeSpan.FromMinutes(1)); await Task.Delay(TimeSpan.FromMinutes(1));
@ -47,15 +47,15 @@ public static class WebsocketManager
public static async Task MonitorSalidomoInstallationTable() public static async Task MonitorSalidomoInstallationTable()
{ {
while (true){ while (true){
Console.WriteLine("TRY TO LOCK FOR MONITOR SALIDOMO INSTALLATIONS\n"); //Console.WriteLine("TRY TO LOCK FOR MONITOR SALIDOMO INSTALLATIONS\n");
lock (InstallationConnections){ lock (InstallationConnections){
Console.WriteLine("MONITOR SALIDOMO INSTALLATIONS\n"); //Console.WriteLine("MONITOR SALIDOMO INSTALLATIONS\n");
foreach (var installationConnection in InstallationConnections) foreach (var installationConnection in InstallationConnections)
{ {
//Console.WriteLine("Installation ID is "+installationConnection.Key); //Console.WriteLine("Installation ID is "+installationConnection.Key);
if (installationConnection.Value.Product == (int)ProductType.Salidomo && (DateTime.Now - installationConnection.Value.Timestamp) < TimeSpan.FromMinutes(60)){ // if (installationConnection.Value.Product == (int)ProductType.Salidomo && (DateTime.Now - installationConnection.Value.Timestamp) < TimeSpan.FromMinutes(60)){
Console.WriteLine("Installation ID is "+installationConnection.Key + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp)); // Console.WriteLine("Installation ID is "+installationConnection.Key + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp));
} // }
if (installationConnection.Value.Product==(int)ProductType.Salidomo && (DateTime.Now - installationConnection.Value.Timestamp) > TimeSpan.FromMinutes(60)) if (installationConnection.Value.Product==(int)ProductType.Salidomo && (DateTime.Now - installationConnection.Value.Timestamp) > TimeSpan.FromMinutes(60))
{ {
@ -64,16 +64,16 @@ public static class WebsocketManager
//Console.WriteLine("timestamp now is is "+(DateTime.Now)); //Console.WriteLine("timestamp now is is "+(DateTime.Now));
Installation installation = Db.Installations.FirstOrDefault(f => f.Product == (int)ProductType.Salidomo && f.Id == installationConnection.Key); Installation installation = Db.Installations.FirstOrDefault(f => f.Product == (int)ProductType.Salidomo && f.Id == installationConnection.Key);
Console.WriteLine("Installation ID is "+installation.Name + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp)); //Console.WriteLine("Installation ID is "+installation.Name + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp));
installation.Status = (int)StatusType.Offline; installation.Status = (int)StatusType.Offline;
installation.Apply(Db.Update); installation.Apply(Db.Update);
installationConnection.Value.Status = (int)StatusType.Offline; installationConnection.Value.Status = (int)StatusType.Offline;
if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);} if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);}
else{Console.WriteLine("NONE IS CONNECTED TO THAT INSTALLATION-------------------------------------------------------------");} //else{Console.WriteLine("NONE IS CONNECTED TO THAT INSTALLATION-------------------------------------------------------------");}
} }
} }
Console.WriteLine("FINISHED WITH UPDATING\n"); //Console.WriteLine("FINISHED WITH UPDATING\n");
} }
await Task.Delay(TimeSpan.FromMinutes(1)); await Task.Delay(TimeSpan.FromMinutes(1));
} }
@ -84,9 +84,9 @@ public static class WebsocketManager
public static async Task MonitorSodistoreInstallationTable() public static async Task MonitorSodistoreInstallationTable()
{ {
while (true){ while (true){
Console.WriteLine("TRY TO LOCK FOR MONITOR SODISTORE INSTALLATIONS\n"); //Console.WriteLine("TRY TO LOCK FOR MONITOR SODISTORE INSTALLATIONS\n");
lock (InstallationConnections){ lock (InstallationConnections){
Console.WriteLine("MONITOR SODISTORE INSTALLATIONS\n"); //Console.WriteLine("MONITOR SODISTORE INSTALLATIONS\n");
foreach (var installationConnection in InstallationConnections) foreach (var installationConnection in InstallationConnections)
{ {
@ -97,16 +97,16 @@ public static class WebsocketManager
//Console.WriteLine("timestamp now is is "+(DateTime.Now)); //Console.WriteLine("timestamp now is is "+(DateTime.Now));
Installation installation = Db.Installations.FirstOrDefault(f => f.Product == (int)ProductType.SodiStoreMax && f.Id == installationConnection.Key); Installation installation = Db.Installations.FirstOrDefault(f => f.Product == (int)ProductType.SodiStoreMax && f.Id == installationConnection.Key);
Console.WriteLine("Installation ID is "+installation.Name + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp)); //Console.WriteLine("Installation ID is "+installation.Name + " diff is "+(DateTime.Now-installationConnection.Value.Timestamp));
installation.Status = (int)StatusType.Offline; installation.Status = (int)StatusType.Offline;
installation.Apply(Db.Update); installation.Apply(Db.Update);
installationConnection.Value.Status = (int)StatusType.Offline; installationConnection.Value.Status = (int)StatusType.Offline;
if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);} if (installationConnection.Value.Connections.Count > 0){InformWebsocketsForInstallation(installationConnection.Key);}
else{Console.WriteLine("NONE IS CONNECTED TO THAT INSTALLATION-------------------------------------------------------------");} //else{Console.WriteLine("NONE IS CONNECTED TO THAT INSTALLATION-------------------------------------------------------------");}
} }
} }
Console.WriteLine("FINISHED WITH UPDATING\n"); //Console.WriteLine("FINISHED WITH UPDATING\n");
} }
await Task.Delay(TimeSpan.FromMinutes(1)); await Task.Delay(TimeSpan.FromMinutes(1));
} }
@ -225,20 +225,20 @@ public static class WebsocketManager
); );
Console.WriteLine("Printing installation connection list"); // Console.WriteLine("Printing installation connection list");
Console.WriteLine("----------------------------------------------"); // Console.WriteLine("----------------------------------------------");
foreach (var installationConnection in InstallationConnections) // foreach (var installationConnection in InstallationConnections)
{ // {
Console.WriteLine("Installation ID: " + installationConnection.Key + " Number of Connections: " + installationConnection.Value.Connections.Count); // Console.WriteLine("Installation ID: " + installationConnection.Key + " Number of Connections: " + installationConnection.Value.Connections.Count);
} // }
Console.WriteLine("----------------------------------------------"); // Console.WriteLine("----------------------------------------------");
} }
} }
lock (InstallationConnections) lock (InstallationConnections)
{ {
//When the front-end terminates the connection, the following code will be executed //When the front-end terminates the connection, the following code will be executed
Console.WriteLine("The connection has been terminated"); //Console.WriteLine("The connection has been terminated");
foreach (var installationConnection in InstallationConnections) foreach (var installationConnection in InstallationConnections)
{ {
if (installationConnection.Value.Connections.Contains(currentWebSocket)) if (installationConnection.Value.Connections.Contains(currentWebSocket))
@ -249,18 +249,18 @@ public static class WebsocketManager
} }
await currentWebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Connection closed by server", CancellationToken.None); await currentWebSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Connection closed by server", CancellationToken.None);
lock (InstallationConnections) // lock (InstallationConnections)
{ // {
//Print the installationConnections dictionary after deleting a websocket // //Print the installationConnections dictionary after deleting a websocket
Console.WriteLine("Print the installation connections list after deleting a websocket"); // Console.WriteLine("Print the installation connections list after deleting a websocket");
Console.WriteLine("----------------------------------------------"); // Console.WriteLine("----------------------------------------------");
foreach (var installationConnection in InstallationConnections) // foreach (var installationConnection in InstallationConnections)
{ // {
Console.WriteLine("Installation ID: " + installationConnection.Key + " Number of Connections: " + installationConnection.Value.Connections.Count); // Console.WriteLine("Installation ID: " + installationConnection.Key + " Number of Connections: " + installationConnection.Value.Connections.Count);
} // }
//
Console.WriteLine("----------------------------------------------"); // Console.WriteLine("----------------------------------------------");
} // }
} }
catch (Exception ex) catch (Exception ex)
{ {