using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.DataTypes;
namespace InnovEnergy.App.Backend.Services;
///
/// Ingests daily energy totals into the DailyEnergyRecord SQLite table.
/// Data source priority: JSON (local) → JSON (S3) → xlsx fallback.
/// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the
/// IngestDailyData API endpoint.
///
public static class DailyIngestionService
{
private static readonly String TmpReportDir =
Environment.CurrentDirectory + "/tmp_report/";
private static readonly String JsonAggregatedDir =
Environment.CurrentDirectory + "/tmp_report/aggregated/";
private static Timer? _dailyTimer;
///
/// Starts the daily scheduler. Call once on app startup.
/// Ingests xlsx data at 01:00 UTC every day.
///
public static void StartScheduler()
{
var now = DateTime.UtcNow;
var next = now.Date.AddDays(1).AddHours(1); // 01:00 UTC tomorrow
_dailyTimer = new Timer(
_ =>
{
try
{
IngestAllInstallationsAsync().GetAwaiter().GetResult();
}
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Scheduler error: {ex.Message}");
}
},
null, next - now, TimeSpan.FromDays(1));
Console.WriteLine($"[DailyIngestion] Scheduler started. Next run: {next:yyyy-MM-dd HH:mm} UTC");
}
///
/// Ingests xlsx data for all SodioHome installations. Safe to call manually.
///
public static async Task IngestAllInstallationsAsync()
{
Console.WriteLine($"[DailyIngestion] Starting ingestion for all SodioHome installations...");
var installations = Db.Installations
.Where(i => i.Product == (Int32)ProductType.SodioHome)
.ToList();
foreach (var installation in installations)
{
try
{
await IngestInstallationAsync(installation.Id);
}
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed for installation {installation.Id}: {ex.Message}");
}
}
Console.WriteLine($"[DailyIngestion] Ingestion complete.");
}
///
/// Ingests data for one installation. Tries JSON (local + S3) and xlsx.
/// Both sources are tried — idempotency checks prevent duplicates.
/// JSON provides recent data; xlsx provides historical data.
///
public static async Task IngestInstallationAsync(Int64 installationId)
{
await TryIngestFromJson(installationId);
IngestFromXlsx(installationId);
}
private static async Task TryIngestFromJson(Int64 installationId)
{
var newDaily = 0;
var newHourly = 0;
var jsonDir = Path.Combine(JsonAggregatedDir, installationId.ToString());
// Collect JSON content from local files
var jsonFiles = Directory.Exists(jsonDir)
? Directory.GetFiles(jsonDir, "*.json")
: Array.Empty();
foreach (var jsonPath in jsonFiles.OrderBy(f => f))
{
var content = File.ReadAllText(jsonPath);
var (d, h) = IngestJsonContent(installationId, content);
newDaily += d;
newHourly += h;
}
// Also try S3 for recent days (yesterday + today) if no local files found
if (jsonFiles.Length == 0)
{
var installation = Db.GetInstallationById(installationId);
if (installation is not null)
{
for (var daysBack = 0; daysBack <= 1; daysBack++)
{
var date = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-daysBack));
var isoDate = date.ToString("yyyy-MM-dd");
if (Db.DailyRecordExists(installationId, isoDate))
continue;
var content = await AggregatedJsonParser.TryReadFromS3(installation, isoDate);
if (content is null) continue;
var (d, h) = IngestJsonContent(installationId, content);
newDaily += d;
newHourly += h;
}
}
}
if (newDaily > 0 || newHourly > 0)
Console.WriteLine($"[DailyIngestion] Installation {installationId} (JSON): {newDaily} day(s), {newHourly} hour(s) ingested.");
return newDaily > 0 || newHourly > 0;
}
public static (Int32 daily, Int32 hourly) IngestJsonContent(Int64 installationId, String content)
{
var newDaily = 0;
var newHourly = 0;
foreach (var day in AggregatedJsonParser.ParseDaily(content))
{
if (Db.DailyRecordExists(installationId, day.Date))
continue;
Db.Create(new DailyEnergyRecord
{
InstallationId = installationId,
Date = day.Date,
PvProduction = day.PvProduction,
LoadConsumption = day.LoadConsumption,
GridImport = day.GridImport,
GridExport = day.GridExport,
BatteryCharged = day.BatteryCharged,
BatteryDischarged = day.BatteryDischarged,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newDaily++;
}
foreach (var hour in AggregatedJsonParser.ParseHourly(content))
{
var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
if (Db.HourlyRecordExists(installationId, dateHour))
continue;
Db.Create(new HourlyEnergyRecord
{
InstallationId = installationId,
Date = hour.DateTime.ToString("yyyy-MM-dd"),
Hour = hour.Hour,
DateHour = dateHour,
DayOfWeek = hour.DayOfWeek,
IsWeekend = hour.IsWeekend,
PvKwh = hour.PvKwh,
LoadKwh = hour.LoadKwh,
GridImportKwh = hour.GridImportKwh,
BatteryChargedKwh = hour.BatteryChargedKwh,
BatteryDischargedKwh = hour.BatteryDischargedKwh,
BattSoC = 0,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newHourly++;
}
return (newDaily, newHourly);
}
private static void IngestFromXlsx(Int64 installationId)
{
if (!Directory.Exists(TmpReportDir))
{
Console.WriteLine($"[DailyIngestion] tmp_report directory not found, skipping.");
return;
}
var xlsxFiles = Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx");
if (xlsxFiles.Length == 0)
{
Console.WriteLine($"[DailyIngestion] No xlsx found for installation {installationId}, skipping.");
return;
}
var newDailyCount = 0;
var newHourlyCount = 0;
var totalParsed = 0;
foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
{
List days;
try { days = ExcelDataParser.Parse(xlsxPath); }
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
continue;
}
totalParsed += days.Count;
foreach (var day in days)
{
if (Db.DailyRecordExists(installationId, day.Date))
continue;
Db.Create(new DailyEnergyRecord
{
InstallationId = installationId,
Date = day.Date,
PvProduction = day.PvProduction,
LoadConsumption = day.LoadConsumption,
GridImport = day.GridImport,
GridExport = day.GridExport,
BatteryCharged = day.BatteryCharged,
BatteryDischarged = day.BatteryDischarged,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newDailyCount++;
}
List hours;
try { hours = ExcelDataParser.ParseHourly(xlsxPath); }
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
continue;
}
foreach (var hour in hours)
{
var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
if (Db.HourlyRecordExists(installationId, dateHour))
continue;
Db.Create(new HourlyEnergyRecord
{
InstallationId = installationId,
Date = hour.DateTime.ToString("yyyy-MM-dd"),
Hour = hour.Hour,
DateHour = dateHour,
DayOfWeek = hour.DayOfWeek,
IsWeekend = hour.IsWeekend,
PvKwh = hour.PvKwh,
LoadKwh = hour.LoadKwh,
GridImportKwh = hour.GridImportKwh,
BatteryChargedKwh = hour.BatteryChargedKwh,
BatteryDischargedKwh = hour.BatteryDischargedKwh,
BattSoC = hour.BattSoC,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newHourlyCount++;
}
}
Console.WriteLine($"[DailyIngestion] Installation {installationId} (xlsx): {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
}
}