diff --git a/csharp/App/Backend/Controller.cs b/csharp/App/Backend/Controller.cs
index 435857e9e..bf7ab8c6b 100644
--- a/csharp/App/Backend/Controller.cs
+++ b/csharp/App/Backend/Controller.cs
@@ -1257,7 +1257,7 @@ public class Controller : ControllerBase
///
/// Returns daily + hourly records for a date range.
- /// DB first; if empty, falls back to xlsx parsing and caches results for future calls.
+ /// Fallback chain: DB → JSON (local + S3) → xlsx. Caches to DB on first parse.
///
[HttpGet(nameof(GetDailyDetailRecords))]
public ActionResult GetDailyDetailRecords(
@@ -1279,73 +1279,107 @@ public class Controller : ControllerBase
var dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
var hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
- // 2. Fallback: parse xlsx + cache to DB
- if (dailyRecords.Count == 0 || hourlyRecords.Count == 0)
+ if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
+ return Ok(FormatResult(dailyRecords, hourlyRecords));
+
+ // 2. Fallback: try JSON (local files + S3)
+ TryIngestFromJson(installationId, installation, fromDate, toDate);
+ dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
+ hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
+
+ if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
+ return Ok(FormatResult(dailyRecords, hourlyRecords));
+
+ // 3. Fallback: parse xlsx + cache to DB
+ TryIngestFromXlsx(installationId, fromDate, toDate);
+ dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
+ hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
+
+ return Ok(FormatResult(dailyRecords, hourlyRecords));
+ }
+
+ private static Object FormatResult(
+ List daily, List hourly) => new
+ {
+ dailyRecords = new { count = daily.Count, records = daily },
+ hourlyRecords = new { count = hourly.Count, records = hourly },
+ };
+
+ private static void TryIngestFromJson(
+ Int64 installationId, Installation installation,
+ DateOnly fromDate, DateOnly toDate)
+ {
+ var jsonDir = Path.Combine(
+ Environment.CurrentDirectory, "tmp_report", "aggregated", installationId.ToString());
+
+ for (var date = fromDate; date <= toDate; date = date.AddDays(1))
{
- var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate);
- if (xlsxFiles.Count > 0)
+ var isoDate = date.ToString("yyyy-MM-dd");
+ var fileName = AggregatedJsonParser.ToJsonFileName(date);
+
+ // Try local file first
+ var localPath = Path.Combine(jsonDir, fileName);
+ String? content = System.IO.File.Exists(localPath) ? System.IO.File.ReadAllText(localPath) : null;
+
+ // Try S3 if no local file
+ content ??= AggregatedJsonParser.TryReadFromS3(installation, isoDate)
+ .GetAwaiter().GetResult();
+
+ if (content is null) continue;
+
+ DailyIngestionService.IngestJsonContent(installationId, content);
+ }
+ }
+
+ private static void TryIngestFromXlsx(
+ Int64 installationId, DateOnly fromDate, DateOnly toDate)
+ {
+ var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate);
+ if (xlsxFiles.Count == 0) return;
+
+ foreach (var xlsxPath in xlsxFiles)
+ {
+ foreach (var day in ExcelDataParser.Parse(xlsxPath))
{
- foreach (var xlsxPath in xlsxFiles)
+ if (Db.DailyRecordExists(installationId, day.Date))
+ continue;
+ Db.Create(new DailyEnergyRecord
{
- if (dailyRecords.Count == 0)
- {
- foreach (var day in ExcelDataParser.Parse(xlsxPath))
- {
- if (Db.DailyRecordExists(installationId, day.Date))
- continue;
- Db.Create(new DailyEnergyRecord
- {
- InstallationId = installationId,
- Date = day.Date,
- PvProduction = day.PvProduction,
- LoadConsumption = day.LoadConsumption,
- GridImport = day.GridImport,
- GridExport = day.GridExport,
- BatteryCharged = day.BatteryCharged,
- BatteryDischarged = day.BatteryDischarged,
- CreatedAt = DateTime.UtcNow.ToString("o"),
- });
- }
- }
+ InstallationId = installationId,
+ Date = day.Date,
+ PvProduction = day.PvProduction,
+ LoadConsumption = day.LoadConsumption,
+ GridImport = day.GridImport,
+ GridExport = day.GridExport,
+ BatteryCharged = day.BatteryCharged,
+ BatteryDischarged = day.BatteryDischarged,
+ CreatedAt = DateTime.UtcNow.ToString("o"),
+ });
+ }
- if (hourlyRecords.Count == 0)
- {
- foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath))
- {
- var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
- if (Db.HourlyRecordExists(installationId, dateHour))
- continue;
- Db.Create(new HourlyEnergyRecord
- {
- InstallationId = installationId,
- Date = hour.DateTime.ToString("yyyy-MM-dd"),
- Hour = hour.Hour,
- DateHour = dateHour,
- DayOfWeek = hour.DayOfWeek,
- IsWeekend = hour.IsWeekend,
- PvKwh = hour.PvKwh,
- LoadKwh = hour.LoadKwh,
- GridImportKwh = hour.GridImportKwh,
- BatteryChargedKwh = hour.BatteryChargedKwh,
- BatteryDischargedKwh = hour.BatteryDischargedKwh,
- BattSoC = hour.BattSoC,
- CreatedAt = DateTime.UtcNow.ToString("o"),
- });
- }
- }
- }
-
- // Re-read from DB (now cached)
- dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
- hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
+ foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath))
+ {
+ var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
+ if (Db.HourlyRecordExists(installationId, dateHour))
+ continue;
+ Db.Create(new HourlyEnergyRecord
+ {
+ InstallationId = installationId,
+ Date = hour.DateTime.ToString("yyyy-MM-dd"),
+ Hour = hour.Hour,
+ DateHour = dateHour,
+ DayOfWeek = hour.DayOfWeek,
+ IsWeekend = hour.IsWeekend,
+ PvKwh = hour.PvKwh,
+ LoadKwh = hour.LoadKwh,
+ GridImportKwh = hour.GridImportKwh,
+ BatteryChargedKwh = hour.BatteryChargedKwh,
+ BatteryDischargedKwh = hour.BatteryDischargedKwh,
+ BattSoC = hour.BattSoC,
+ CreatedAt = DateTime.UtcNow.ToString("o"),
+ });
}
}
-
- return Ok(new
- {
- dailyRecords = new { count = dailyRecords.Count, records = dailyRecords },
- hourlyRecords = new { count = hourlyRecords.Count, records = hourlyRecords },
- });
}
///
diff --git a/csharp/App/Backend/Services/AggregatedJsonParser.cs b/csharp/App/Backend/Services/AggregatedJsonParser.cs
new file mode 100644
index 000000000..bcc613c8d
--- /dev/null
+++ b/csharp/App/Backend/Services/AggregatedJsonParser.cs
@@ -0,0 +1,162 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using InnovEnergy.App.Backend.DataTypes;
+using InnovEnergy.App.Backend.DataTypes.Methods;
+using InnovEnergy.Lib.S3Utils;
+using InnovEnergy.Lib.S3Utils.DataTypes;
+using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
+
+namespace InnovEnergy.App.Backend.Services;
+
+///
+/// Parses NDJSON aggregated data files generated by SodistoreHome devices.
+/// Each file (DDMMYYYY.json) contains one JSON object per line:
+/// - Type "Hourly": per-hour kWh values (already computed, no diffing needed)
+/// - Type "Daily": daily totals
+///
+public static class AggregatedJsonParser
+{
+ private static readonly JsonSerializerOptions JsonOpts = new()
+ {
+ PropertyNameCaseInsensitive = true,
+ NumberHandling = JsonNumberHandling.AllowReadingFromString,
+ };
+
+ public static List ParseDaily(String ndjsonContent)
+ {
+ var dailyByDate = new SortedDictionary();
+
+ foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+ {
+ if (!line.Contains("\"Type\":\"Daily\""))
+ continue;
+
+ try
+ {
+ var raw = JsonSerializer.Deserialize(line, JsonOpts);
+ if (raw is null) continue;
+
+ var date = raw.Timestamp.ToString("yyyy-MM-dd");
+
+ dailyByDate[date] = new DailyEnergyData
+ {
+ Date = date,
+ PvProduction = Math.Round(raw.DailySelfGeneratedElectricity, 4),
+ GridImport = Math.Round(raw.DailyElectricityPurchased, 4),
+ GridExport = Math.Round(raw.DailyElectricityFed, 4),
+ BatteryCharged = Math.Round(raw.BatteryDailyChargeEnergy, 4),
+ BatteryDischarged = Math.Round(raw.BatteryDailyDischargeEnergy, 4),
+ LoadConsumption = Math.Round(raw.DailyLoadPowerConsumption, 4),
+ };
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"[AggregatedJsonParser] Skipping daily line: {ex.Message}");
+ }
+ }
+
+ Console.WriteLine($"[AggregatedJsonParser] Parsed {dailyByDate.Count} daily record(s)");
+ return dailyByDate.Values.ToList();
+ }
+
+ public static List ParseHourly(String ndjsonContent)
+ {
+ var result = new List();
+
+ foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+ {
+ if (!line.Contains("\"Type\":\"Hourly\""))
+ continue;
+
+ try
+ {
+ var raw = JsonSerializer.Deserialize(line, JsonOpts);
+ if (raw is null) continue;
+
+ var dt = new DateTime(
+ raw.Timestamp.Year, raw.Timestamp.Month, raw.Timestamp.Day,
+ raw.Timestamp.Hour, 0, 0);
+
+ result.Add(new HourlyEnergyData
+ {
+ DateTime = dt,
+ Hour = dt.Hour,
+ DayOfWeek = dt.DayOfWeek.ToString(),
+ IsWeekend = dt.DayOfWeek is System.DayOfWeek.Saturday or System.DayOfWeek.Sunday,
+ PvKwh = Math.Round(raw.SelfGeneratedElectricity, 4),
+ GridImportKwh = Math.Round(raw.ElectricityPurchased, 4),
+ BatteryChargedKwh = Math.Round(raw.BatteryChargeEnergy, 4),
+ BatteryDischargedKwh = Math.Round(raw.BatteryDischargeEnergy, 4),
+ LoadKwh = Math.Round(raw.LoadPowerConsumption, 4),
+ });
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"[AggregatedJsonParser] Skipping hourly line: {ex.Message}");
+ }
+ }
+
+ Console.WriteLine($"[AggregatedJsonParser] Parsed {result.Count} hourly record(s)");
+ return result;
+ }
+
+ ///
+ /// Converts ISO date "yyyy-MM-dd" to device filename format "ddMMyyyy".
+ ///
+ public static String ToJsonFileName(String isoDate)
+ {
+ var d = DateOnly.ParseExact(isoDate, "yyyy-MM-dd");
+ return d.ToString("ddMMyyyy") + ".json";
+ }
+
+ public static String ToJsonFileName(DateOnly date) => date.ToString("ddMMyyyy") + ".json";
+
+ ///
+ /// Tries to read an aggregated JSON file from the installation's S3 bucket.
+ /// S3 key: DDMMYYYY.json (directly in bucket root).
+ /// Returns file content or null if not found / error.
+ ///
+ public static async Task TryReadFromS3(Installation installation, String isoDate)
+ {
+ try
+ {
+ var fileName = ToJsonFileName(isoDate);
+ var region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!);
+ var bucket = region.Bucket(installation.BucketName());
+ var s3Url = bucket.Path(fileName);
+
+ return await s3Url.GetObjectAsString();
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"[AggregatedJsonParser] S3 read failed for {isoDate}: {ex.Message}");
+ return null;
+ }
+ }
+
+ // --- JSON DTOs ---
+
+ private sealed class HourlyJsonDto
+ {
+ public String Type { get; set; } = "";
+ public DateTime Timestamp { get; set; }
+ public Double SelfGeneratedElectricity { get; set; }
+ public Double ElectricityPurchased { get; set; }
+ public Double ElectricityFed { get; set; }
+ public Double BatteryChargeEnergy { get; set; }
+ public Double BatteryDischargeEnergy { get; set; }
+ public Double LoadPowerConsumption { get; set; }
+ }
+
+ private sealed class DailyJsonDto
+ {
+ public String Type { get; set; } = "";
+ public DateTime Timestamp { get; set; }
+ public Double DailySelfGeneratedElectricity { get; set; }
+ public Double DailyElectricityPurchased { get; set; }
+ public Double DailyElectricityFed { get; set; }
+ public Double BatteryDailyChargeEnergy { get; set; }
+ public Double BatteryDailyDischargeEnergy { get; set; }
+ public Double DailyLoadPowerConsumption { get; set; }
+ }
+}
diff --git a/csharp/App/Backend/Services/DailyIngestionService.cs b/csharp/App/Backend/Services/DailyIngestionService.cs
index b7e5a3798..f82e46c56 100644
--- a/csharp/App/Backend/Services/DailyIngestionService.cs
+++ b/csharp/App/Backend/Services/DailyIngestionService.cs
@@ -4,12 +4,8 @@ using InnovEnergy.App.Backend.DataTypes;
namespace InnovEnergy.App.Backend.Services;
///
-/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table.
-/// This is the source-of-truth population step for the report pipeline.
-///
-/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx
-/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor)
-///
+/// Ingests daily energy totals into the DailyEnergyRecord SQLite table.
+/// Data source priority: JSON (local) → JSON (S3) → xlsx fallback.
/// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the
/// IngestDailyData API endpoint.
///
@@ -18,6 +14,9 @@ public static class DailyIngestionService
private static readonly String TmpReportDir =
Environment.CurrentDirectory + "/tmp_report/";
+ private static readonly String JsonAggregatedDir =
+ Environment.CurrentDirectory + "/tmp_report/aggregated/";
+
private static Timer? _dailyTimer;
///
@@ -73,11 +72,119 @@ public static class DailyIngestionService
}
///
- /// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores
- /// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per
- /// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent.
+ /// Ingests data for one installation. Tries JSON (local + S3) and xlsx.
+ /// Both sources are tried — idempotency checks prevent duplicates.
+ /// JSON provides recent data; xlsx provides historical data.
///
public static async Task IngestInstallationAsync(Int64 installationId)
+ {
+ await TryIngestFromJson(installationId);
+ IngestFromXlsx(installationId);
+ }
+
+ private static async Task TryIngestFromJson(Int64 installationId)
+ {
+ var newDaily = 0;
+ var newHourly = 0;
+ var jsonDir = Path.Combine(JsonAggregatedDir, installationId.ToString());
+
+ // Collect JSON content from local files
+ var jsonFiles = Directory.Exists(jsonDir)
+ ? Directory.GetFiles(jsonDir, "*.json")
+ : Array.Empty();
+
+ foreach (var jsonPath in jsonFiles.OrderBy(f => f))
+ {
+ var content = File.ReadAllText(jsonPath);
+ var (d, h) = IngestJsonContent(installationId, content);
+ newDaily += d;
+ newHourly += h;
+ }
+
+ // Also try S3 for recent days (yesterday + today) if no local files found
+ if (jsonFiles.Length == 0)
+ {
+ var installation = Db.GetInstallationById(installationId);
+ if (installation is not null)
+ {
+ for (var daysBack = 0; daysBack <= 1; daysBack++)
+ {
+ var date = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-daysBack));
+ var isoDate = date.ToString("yyyy-MM-dd");
+
+ if (Db.DailyRecordExists(installationId, isoDate))
+ continue;
+
+ var content = await AggregatedJsonParser.TryReadFromS3(installation, isoDate);
+ if (content is null) continue;
+
+ var (d, h) = IngestJsonContent(installationId, content);
+ newDaily += d;
+ newHourly += h;
+ }
+ }
+ }
+
+ if (newDaily > 0 || newHourly > 0)
+ Console.WriteLine($"[DailyIngestion] Installation {installationId} (JSON): {newDaily} day(s), {newHourly} hour(s) ingested.");
+
+ return newDaily > 0 || newHourly > 0;
+ }
+
+ public static (Int32 daily, Int32 hourly) IngestJsonContent(Int64 installationId, String content)
+ {
+ var newDaily = 0;
+ var newHourly = 0;
+
+ foreach (var day in AggregatedJsonParser.ParseDaily(content))
+ {
+ if (Db.DailyRecordExists(installationId, day.Date))
+ continue;
+
+ Db.Create(new DailyEnergyRecord
+ {
+ InstallationId = installationId,
+ Date = day.Date,
+ PvProduction = day.PvProduction,
+ LoadConsumption = day.LoadConsumption,
+ GridImport = day.GridImport,
+ GridExport = day.GridExport,
+ BatteryCharged = day.BatteryCharged,
+ BatteryDischarged = day.BatteryDischarged,
+ CreatedAt = DateTime.UtcNow.ToString("o"),
+ });
+ newDaily++;
+ }
+
+ foreach (var hour in AggregatedJsonParser.ParseHourly(content))
+ {
+ var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
+ if (Db.HourlyRecordExists(installationId, dateHour))
+ continue;
+
+ Db.Create(new HourlyEnergyRecord
+ {
+ InstallationId = installationId,
+ Date = hour.DateTime.ToString("yyyy-MM-dd"),
+ Hour = hour.Hour,
+ DateHour = dateHour,
+ DayOfWeek = hour.DayOfWeek,
+ IsWeekend = hour.IsWeekend,
+ PvKwh = hour.PvKwh,
+ LoadKwh = hour.LoadKwh,
+ GridImportKwh = hour.GridImportKwh,
+ BatteryChargedKwh = hour.BatteryChargedKwh,
+ BatteryDischargedKwh = hour.BatteryDischargedKwh,
+ BattSoC = 0,
+ CreatedAt = DateTime.UtcNow.ToString("o"),
+ });
+ newHourly++;
+ }
+
+ return (newDaily, newHourly);
+ }
+
+ private static void IngestFromXlsx(Int64 installationId)
{
if (!Directory.Exists(TmpReportDir))
{
@@ -98,12 +205,8 @@ public static class DailyIngestionService
foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
{
- // Ingest daily records
List days;
- try
- {
- days = ExcelDataParser.Parse(xlsxPath);
- }
+ try { days = ExcelDataParser.Parse(xlsxPath); }
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
@@ -132,12 +235,8 @@ public static class DailyIngestionService
newDailyCount++;
}
- // Ingest hourly records
List hours;
- try
- {
- hours = ExcelDataParser.ParseHourly(xlsxPath);
- }
+ try { hours = ExcelDataParser.ParseHourly(xlsxPath); }
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
@@ -170,7 +269,6 @@ public static class DailyIngestionService
}
}
- Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
- await Task.CompletedTask;
+ Console.WriteLine($"[DailyIngestion] Installation {installationId} (xlsx): {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
}
}