digested hourly and daily data from S3 for Sinexcel

csharp/App/Backend/Controller.cs

@@ -1257,7 +1257,7 @@ public class Controller : ControllerBase
 
     /// <summary>
     /// Returns daily + hourly records for a date range.
-    /// DB first; if empty, falls back to xlsx parsing and caches results for future calls.
+    /// Fallback chain: DB → JSON (local + S3) → xlsx. Caches to DB on first parse.
     /// </summary>
     [HttpGet(nameof(GetDailyDetailRecords))]
     public ActionResult GetDailyDetailRecords(
@@ -1279,15 +1279,65 @@ public class Controller : ControllerBase
         var dailyRecords  = Db.GetDailyRecords(installationId, fromDate, toDate);
         var hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
 
-        // 2. Fallback: parse xlsx + cache to DB
+        if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
-        if (dailyRecords.Count == 0 || hourlyRecords.Count == 0)
+            return Ok(FormatResult(dailyRecords, hourlyRecords));
+
+        // 2. Fallback: try JSON (local files + S3)
+        TryIngestFromJson(installationId, installation, fromDate, toDate);
+        dailyRecords  = Db.GetDailyRecords(installationId, fromDate, toDate);
+        hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
+
+        if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
+            return Ok(FormatResult(dailyRecords, hourlyRecords));
+
+        // 3. Fallback: parse xlsx + cache to DB
+        TryIngestFromXlsx(installationId, fromDate, toDate);
+        dailyRecords  = Db.GetDailyRecords(installationId, fromDate, toDate);
+        hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
+
+        return Ok(FormatResult(dailyRecords, hourlyRecords));
+    }
+
+    private static Object FormatResult(
+        List<DailyEnergyRecord> daily, List<HourlyEnergyRecord> hourly) => new
+    {
+        dailyRecords  = new { count = daily.Count,  records = daily },
+        hourlyRecords = new { count = hourly.Count, records = hourly },
+    };
+
+    private static void TryIngestFromJson(
+        Int64 installationId, Installation installation,
+        DateOnly fromDate, DateOnly toDate)
+    {
+        var jsonDir = Path.Combine(
+            Environment.CurrentDirectory, "tmp_report", "aggregated", installationId.ToString());
+
+        for (var date = fromDate; date <= toDate; date = date.AddDays(1))
+        {
+            var isoDate  = date.ToString("yyyy-MM-dd");
+            var fileName = AggregatedJsonParser.ToJsonFileName(date);
+
+            // Try local file first
+            var localPath = Path.Combine(jsonDir, fileName);
+            String? content = System.IO.File.Exists(localPath) ? System.IO.File.ReadAllText(localPath) : null;
+
+            // Try S3 if no local file
+            content ??= AggregatedJsonParser.TryReadFromS3(installation, isoDate)
+                .GetAwaiter().GetResult();
+
+            if (content is null) continue;
+
+            DailyIngestionService.IngestJsonContent(installationId, content);
+        }
+    }
+
+    private static void TryIngestFromXlsx(
+        Int64 installationId, DateOnly fromDate, DateOnly toDate)
     {
         var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate);
-            if (xlsxFiles.Count > 0)
+        if (xlsxFiles.Count == 0) return;
-            {
+
         foreach (var xlsxPath in xlsxFiles)
-                {
-                    if (dailyRecords.Count == 0)
         {
             foreach (var day in ExcelDataParser.Parse(xlsxPath))
             {
@@ -1306,10 +1356,7 @@ public class Controller : ControllerBase
                     CreatedAt         = DateTime.UtcNow.ToString("o"),
                 });
             }
-                    }
 
-                    if (hourlyRecords.Count == 0)
-                    {
             foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath))
             {
                 var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
@@ -1335,19 +1382,6 @@ public class Controller : ControllerBase
         }
     }
 
-                // Re-read from DB (now cached)
-                dailyRecords  = Db.GetDailyRecords(installationId, fromDate, toDate);
-                hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
-            }
-        }
-
-        return Ok(new
-        {
-            dailyRecords  = new { count = dailyRecords.Count, records = dailyRecords },
-            hourlyRecords = new { count = hourlyRecords.Count, records = hourlyRecords },
-        });
-    }
-
     /// <summary>
     /// Deletes DailyEnergyRecord rows for an installation in the given date range.
     /// Safe to use during testing — only removes daily records, not report summaries.

csharp/App/Backend/Services/AggregatedJsonParser.cs

@@ -0,0 +1,162 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using InnovEnergy.App.Backend.DataTypes;
+using InnovEnergy.App.Backend.DataTypes.Methods;
+using InnovEnergy.Lib.S3Utils;
+using InnovEnergy.Lib.S3Utils.DataTypes;
+using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
+
+namespace InnovEnergy.App.Backend.Services;
+
+/// <summary>
+/// Parses NDJSON aggregated data files generated by SodistoreHome devices.
+/// Each file (DDMMYYYY.json) contains one JSON object per line:
+///   - Type "Hourly": per-hour kWh values (already computed, no diffing needed)
+///   - Type "Daily": daily totals
+/// </summary>
+public static class AggregatedJsonParser
+{
+    private static readonly JsonSerializerOptions JsonOpts = new()
+    {
+        PropertyNameCaseInsensitive = true,
+        NumberHandling = JsonNumberHandling.AllowReadingFromString,
+    };
+
+    public static List<DailyEnergyData> ParseDaily(String ndjsonContent)
+    {
+        var dailyByDate = new SortedDictionary<String, DailyEnergyData>();
+
+        foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+        {
+            if (!line.Contains("\"Type\":\"Daily\""))
+                continue;
+
+            try
+            {
+                var raw = JsonSerializer.Deserialize<DailyJsonDto>(line, JsonOpts);
+                if (raw is null) continue;
+
+                var date = raw.Timestamp.ToString("yyyy-MM-dd");
+
+                dailyByDate[date] = new DailyEnergyData
+                {
+                    Date              = date,
+                    PvProduction      = Math.Round(raw.DailySelfGeneratedElectricity, 4),
+                    GridImport        = Math.Round(raw.DailyElectricityPurchased, 4),
+                    GridExport        = Math.Round(raw.DailyElectricityFed, 4),
+                    BatteryCharged    = Math.Round(raw.BatteryDailyChargeEnergy, 4),
+                    BatteryDischarged = Math.Round(raw.BatteryDailyDischargeEnergy, 4),
+                    LoadConsumption   = Math.Round(raw.DailyLoadPowerConsumption, 4),
+                };
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine($"[AggregatedJsonParser] Skipping daily line: {ex.Message}");
+            }
+        }
+
+        Console.WriteLine($"[AggregatedJsonParser] Parsed {dailyByDate.Count} daily record(s)");
+        return dailyByDate.Values.ToList();
+    }
+
+    public static List<HourlyEnergyData> ParseHourly(String ndjsonContent)
+    {
+        var result = new List<HourlyEnergyData>();
+
+        foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
+        {
+            if (!line.Contains("\"Type\":\"Hourly\""))
+                continue;
+
+            try
+            {
+                var raw = JsonSerializer.Deserialize<HourlyJsonDto>(line, JsonOpts);
+                if (raw is null) continue;
+
+                var dt = new DateTime(
+                    raw.Timestamp.Year, raw.Timestamp.Month, raw.Timestamp.Day,
+                    raw.Timestamp.Hour, 0, 0);
+
+                result.Add(new HourlyEnergyData
+                {
+                    DateTime             = dt,
+                    Hour                 = dt.Hour,
+                    DayOfWeek            = dt.DayOfWeek.ToString(),
+                    IsWeekend            = dt.DayOfWeek is System.DayOfWeek.Saturday or System.DayOfWeek.Sunday,
+                    PvKwh                = Math.Round(raw.SelfGeneratedElectricity, 4),
+                    GridImportKwh        = Math.Round(raw.ElectricityPurchased, 4),
+                    BatteryChargedKwh    = Math.Round(raw.BatteryChargeEnergy, 4),
+                    BatteryDischargedKwh = Math.Round(raw.BatteryDischargeEnergy, 4),
+                    LoadKwh              = Math.Round(raw.LoadPowerConsumption, 4),
+                });
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine($"[AggregatedJsonParser] Skipping hourly line: {ex.Message}");
+            }
+        }
+
+        Console.WriteLine($"[AggregatedJsonParser] Parsed {result.Count} hourly record(s)");
+        return result;
+    }
+
+    /// <summary>
+    /// Converts ISO date "yyyy-MM-dd" to device filename format "ddMMyyyy".
+    /// </summary>
+    public static String ToJsonFileName(String isoDate)
+    {
+        var d = DateOnly.ParseExact(isoDate, "yyyy-MM-dd");
+        return d.ToString("ddMMyyyy") + ".json";
+    }
+
+    public static String ToJsonFileName(DateOnly date) => date.ToString("ddMMyyyy") + ".json";
+
+    /// <summary>
+    /// Tries to read an aggregated JSON file from the installation's S3 bucket.
+    /// S3 key: DDMMYYYY.json (directly in bucket root).
+    /// Returns file content or null if not found / error.
+    /// </summary>
+    public static async Task<String?> TryReadFromS3(Installation installation, String isoDate)
+    {
+        try
+        {
+            var fileName = ToJsonFileName(isoDate);
+            var region   = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!);
+            var bucket   = region.Bucket(installation.BucketName());
+            var s3Url    = bucket.Path(fileName);
+
+            return await s3Url.GetObjectAsString();
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine($"[AggregatedJsonParser] S3 read failed for {isoDate}: {ex.Message}");
+            return null;
+        }
+    }
+
+    // --- JSON DTOs ---
+
+    private sealed class HourlyJsonDto
+    {
+        public String   Type                     { get; set; } = "";
+        public DateTime Timestamp                { get; set; }
+        public Double   SelfGeneratedElectricity { get; set; }
+        public Double   ElectricityPurchased     { get; set; }
+        public Double   ElectricityFed           { get; set; }
+        public Double   BatteryChargeEnergy      { get; set; }
+        public Double   BatteryDischargeEnergy   { get; set; }
+        public Double   LoadPowerConsumption     { get; set; }
+    }
+
+    private sealed class DailyJsonDto
+    {
+        public String   Type                          { get; set; } = "";
+        public DateTime Timestamp                     { get; set; }
+        public Double   DailySelfGeneratedElectricity { get; set; }
+        public Double   DailyElectricityPurchased     { get; set; }
+        public Double   DailyElectricityFed           { get; set; }
+        public Double   BatteryDailyChargeEnergy      { get; set; }
+        public Double   BatteryDailyDischargeEnergy   { get; set; }
+        public Double   DailyLoadPowerConsumption     { get; set; }
+    }
+}

csharp/App/Backend/Services/DailyIngestionService.cs

@@ -4,12 +4,8 @@ using InnovEnergy.App.Backend.DataTypes;
 namespace InnovEnergy.App.Backend.Services;
 
 /// <summary>
-/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table.
+/// Ingests daily energy totals into the DailyEnergyRecord SQLite table.
-/// This is the source-of-truth population step for the report pipeline.
+/// Data source priority: JSON (local) → JSON (S3) → xlsx fallback.
-///
-/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx
-/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor)
-///
 /// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the
 /// IngestDailyData API endpoint.
 /// </summary>
@@ -18,6 +14,9 @@ public static class DailyIngestionService
     private static readonly String TmpReportDir =
         Environment.CurrentDirectory + "/tmp_report/";
 
+    private static readonly String JsonAggregatedDir =
+        Environment.CurrentDirectory + "/tmp_report/aggregated/";
+
     private static Timer? _dailyTimer;
 
     /// <summary>
@@ -73,11 +72,119 @@ public static class DailyIngestionService
     }
 
     /// <summary>
-    /// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores
+    /// Ingests data for one installation. Tries JSON (local + S3) and xlsx.
-    /// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per
+    /// Both sources are tried — idempotency checks prevent duplicates.
-    /// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent.
+    /// JSON provides recent data; xlsx provides historical data.
     /// </summary>
     public static async Task IngestInstallationAsync(Int64 installationId)
+    {
+        await TryIngestFromJson(installationId);
+        IngestFromXlsx(installationId);
+    }
+
+    private static async Task<Boolean> TryIngestFromJson(Int64 installationId)
+    {
+        var newDaily  = 0;
+        var newHourly = 0;
+        var jsonDir   = Path.Combine(JsonAggregatedDir, installationId.ToString());
+
+        // Collect JSON content from local files
+        var jsonFiles = Directory.Exists(jsonDir)
+            ? Directory.GetFiles(jsonDir, "*.json")
+            : Array.Empty<String>();
+
+        foreach (var jsonPath in jsonFiles.OrderBy(f => f))
+        {
+            var content = File.ReadAllText(jsonPath);
+            var (d, h) = IngestJsonContent(installationId, content);
+            newDaily  += d;
+            newHourly += h;
+        }
+
+        // Also try S3 for recent days (yesterday + today) if no local files found
+        if (jsonFiles.Length == 0)
+        {
+            var installation = Db.GetInstallationById(installationId);
+            if (installation is not null)
+            {
+                for (var daysBack = 0; daysBack <= 1; daysBack++)
+                {
+                    var date    = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-daysBack));
+                    var isoDate = date.ToString("yyyy-MM-dd");
+
+                    if (Db.DailyRecordExists(installationId, isoDate))
+                        continue;
+
+                    var content = await AggregatedJsonParser.TryReadFromS3(installation, isoDate);
+                    if (content is null) continue;
+
+                    var (d, h) = IngestJsonContent(installationId, content);
+                    newDaily  += d;
+                    newHourly += h;
+                }
+            }
+        }
+
+        if (newDaily > 0 || newHourly > 0)
+            Console.WriteLine($"[DailyIngestion] Installation {installationId} (JSON): {newDaily} day(s), {newHourly} hour(s) ingested.");
+
+        return newDaily > 0 || newHourly > 0;
+    }
+
+    public static (Int32 daily, Int32 hourly) IngestJsonContent(Int64 installationId, String content)
+    {
+        var newDaily  = 0;
+        var newHourly = 0;
+
+        foreach (var day in AggregatedJsonParser.ParseDaily(content))
+        {
+            if (Db.DailyRecordExists(installationId, day.Date))
+                continue;
+
+            Db.Create(new DailyEnergyRecord
+            {
+                InstallationId    = installationId,
+                Date              = day.Date,
+                PvProduction      = day.PvProduction,
+                LoadConsumption   = day.LoadConsumption,
+                GridImport        = day.GridImport,
+                GridExport        = day.GridExport,
+                BatteryCharged    = day.BatteryCharged,
+                BatteryDischarged = day.BatteryDischarged,
+                CreatedAt         = DateTime.UtcNow.ToString("o"),
+            });
+            newDaily++;
+        }
+
+        foreach (var hour in AggregatedJsonParser.ParseHourly(content))
+        {
+            var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
+            if (Db.HourlyRecordExists(installationId, dateHour))
+                continue;
+
+            Db.Create(new HourlyEnergyRecord
+            {
+                InstallationId       = installationId,
+                Date                 = hour.DateTime.ToString("yyyy-MM-dd"),
+                Hour                 = hour.Hour,
+                DateHour             = dateHour,
+                DayOfWeek            = hour.DayOfWeek,
+                IsWeekend            = hour.IsWeekend,
+                PvKwh                = hour.PvKwh,
+                LoadKwh              = hour.LoadKwh,
+                GridImportKwh        = hour.GridImportKwh,
+                BatteryChargedKwh    = hour.BatteryChargedKwh,
+                BatteryDischargedKwh = hour.BatteryDischargedKwh,
+                BattSoC              = 0,
+                CreatedAt            = DateTime.UtcNow.ToString("o"),
+            });
+            newHourly++;
+        }
+
+        return (newDaily, newHourly);
+    }
+
+    private static void IngestFromXlsx(Int64 installationId)
     {
         if (!Directory.Exists(TmpReportDir))
         {
@@ -98,12 +205,8 @@ public static class DailyIngestionService
 
         foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
         {
-            // Ingest daily records
             List<DailyEnergyData> days;
-            try
+            try { days = ExcelDataParser.Parse(xlsxPath); }
-            {
-                days = ExcelDataParser.Parse(xlsxPath);
-            }
             catch (Exception ex)
             {
                 Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
@@ -132,12 +235,8 @@ public static class DailyIngestionService
                 newDailyCount++;
             }
 
-            // Ingest hourly records
             List<HourlyEnergyData> hours;
-            try
+            try { hours = ExcelDataParser.ParseHourly(xlsxPath); }
-            {
-                hours = ExcelDataParser.ParseHourly(xlsxPath);
-            }
             catch (Exception ex)
             {
                 Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
@@ -170,7 +269,6 @@ public static class DailyIngestionService
             }
         }
 
-        Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
+        Console.WriteLine($"[DailyIngestion] Installation {installationId} (xlsx): {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
-        await Task.CompletedTask;
     }
 }