From 1306ae6b9f9f380e1f8a8a809b2cfa1da6e526ba Mon Sep 17 00:00:00 2001 From: Yinyin Liu Date: Wed, 11 Mar 2026 11:43:55 +0100 Subject: [PATCH] digested hourly and daily data from S3 for Sinexcel --- csharp/App/Backend/Controller.cs | 158 ++++++++++------- .../Backend/Services/AggregatedJsonParser.cs | 162 ++++++++++++++++++ .../Backend/Services/DailyIngestionService.cs | 140 ++++++++++++--- 3 files changed, 377 insertions(+), 83 deletions(-) create mode 100644 csharp/App/Backend/Services/AggregatedJsonParser.cs diff --git a/csharp/App/Backend/Controller.cs b/csharp/App/Backend/Controller.cs index 435857e9e..bf7ab8c6b 100644 --- a/csharp/App/Backend/Controller.cs +++ b/csharp/App/Backend/Controller.cs @@ -1257,7 +1257,7 @@ public class Controller : ControllerBase /// /// Returns daily + hourly records for a date range. - /// DB first; if empty, falls back to xlsx parsing and caches results for future calls. + /// Fallback chain: DB → JSON (local + S3) → xlsx. Caches to DB on first parse. /// [HttpGet(nameof(GetDailyDetailRecords))] public ActionResult GetDailyDetailRecords( @@ -1279,73 +1279,107 @@ public class Controller : ControllerBase var dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate); var hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate); - // 2. Fallback: parse xlsx + cache to DB - if (dailyRecords.Count == 0 || hourlyRecords.Count == 0) + if (dailyRecords.Count > 0 && hourlyRecords.Count > 0) + return Ok(FormatResult(dailyRecords, hourlyRecords)); + + // 2. Fallback: try JSON (local files + S3) + TryIngestFromJson(installationId, installation, fromDate, toDate); + dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate); + hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate); + + if (dailyRecords.Count > 0 && hourlyRecords.Count > 0) + return Ok(FormatResult(dailyRecords, hourlyRecords)); + + // 3. Fallback: parse xlsx + cache to DB + TryIngestFromXlsx(installationId, fromDate, toDate); + dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate); + hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate); + + return Ok(FormatResult(dailyRecords, hourlyRecords)); + } + + private static Object FormatResult( + List daily, List hourly) => new + { + dailyRecords = new { count = daily.Count, records = daily }, + hourlyRecords = new { count = hourly.Count, records = hourly }, + }; + + private static void TryIngestFromJson( + Int64 installationId, Installation installation, + DateOnly fromDate, DateOnly toDate) + { + var jsonDir = Path.Combine( + Environment.CurrentDirectory, "tmp_report", "aggregated", installationId.ToString()); + + for (var date = fromDate; date <= toDate; date = date.AddDays(1)) { - var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate); - if (xlsxFiles.Count > 0) + var isoDate = date.ToString("yyyy-MM-dd"); + var fileName = AggregatedJsonParser.ToJsonFileName(date); + + // Try local file first + var localPath = Path.Combine(jsonDir, fileName); + String? content = System.IO.File.Exists(localPath) ? System.IO.File.ReadAllText(localPath) : null; + + // Try S3 if no local file + content ??= AggregatedJsonParser.TryReadFromS3(installation, isoDate) + .GetAwaiter().GetResult(); + + if (content is null) continue; + + DailyIngestionService.IngestJsonContent(installationId, content); + } + } + + private static void TryIngestFromXlsx( + Int64 installationId, DateOnly fromDate, DateOnly toDate) + { + var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate); + if (xlsxFiles.Count == 0) return; + + foreach (var xlsxPath in xlsxFiles) + { + foreach (var day in ExcelDataParser.Parse(xlsxPath)) { - foreach (var xlsxPath in xlsxFiles) + if (Db.DailyRecordExists(installationId, day.Date)) + continue; + Db.Create(new DailyEnergyRecord { - if (dailyRecords.Count == 0) - { - foreach (var day in ExcelDataParser.Parse(xlsxPath)) - { - if (Db.DailyRecordExists(installationId, day.Date)) - continue; - Db.Create(new DailyEnergyRecord - { - InstallationId = installationId, - Date = day.Date, - PvProduction = day.PvProduction, - LoadConsumption = day.LoadConsumption, - GridImport = day.GridImport, - GridExport = day.GridExport, - BatteryCharged = day.BatteryCharged, - BatteryDischarged = day.BatteryDischarged, - CreatedAt = DateTime.UtcNow.ToString("o"), - }); - } - } + InstallationId = installationId, + Date = day.Date, + PvProduction = day.PvProduction, + LoadConsumption = day.LoadConsumption, + GridImport = day.GridImport, + GridExport = day.GridExport, + BatteryCharged = day.BatteryCharged, + BatteryDischarged = day.BatteryDischarged, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + } - if (hourlyRecords.Count == 0) - { - foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath)) - { - var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}"; - if (Db.HourlyRecordExists(installationId, dateHour)) - continue; - Db.Create(new HourlyEnergyRecord - { - InstallationId = installationId, - Date = hour.DateTime.ToString("yyyy-MM-dd"), - Hour = hour.Hour, - DateHour = dateHour, - DayOfWeek = hour.DayOfWeek, - IsWeekend = hour.IsWeekend, - PvKwh = hour.PvKwh, - LoadKwh = hour.LoadKwh, - GridImportKwh = hour.GridImportKwh, - BatteryChargedKwh = hour.BatteryChargedKwh, - BatteryDischargedKwh = hour.BatteryDischargedKwh, - BattSoC = hour.BattSoC, - CreatedAt = DateTime.UtcNow.ToString("o"), - }); - } - } - } - - // Re-read from DB (now cached) - dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate); - hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate); + foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath)) + { + var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}"; + if (Db.HourlyRecordExists(installationId, dateHour)) + continue; + Db.Create(new HourlyEnergyRecord + { + InstallationId = installationId, + Date = hour.DateTime.ToString("yyyy-MM-dd"), + Hour = hour.Hour, + DateHour = dateHour, + DayOfWeek = hour.DayOfWeek, + IsWeekend = hour.IsWeekend, + PvKwh = hour.PvKwh, + LoadKwh = hour.LoadKwh, + GridImportKwh = hour.GridImportKwh, + BatteryChargedKwh = hour.BatteryChargedKwh, + BatteryDischargedKwh = hour.BatteryDischargedKwh, + BattSoC = hour.BattSoC, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); } } - - return Ok(new - { - dailyRecords = new { count = dailyRecords.Count, records = dailyRecords }, - hourlyRecords = new { count = hourlyRecords.Count, records = hourlyRecords }, - }); } /// diff --git a/csharp/App/Backend/Services/AggregatedJsonParser.cs b/csharp/App/Backend/Services/AggregatedJsonParser.cs new file mode 100644 index 000000000..bcc613c8d --- /dev/null +++ b/csharp/App/Backend/Services/AggregatedJsonParser.cs @@ -0,0 +1,162 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using InnovEnergy.App.Backend.DataTypes; +using InnovEnergy.App.Backend.DataTypes.Methods; +using InnovEnergy.Lib.S3Utils; +using InnovEnergy.Lib.S3Utils.DataTypes; +using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region; + +namespace InnovEnergy.App.Backend.Services; + +/// +/// Parses NDJSON aggregated data files generated by SodistoreHome devices. +/// Each file (DDMMYYYY.json) contains one JSON object per line: +/// - Type "Hourly": per-hour kWh values (already computed, no diffing needed) +/// - Type "Daily": daily totals +/// +public static class AggregatedJsonParser +{ + private static readonly JsonSerializerOptions JsonOpts = new() + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + }; + + public static List ParseDaily(String ndjsonContent) + { + var dailyByDate = new SortedDictionary(); + + foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + if (!line.Contains("\"Type\":\"Daily\"")) + continue; + + try + { + var raw = JsonSerializer.Deserialize(line, JsonOpts); + if (raw is null) continue; + + var date = raw.Timestamp.ToString("yyyy-MM-dd"); + + dailyByDate[date] = new DailyEnergyData + { + Date = date, + PvProduction = Math.Round(raw.DailySelfGeneratedElectricity, 4), + GridImport = Math.Round(raw.DailyElectricityPurchased, 4), + GridExport = Math.Round(raw.DailyElectricityFed, 4), + BatteryCharged = Math.Round(raw.BatteryDailyChargeEnergy, 4), + BatteryDischarged = Math.Round(raw.BatteryDailyDischargeEnergy, 4), + LoadConsumption = Math.Round(raw.DailyLoadPowerConsumption, 4), + }; + } + catch (Exception ex) + { + Console.Error.WriteLine($"[AggregatedJsonParser] Skipping daily line: {ex.Message}"); + } + } + + Console.WriteLine($"[AggregatedJsonParser] Parsed {dailyByDate.Count} daily record(s)"); + return dailyByDate.Values.ToList(); + } + + public static List ParseHourly(String ndjsonContent) + { + var result = new List(); + + foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + if (!line.Contains("\"Type\":\"Hourly\"")) + continue; + + try + { + var raw = JsonSerializer.Deserialize(line, JsonOpts); + if (raw is null) continue; + + var dt = new DateTime( + raw.Timestamp.Year, raw.Timestamp.Month, raw.Timestamp.Day, + raw.Timestamp.Hour, 0, 0); + + result.Add(new HourlyEnergyData + { + DateTime = dt, + Hour = dt.Hour, + DayOfWeek = dt.DayOfWeek.ToString(), + IsWeekend = dt.DayOfWeek is System.DayOfWeek.Saturday or System.DayOfWeek.Sunday, + PvKwh = Math.Round(raw.SelfGeneratedElectricity, 4), + GridImportKwh = Math.Round(raw.ElectricityPurchased, 4), + BatteryChargedKwh = Math.Round(raw.BatteryChargeEnergy, 4), + BatteryDischargedKwh = Math.Round(raw.BatteryDischargeEnergy, 4), + LoadKwh = Math.Round(raw.LoadPowerConsumption, 4), + }); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[AggregatedJsonParser] Skipping hourly line: {ex.Message}"); + } + } + + Console.WriteLine($"[AggregatedJsonParser] Parsed {result.Count} hourly record(s)"); + return result; + } + + /// + /// Converts ISO date "yyyy-MM-dd" to device filename format "ddMMyyyy". + /// + public static String ToJsonFileName(String isoDate) + { + var d = DateOnly.ParseExact(isoDate, "yyyy-MM-dd"); + return d.ToString("ddMMyyyy") + ".json"; + } + + public static String ToJsonFileName(DateOnly date) => date.ToString("ddMMyyyy") + ".json"; + + /// + /// Tries to read an aggregated JSON file from the installation's S3 bucket. + /// S3 key: DDMMYYYY.json (directly in bucket root). + /// Returns file content or null if not found / error. + /// + public static async Task TryReadFromS3(Installation installation, String isoDate) + { + try + { + var fileName = ToJsonFileName(isoDate); + var region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!); + var bucket = region.Bucket(installation.BucketName()); + var s3Url = bucket.Path(fileName); + + return await s3Url.GetObjectAsString(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[AggregatedJsonParser] S3 read failed for {isoDate}: {ex.Message}"); + return null; + } + } + + // --- JSON DTOs --- + + private sealed class HourlyJsonDto + { + public String Type { get; set; } = ""; + public DateTime Timestamp { get; set; } + public Double SelfGeneratedElectricity { get; set; } + public Double ElectricityPurchased { get; set; } + public Double ElectricityFed { get; set; } + public Double BatteryChargeEnergy { get; set; } + public Double BatteryDischargeEnergy { get; set; } + public Double LoadPowerConsumption { get; set; } + } + + private sealed class DailyJsonDto + { + public String Type { get; set; } = ""; + public DateTime Timestamp { get; set; } + public Double DailySelfGeneratedElectricity { get; set; } + public Double DailyElectricityPurchased { get; set; } + public Double DailyElectricityFed { get; set; } + public Double BatteryDailyChargeEnergy { get; set; } + public Double BatteryDailyDischargeEnergy { get; set; } + public Double DailyLoadPowerConsumption { get; set; } + } +} diff --git a/csharp/App/Backend/Services/DailyIngestionService.cs b/csharp/App/Backend/Services/DailyIngestionService.cs index b7e5a3798..f82e46c56 100644 --- a/csharp/App/Backend/Services/DailyIngestionService.cs +++ b/csharp/App/Backend/Services/DailyIngestionService.cs @@ -4,12 +4,8 @@ using InnovEnergy.App.Backend.DataTypes; namespace InnovEnergy.App.Backend.Services; /// -/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table. -/// This is the source-of-truth population step for the report pipeline. -/// -/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx -/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor) -/// +/// Ingests daily energy totals into the DailyEnergyRecord SQLite table. +/// Data source priority: JSON (local) → JSON (S3) → xlsx fallback. /// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the /// IngestDailyData API endpoint. /// @@ -18,6 +14,9 @@ public static class DailyIngestionService private static readonly String TmpReportDir = Environment.CurrentDirectory + "/tmp_report/"; + private static readonly String JsonAggregatedDir = + Environment.CurrentDirectory + "/tmp_report/aggregated/"; + private static Timer? _dailyTimer; /// @@ -73,11 +72,119 @@ public static class DailyIngestionService } /// - /// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores - /// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per - /// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent. + /// Ingests data for one installation. Tries JSON (local + S3) and xlsx. + /// Both sources are tried — idempotency checks prevent duplicates. + /// JSON provides recent data; xlsx provides historical data. /// public static async Task IngestInstallationAsync(Int64 installationId) + { + await TryIngestFromJson(installationId); + IngestFromXlsx(installationId); + } + + private static async Task TryIngestFromJson(Int64 installationId) + { + var newDaily = 0; + var newHourly = 0; + var jsonDir = Path.Combine(JsonAggregatedDir, installationId.ToString()); + + // Collect JSON content from local files + var jsonFiles = Directory.Exists(jsonDir) + ? Directory.GetFiles(jsonDir, "*.json") + : Array.Empty(); + + foreach (var jsonPath in jsonFiles.OrderBy(f => f)) + { + var content = File.ReadAllText(jsonPath); + var (d, h) = IngestJsonContent(installationId, content); + newDaily += d; + newHourly += h; + } + + // Also try S3 for recent days (yesterday + today) if no local files found + if (jsonFiles.Length == 0) + { + var installation = Db.GetInstallationById(installationId); + if (installation is not null) + { + for (var daysBack = 0; daysBack <= 1; daysBack++) + { + var date = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-daysBack)); + var isoDate = date.ToString("yyyy-MM-dd"); + + if (Db.DailyRecordExists(installationId, isoDate)) + continue; + + var content = await AggregatedJsonParser.TryReadFromS3(installation, isoDate); + if (content is null) continue; + + var (d, h) = IngestJsonContent(installationId, content); + newDaily += d; + newHourly += h; + } + } + } + + if (newDaily > 0 || newHourly > 0) + Console.WriteLine($"[DailyIngestion] Installation {installationId} (JSON): {newDaily} day(s), {newHourly} hour(s) ingested."); + + return newDaily > 0 || newHourly > 0; + } + + public static (Int32 daily, Int32 hourly) IngestJsonContent(Int64 installationId, String content) + { + var newDaily = 0; + var newHourly = 0; + + foreach (var day in AggregatedJsonParser.ParseDaily(content)) + { + if (Db.DailyRecordExists(installationId, day.Date)) + continue; + + Db.Create(new DailyEnergyRecord + { + InstallationId = installationId, + Date = day.Date, + PvProduction = day.PvProduction, + LoadConsumption = day.LoadConsumption, + GridImport = day.GridImport, + GridExport = day.GridExport, + BatteryCharged = day.BatteryCharged, + BatteryDischarged = day.BatteryDischarged, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + newDaily++; + } + + foreach (var hour in AggregatedJsonParser.ParseHourly(content)) + { + var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}"; + if (Db.HourlyRecordExists(installationId, dateHour)) + continue; + + Db.Create(new HourlyEnergyRecord + { + InstallationId = installationId, + Date = hour.DateTime.ToString("yyyy-MM-dd"), + Hour = hour.Hour, + DateHour = dateHour, + DayOfWeek = hour.DayOfWeek, + IsWeekend = hour.IsWeekend, + PvKwh = hour.PvKwh, + LoadKwh = hour.LoadKwh, + GridImportKwh = hour.GridImportKwh, + BatteryChargedKwh = hour.BatteryChargedKwh, + BatteryDischargedKwh = hour.BatteryDischargedKwh, + BattSoC = 0, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + newHourly++; + } + + return (newDaily, newHourly); + } + + private static void IngestFromXlsx(Int64 installationId) { if (!Directory.Exists(TmpReportDir)) { @@ -98,12 +205,8 @@ public static class DailyIngestionService foreach (var xlsxPath in xlsxFiles.OrderBy(f => f)) { - // Ingest daily records List days; - try - { - days = ExcelDataParser.Parse(xlsxPath); - } + try { days = ExcelDataParser.Parse(xlsxPath); } catch (Exception ex) { Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}"); @@ -132,12 +235,8 @@ public static class DailyIngestionService newDailyCount++; } - // Ingest hourly records List hours; - try - { - hours = ExcelDataParser.ParseHourly(xlsxPath); - } + try { hours = ExcelDataParser.ParseHourly(xlsxPath); } catch (Exception ex) { Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}"); @@ -170,7 +269,6 @@ public static class DailyIngestionService } } - Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s))."); - await Task.CompletedTask; + Console.WriteLine($"[DailyIngestion] Installation {installationId} (xlsx): {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s))."); } }