From 35b64c33186df2877d86fe5538c838b1c61a0719 Mon Sep 17 00:00:00 2001 From: Yinyin Liu Date: Mon, 2 Mar 2026 18:52:36 +0100 Subject: [PATCH] store hourly energy records in SQLite and remove file cache --- .../Backend/DataTypes/HourlyEnergyRecord.cs | 37 ++++++ csharp/App/Backend/Database/Create.cs | 1 + csharp/App/Backend/Database/Db.cs | 4 + csharp/App/Backend/Database/Delete.cs | 12 +- csharp/App/Backend/Database/Read.cs | 25 ++++ .../Backend/Services/DailyIngestionService.cs | 49 +++++++- .../Services/ReportAggregationService.cs | 4 +- .../Backend/Services/WeeklyReportService.cs | 110 ++++++------------ 8 files changed, 162 insertions(+), 80 deletions(-) create mode 100644 csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs diff --git a/csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs b/csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs new file mode 100644 index 000000000..47899212e --- /dev/null +++ b/csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs @@ -0,0 +1,37 @@ +using SQLite; + +namespace InnovEnergy.App.Backend.DataTypes; + +public class HourlyEnergyRecord +{ + [PrimaryKey, AutoIncrement] + public Int64 Id { get; set; } + + [Indexed] + public Int64 InstallationId { get; set; } + + // "YYYY-MM-DD" — used for range queries (same pattern as DailyEnergyRecord) + [Indexed] + public String Date { get; set; } = ""; + + // 0–23 + public Int32 Hour { get; set; } + + // "YYYY-MM-DD HH" — used for idempotency check + public String DateHour { get; set; } = ""; + + public String DayOfWeek { get; set; } = ""; + public Boolean IsWeekend { get; set; } + + // Energy for this hour (kWh) + public Double PvKwh { get; set; } + public Double LoadKwh { get; set; } + public Double GridImportKwh { get; set; } + public Double BatteryChargedKwh { get; set; } + public Double BatteryDischargedKwh { get; set; } + + // Instantaneous state of charge at snapshot time (%) + public Double BattSoC { get; set; } + + public String CreatedAt { get; set; } = ""; +} diff --git a/csharp/App/Backend/Database/Create.cs b/csharp/App/Backend/Database/Create.cs index 051fcfb31..98fee3b32 100644 --- a/csharp/App/Backend/Database/Create.cs +++ b/csharp/App/Backend/Database/Create.cs @@ -72,6 +72,7 @@ public static partial class Db public static Boolean Create(MonthlyReportSummary report) => Insert(report); public static Boolean Create(YearlyReportSummary report) => Insert(report); public static Boolean Create(DailyEnergyRecord record) => Insert(record); + public static Boolean Create(HourlyEnergyRecord record) => Insert(record); public static Boolean Create(AiInsightCache cache) => Insert(cache); public static void HandleAction(UserAction newAction) diff --git a/csharp/App/Backend/Database/Db.cs b/csharp/App/Backend/Database/Db.cs index 3b1233219..69c61b50c 100644 --- a/csharp/App/Backend/Database/Db.cs +++ b/csharp/App/Backend/Database/Db.cs @@ -29,6 +29,7 @@ public static partial class Db public static TableQuery MonthlyReports => Connection.Table(); public static TableQuery YearlyReports => Connection.Table(); public static TableQuery DailyRecords => Connection.Table(); + public static TableQuery HourlyRecords => Connection.Table(); public static TableQuery AiInsightCaches => Connection.Table(); @@ -60,6 +61,7 @@ public static partial class Db Connection.CreateTable(); Connection.CreateTable(); Connection.CreateTable(); + Connection.CreateTable(); Connection.CreateTable(); }); @@ -102,6 +104,8 @@ public static partial class Db fileConnection.CreateTable(); fileConnection.CreateTable(); fileConnection.CreateTable(); + fileConnection.CreateTable(); + fileConnection.CreateTable(); return fileConnection; //return CopyDbToMemory(fileConnection); diff --git a/csharp/App/Backend/Database/Delete.cs b/csharp/App/Backend/Database/Delete.cs index 9e613743f..438fa995c 100644 --- a/csharp/App/Backend/Database/Delete.cs +++ b/csharp/App/Backend/Database/Delete.cs @@ -200,6 +200,16 @@ public static partial class Db foreach (var id in oldDailyIds) DailyRecords.Delete(r => r.Id == id); + // Hourly records older than 3 months (sufficient for pattern detection, 24x more rows than daily) + var hourlyCutoff = DateOnly.FromDateTime(DateTime.UtcNow).AddMonths(-3).ToString("yyyy-MM-dd"); + var oldHourlyIds = HourlyRecords + .ToList() + .Where(r => String.Compare(r.Date, hourlyCutoff, StringComparison.Ordinal) < 0) + .Select(r => r.Id) + .ToList(); + foreach (var id in oldHourlyIds) + HourlyRecords.Delete(r => r.Id == id); + // Weekly summaries older than 1 year var oldWeeklyIds = WeeklyReports .ToList() @@ -231,6 +241,6 @@ public static partial class Db AiInsightCaches.Delete(c => c.Id == id); Backup(); - Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff})."); + Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldHourlyIds.Count} hourly, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff})."); } } \ No newline at end of file diff --git a/csharp/App/Backend/Database/Read.cs b/csharp/App/Backend/Database/Read.cs index a9eb013c3..0dccf3f78 100644 --- a/csharp/App/Backend/Database/Read.cs +++ b/csharp/App/Backend/Database/Read.cs @@ -120,6 +120,31 @@ public static partial class Db => DailyRecords .Any(r => r.InstallationId == installationId && r.Date == date); + // ── HourlyEnergyRecord Queries ───────────────────────────────────── + + /// + /// Returns hourly records for an installation within [from, to] inclusive, ordered by date+hour. + /// + public static List GetHourlyRecords(Int64 installationId, DateOnly from, DateOnly to) + { + var fromStr = from.ToString("yyyy-MM-dd"); + var toStr = to.ToString("yyyy-MM-dd"); + return HourlyRecords + .Where(r => r.InstallationId == installationId) + .ToList() + .Where(r => String.Compare(r.Date, fromStr, StringComparison.Ordinal) >= 0 + && String.Compare(r.Date, toStr, StringComparison.Ordinal) <= 0) + .OrderBy(r => r.Date).ThenBy(r => r.Hour) + .ToList(); + } + + /// + /// Returns true if an hourly record already exists for this installation+dateHour (idempotency check). + /// + public static Boolean HourlyRecordExists(Int64 installationId, String dateHour) + => HourlyRecords + .Any(r => r.InstallationId == installationId && r.DateHour == dateHour); + // ── AiInsightCache Queries ───────────────────────────────────────── /// diff --git a/csharp/App/Backend/Services/DailyIngestionService.cs b/csharp/App/Backend/Services/DailyIngestionService.cs index e8d0f8ae9..b7e5a3798 100644 --- a/csharp/App/Backend/Services/DailyIngestionService.cs +++ b/csharp/App/Backend/Services/DailyIngestionService.cs @@ -92,11 +92,13 @@ public static class DailyIngestionService return; } - var newCount = 0; - var totalParsed = 0; + var newDailyCount = 0; + var newHourlyCount = 0; + var totalParsed = 0; foreach (var xlsxPath in xlsxFiles.OrderBy(f => f)) { + // Ingest daily records List days; try { @@ -104,7 +106,7 @@ public static class DailyIngestionService } catch (Exception ex) { - Console.Error.WriteLine($"[DailyIngestion] Failed to parse {Path.GetFileName(xlsxPath)}: {ex.Message}"); + Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}"); continue; } @@ -127,11 +129,48 @@ public static class DailyIngestionService BatteryDischarged = day.BatteryDischarged, CreatedAt = DateTime.UtcNow.ToString("o"), }); - newCount++; + newDailyCount++; + } + + // Ingest hourly records + List hours; + try + { + hours = ExcelDataParser.ParseHourly(xlsxPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}"); + continue; + } + + foreach (var hour in hours) + { + var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}"; + if (Db.HourlyRecordExists(installationId, dateHour)) + continue; + + Db.Create(new HourlyEnergyRecord + { + InstallationId = installationId, + Date = hour.DateTime.ToString("yyyy-MM-dd"), + Hour = hour.Hour, + DateHour = dateHour, + DayOfWeek = hour.DayOfWeek, + IsWeekend = hour.IsWeekend, + PvKwh = hour.PvKwh, + LoadKwh = hour.LoadKwh, + GridImportKwh = hour.GridImportKwh, + BatteryChargedKwh = hour.BatteryChargedKwh, + BatteryDischargedKwh = hour.BatteryDischargedKwh, + BattSoC = hour.BattSoC, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + newHourlyCount++; } } - Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newCount} new day(s) ingested ({totalParsed} total across {xlsxFiles.Length} file(s))."); + Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s))."); await Task.CompletedTask; } } diff --git a/csharp/App/Backend/Services/ReportAggregationService.cs b/csharp/App/Backend/Services/ReportAggregationService.cs index 3dbc40e62..d94ce7206 100644 --- a/csharp/App/Backend/Services/ReportAggregationService.cs +++ b/csharp/App/Backend/Services/ReportAggregationService.cs @@ -688,13 +688,13 @@ Write a concise monthly performance summary in {langName} (4 bullet points, plai MONTHLY FACTS for {monthName} ({weekCount} days of data): - PV production: {totalPv:F1} kWh - Total consumption: {totalConsump:F1} kWh -- Self-sufficiency: {selfSufficiency:F1}% (powered by solar + battery, not grid) +- Self-sufficiency: {selfSufficiency:F1}% (share of energy covered by solar, without drawing from grid) - Battery: {totalBattChg:F1} kWh charged, {totalBattDis:F1} kWh discharged, efficiency {batteryEff:F1}% - Energy saved: {energySaved:F1} kWh = ~{savingsCHF:F0} CHF saved (at {ElectricityPriceCHF} CHF/kWh) INSTRUCTIONS: 1. Savings: state exactly how much energy and money was saved this month. Positive framing. -2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""X% of your home ran on solar + battery""). Do NOT mention raw grid import kWh. +2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""Your solar system covered X% of your home's energy needs""). Do NOT mention battery here. Do NOT mention raw grid import kWh. 3. Battery: comment on battery utilization and efficiency. If efficiency < 80%, note it may need attention. 4. Tip: one specific actionable suggestion based on the weakest metric: {weakMetric}. If general, suggest the most impactful habit change based on the numbers above. diff --git a/csharp/App/Backend/Services/WeeklyReportService.cs b/csharp/App/Backend/Services/WeeklyReportService.cs index 28a1d2833..0557987cf 100644 --- a/csharp/App/Backend/Services/WeeklyReportService.cs +++ b/csharp/App/Backend/Services/WeeklyReportService.cs @@ -66,55 +66,30 @@ public static class WeeklyReportService (prevMon, prevSun) = PreviousCalendarWeek(); } - // Skip file cache when a specific week is requested (avoid stale or wrong-week hits) - var cachePath = weekStartOverride.HasValue - ? null - : TmpReportDir + $"{installationId}_{language}_{curMon:yyyy-MM-dd}.cache.json"; - - // Use cache if it exists and is less than 6 hours old (skipped in override mode) - if (cachePath != null && File.Exists(cachePath)) - { - var cacheAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(cachePath); - if (cacheAge.TotalHours < 6) - { - try - { - var cached = JsonConvert.DeserializeObject( - await File.ReadAllTextAsync(cachePath)); - if (cached != null) - { - Console.WriteLine($"[WeeklyReportService] Returning cached report for installation {installationId} ({language}), week {curMon:yyyy-MM-dd}."); - return cached; - } - } - catch - { - // Cache corrupt — regenerate - } - } - } - - // 1. Try to load daily records from SQLite for the calendar weeks + // 1. Load daily records from SQLite var currentWeekDays = Db.GetDailyRecords(installationId, curMon, curSun) - .Select(r => ToDailyEnergyData(r)).ToList(); + .Select(ToDailyEnergyData).ToList(); var previousWeekDays = Db.GetDailyRecords(installationId, prevMon, prevSun) - .Select(r => ToDailyEnergyData(r)).ToList(); + .Select(ToDailyEnergyData).ToList(); - // 2. Fallback: if DB empty for current week, parse all xlsx files on the fly - var xlsxFiles = Directory.Exists(TmpReportDir) - ? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList() - : new List(); - - if (currentWeekDays.Count == 0 && xlsxFiles.Count > 0) + // 2. Fallback: if DB empty, parse xlsx on the fly (pre-ingestion scenario) + if (currentWeekDays.Count == 0) { - Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx."); - var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList(); - currentWeekDays = allDaysParsed - .Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; }) - .ToList(); - previousWeekDays = allDaysParsed - .Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; }) - .ToList(); + var xlsxFiles = Directory.Exists(TmpReportDir) + ? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList() + : new List(); + + if (xlsxFiles.Count > 0) + { + Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx."); + var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList(); + currentWeekDays = allDaysParsed + .Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; }) + .ToList(); + previousWeekDays = allDaysParsed + .Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; }) + .ToList(); + } } if (currentWeekDays.Count == 0) @@ -122,36 +97,13 @@ public static class WeeklyReportService $"No energy data available for week {curMon:yyyy-MM-dd}–{curSun:yyyy-MM-dd}. " + "Upload an xlsx file or wait for daily ingestion."); - // 3. Load hourly data from ALL xlsx files for behavioral analysis (current week only). - // Combine all files so any week can find its hourly data regardless of file split. - // Future: replace with S3 hourly fetch. - var allHourly = xlsxFiles - .SelectMany(p => { try { return ExcelDataParser.ParseHourly(p); } catch { return Enumerable.Empty(); } }) - .ToList(); - var curMonDt = curMon.ToDateTime(TimeOnly.MinValue); - var curSunDt = curSun.ToDateTime(TimeOnly.MaxValue); - var currentHourlyData = allHourly - .Where(h => h.DateTime >= curMonDt && h.DateTime <= curSunDt) - .ToList(); + // 3. Load hourly records from SQLite for behavioral analysis + var currentHourlyData = Db.GetHourlyRecords(installationId, curMon, curSun) + .Select(ToHourlyEnergyData).ToList(); - var report = await GenerateReportFromDataAsync( + return await GenerateReportFromDataAsync( currentWeekDays, previousWeekDays, currentHourlyData, installationName, language, curMon, curSun); - - // Write cache (skipped in override mode) - if (cachePath != null) - { - try - { - await File.WriteAllTextAsync(cachePath, JsonConvert.SerializeObject(report)); - } - catch (Exception ex) - { - Console.Error.WriteLine($"[WeeklyReportService] Could not write cache: {ex.Message}"); - } - } - - return report; } // ── Conversion helpers ───────────────────────────────────────────── @@ -167,6 +119,20 @@ public static class WeeklyReportService BatteryDischarged = r.BatteryDischarged, }; + private static HourlyEnergyData ToHourlyEnergyData(HourlyEnergyRecord r) => new() + { + DateTime = DateTime.ParseExact(r.DateHour, "yyyy-MM-dd HH", null), + Hour = r.Hour, + DayOfWeek = r.DayOfWeek, + IsWeekend = r.IsWeekend, + PvKwh = r.PvKwh, + LoadKwh = r.LoadKwh, + GridImportKwh = r.GridImportKwh, + BatteryChargedKwh = r.BatteryChargedKwh, + BatteryDischargedKwh = r.BatteryDischargedKwh, + BattSoC = r.BattSoC, + }; + /// /// Core report generation. Accepts pre-split current/previous week data and hourly intervals. /// weekStart/weekEnd are used to set PeriodStart/PeriodEnd on the response.