From 1761914f24d53762f232bcc057adb02c5bd9d652 Mon Sep 17 00:00:00 2001 From: Yinyin Liu Date: Mon, 2 Mar 2026 12:49:46 +0100 Subject: [PATCH] restructured data pipeline for report system and updated the way to create monthly and yearly report --- csharp/App/Backend/Controller.cs | 173 +++++++++-- .../App/Backend/DataTypes/AiInsightCache.cs | 29 ++ .../Backend/DataTypes/DailyEnergyRecord.cs | 31 ++ csharp/App/Backend/Database/Create.cs | 2 + csharp/App/Backend/Database/Db.cs | 5 + csharp/App/Backend/Database/Delete.cs | 53 ++++ csharp/App/Backend/Database/Read.cs | 37 +++ csharp/App/Backend/Program.cs | 3 +- .../Backend/Services/DailyIngestionService.cs | 137 ++++++++ .../Services/ReportAggregationService.cs | 292 +++++++++++++----- .../Backend/Services/ReportEmailService.cs | 6 +- .../Backend/Services/WeeklyReportService.cs | 184 ++++++++--- .../SodiohomeInstallations/WeeklyReport.tsx | 111 ++++--- 13 files changed, 886 insertions(+), 177 deletions(-) create mode 100644 csharp/App/Backend/DataTypes/AiInsightCache.cs create mode 100644 csharp/App/Backend/DataTypes/DailyEnergyRecord.cs create mode 100644 csharp/App/Backend/Services/DailyIngestionService.cs diff --git a/csharp/App/Backend/Controller.cs b/csharp/App/Backend/Controller.cs index 9ddc14614..b9278ccb6 100644 --- a/csharp/App/Backend/Controller.cs +++ b/csharp/App/Backend/Controller.cs @@ -901,7 +901,8 @@ public class Controller : ControllerBase /// Returns JSON with daily data, weekly totals, ratios, and AI insight. /// [HttpGet(nameof(GetWeeklyReport))] - public async Task> GetWeeklyReport(Int64 installationId, Token authToken, string? language = null) + public async Task> GetWeeklyReport( + Int64 installationId, Token authToken, String? language = null, String? weekStart = null) { var user = Db.GetSession(authToken)?.User; if (user == null) @@ -911,17 +912,23 @@ public class Controller : ControllerBase if (installation is null || !user.HasAccessTo(installation)) return Unauthorized(); - var filePath = Environment.CurrentDirectory + "/tmp_report/" + installationId + ".xlsx"; - if (!System.IO.File.Exists(filePath)) - return NotFound($"No report data file found. Please place the Excel export at: tmp_report/{installationId}.xlsx"); + // Parse optional weekStart override (must be a Monday; any valid date accepted for flexibility) + DateOnly? weekStartDate = null; + if (!String.IsNullOrEmpty(weekStart)) + { + if (!DateOnly.TryParseExact(weekStart, "yyyy-MM-dd", out var parsed)) + return BadRequest("weekStart must be in yyyy-MM-dd format."); + weekStartDate = parsed; + } try { var lang = language ?? user.Language ?? "en"; - var report = await WeeklyReportService.GenerateReportAsync(installationId, installation.InstallationName, lang); + var report = await WeeklyReportService.GenerateReportAsync( + installationId, installation.InstallationName, lang, weekStartDate); - // Persist weekly summary for future monthly aggregation (idempotent) - ReportAggregationService.SaveWeeklySummary(installationId, report); + // Persist weekly summary and seed AiInsightCache for this language + ReportAggregationService.SaveWeeklySummary(installationId, report, lang); return Ok(report); } @@ -991,7 +998,8 @@ public class Controller : ControllerBase } [HttpGet(nameof(GetMonthlyReports))] - public ActionResult> GetMonthlyReports(Int64 installationId, Token authToken) + public async Task>> GetMonthlyReports( + Int64 installationId, Token authToken, String? language = null) { var user = Db.GetSession(authToken)?.User; if (user == null) @@ -1001,11 +1009,16 @@ public class Controller : ControllerBase if (installation is null || !user.HasAccessTo(installation)) return Unauthorized(); - return Ok(Db.GetMonthlyReports(installationId)); + var lang = language ?? user.Language ?? "en"; + var reports = Db.GetMonthlyReports(installationId); + foreach (var report in reports) + report.AiInsight = await ReportAggregationService.GetOrGenerateMonthlyInsightAsync(report, lang); + return Ok(reports); } [HttpGet(nameof(GetYearlyReports))] - public ActionResult> GetYearlyReports(Int64 installationId, Token authToken) + public async Task>> GetYearlyReports( + Int64 installationId, Token authToken, String? language = null) { var user = Db.GetSession(authToken)?.User; if (user == null) @@ -1015,12 +1028,16 @@ public class Controller : ControllerBase if (installation is null || !user.HasAccessTo(installation)) return Unauthorized(); - return Ok(Db.GetYearlyReports(installationId)); + var lang = language ?? user.Language ?? "en"; + var reports = Db.GetYearlyReports(installationId); + foreach (var report in reports) + report.AiInsight = await ReportAggregationService.GetOrGenerateYearlyInsightAsync(report, lang); + return Ok(reports); } /// /// Manually trigger monthly aggregation for an installation. - /// Aggregates weekly reports for the specified year/month into a monthly report. + /// Computes monthly report from daily records for the specified year/month. /// [HttpPost(nameof(TriggerMonthlyAggregation))] public async Task TriggerMonthlyAggregation(Int64 installationId, Int32 year, Int32 month, Token authToken) @@ -1039,11 +1056,11 @@ public class Controller : ControllerBase try { var lang = user.Language ?? "en"; - var weekCount = await ReportAggregationService.TriggerMonthlyAggregationAsync(installationId, year, month, lang); - if (weekCount == 0) - return NotFound($"No weekly reports found for {year}-{month:D2}."); + var dayCount = await ReportAggregationService.TriggerMonthlyAggregationAsync(installationId, year, month, lang); + if (dayCount == 0) + return NotFound($"No daily records found for {year}-{month:D2}."); - return Ok(new { message = $"Monthly report created from {weekCount} weekly reports for {year}-{month:D2}." }); + return Ok(new { message = $"Monthly report created from {dayCount} daily records for {year}-{month:D2}." }); } catch (Exception ex) { @@ -1083,6 +1100,119 @@ public class Controller : ControllerBase } } + /// + /// Manually trigger xlsx ingestion for all SodioHome installations. + /// Scans tmp_report/ for all matching xlsx files and ingests any new days. + /// + [HttpPost(nameof(IngestAllDailyData))] + public async Task IngestAllDailyData(Token authToken) + { + var user = Db.GetSession(authToken)?.User; + if (user == null) + return Unauthorized(); + + try + { + await DailyIngestionService.IngestAllInstallationsAsync(); + return Ok(new { message = "Daily data ingestion triggered for all installations." }); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[IngestAllDailyData] Error: {ex.Message}"); + return BadRequest($"Failed to ingest: {ex.Message}"); + } + } + + /// + /// Manually trigger xlsx ingestion for one installation. + /// Parses tmp_report/{installationId}*.xlsx and stores any new days in DailyEnergyRecord. + /// + [HttpPost(nameof(IngestDailyData))] + public async Task IngestDailyData(Int64 installationId, Token authToken) + { + var user = Db.GetSession(authToken)?.User; + if (user == null) + return Unauthorized(); + + var installation = Db.GetInstallationById(installationId); + if (installation is null || !user.HasAccessTo(installation)) + return Unauthorized(); + + try + { + await DailyIngestionService.IngestInstallationAsync(installationId); + return Ok(new { message = $"Daily data ingestion triggered for installation {installationId}." }); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[IngestDailyData] Error: {ex.Message}"); + return BadRequest($"Failed to ingest: {ex.Message}"); + } + } + + // ── Debug / Inspection Endpoints ────────────────────────────────── + + /// + /// Returns the stored DailyEnergyRecord rows for an installation and date range. + /// Use this to verify that xlsx ingestion worked correctly before generating reports. + /// + [HttpGet(nameof(GetDailyRecords))] + public ActionResult> GetDailyRecords( + Int64 installationId, String from, String to, Token authToken) + { + var user = Db.GetSession(authToken)?.User; + if (user == null) + return Unauthorized(); + + var installation = Db.GetInstallationById(installationId); + if (installation is null || !user.HasAccessTo(installation)) + return Unauthorized(); + + if (!DateOnly.TryParseExact(from, "yyyy-MM-dd", out var fromDate) || + !DateOnly.TryParseExact(to, "yyyy-MM-dd", out var toDate)) + return BadRequest("from and to must be in yyyy-MM-dd format."); + + var records = Db.GetDailyRecords(installationId, fromDate, toDate); + return Ok(new { count = records.Count, records }); + } + + /// + /// Deletes DailyEnergyRecord rows for an installation in the given date range. + /// Safe to use during testing — only removes daily records, not report summaries. + /// Allows re-ingesting the same xlsx files after correcting data. + /// + [HttpDelete(nameof(DeleteDailyRecords))] + public ActionResult DeleteDailyRecords( + Int64 installationId, String from, String to, Token authToken) + { + var user = Db.GetSession(authToken)?.User; + if (user == null) + return Unauthorized(); + + var installation = Db.GetInstallationById(installationId); + if (installation is null || !user.HasAccessTo(installation)) + return Unauthorized(); + + if (!DateOnly.TryParseExact(from, "yyyy-MM-dd", out var fromDate) || + !DateOnly.TryParseExact(to, "yyyy-MM-dd", out var toDate)) + return BadRequest("from and to must be in yyyy-MM-dd format."); + + var fromStr = fromDate.ToString("yyyy-MM-dd"); + var toStr = toDate.ToString("yyyy-MM-dd"); + var toDelete = Db.DailyRecords + .Where(r => r.InstallationId == installationId) + .ToList() + .Where(r => String.Compare(r.Date, fromStr, StringComparison.Ordinal) >= 0 + && String.Compare(r.Date, toStr, StringComparison.Ordinal) <= 0) + .ToList(); + + foreach (var record in toDelete) + Db.DailyRecords.Delete(r => r.Id == record.Id); + + Console.WriteLine($"[DeleteDailyRecords] Deleted {toDelete.Count} records for installation {installationId} ({from}–{to})."); + return Ok(new { deleted = toDelete.Count, from, to }); + } + [HttpPost(nameof(SendMonthlyReportEmail))] public async Task SendMonthlyReportEmail(Int64 installationId, Int32 year, Int32 month, String emailAddress, Token authToken) { @@ -1101,6 +1231,7 @@ public class Controller : ControllerBase try { var lang = user.Language ?? "en"; + report.AiInsight = await ReportAggregationService.GetOrGenerateMonthlyInsightAsync(report, lang); await ReportEmailService.SendMonthlyReportEmailAsync(report, installation.InstallationName, emailAddress, lang); return Ok(new { message = $"Monthly report sent to {emailAddress}" }); } @@ -1129,6 +1260,7 @@ public class Controller : ControllerBase try { var lang = user.Language ?? "en"; + report.AiInsight = await ReportAggregationService.GetOrGenerateYearlyInsightAsync(report, lang); await ReportEmailService.SendYearlyReportEmailAsync(report, installation.InstallationName, emailAddress, lang); return Ok(new { message = $"Yearly report sent to {emailAddress}" }); } @@ -1140,7 +1272,8 @@ public class Controller : ControllerBase } [HttpGet(nameof(GetWeeklyReportSummaries))] - public ActionResult> GetWeeklyReportSummaries(Int64 installationId, Int32 year, Int32 month, Token authToken) + public async Task>> GetWeeklyReportSummaries( + Int64 installationId, Int32 year, Int32 month, Token authToken, String? language = null) { var user = Db.GetSession(authToken)?.User; if (user == null) @@ -1150,7 +1283,11 @@ public class Controller : ControllerBase if (installation is null || !user.HasAccessTo(installation)) return Unauthorized(); - return Ok(Db.GetWeeklyReportsForMonth(installationId, year, month)); + var lang = language ?? user.Language ?? "en"; + var summaries = Db.GetWeeklyReportsForMonth(installationId, year, month); + foreach (var s in summaries) + s.AiInsight = await ReportAggregationService.GetOrGenerateWeeklyInsightAsync(s, lang); + return Ok(summaries); } [HttpPut(nameof(UpdateFolder))] diff --git a/csharp/App/Backend/DataTypes/AiInsightCache.cs b/csharp/App/Backend/DataTypes/AiInsightCache.cs new file mode 100644 index 000000000..ea411e9d7 --- /dev/null +++ b/csharp/App/Backend/DataTypes/AiInsightCache.cs @@ -0,0 +1,29 @@ +using SQLite; + +namespace InnovEnergy.App.Backend.DataTypes; + +/// +/// Per-language AI insight cache for weekly, monthly, and yearly report summaries. +/// Keyed by (ReportType, ReportId, Language) — generated once on first request per language, +/// then reused for UI reads and email sends. Never store language-specific text in the +/// summary tables themselves; always go through this cache. +/// +public class AiInsightCache +{ + [PrimaryKey, AutoIncrement] + public Int64 Id { get; set; } + + /// "weekly" | "monthly" | "yearly" + [Indexed] + public String ReportType { get; set; } = ""; + + /// FK to WeeklyReportSummary.Id / MonthlyReportSummary.Id / YearlyReportSummary.Id + [Indexed] + public Int64 ReportId { get; set; } + + /// ISO 639-1 language code: "en" | "de" | "fr" | "it" + public String Language { get; set; } = "en"; + + public String InsightText { get; set; } = ""; + public String CreatedAt { get; set; } = ""; +} diff --git a/csharp/App/Backend/DataTypes/DailyEnergyRecord.cs b/csharp/App/Backend/DataTypes/DailyEnergyRecord.cs new file mode 100644 index 000000000..713f7c224 --- /dev/null +++ b/csharp/App/Backend/DataTypes/DailyEnergyRecord.cs @@ -0,0 +1,31 @@ +using SQLite; + +namespace InnovEnergy.App.Backend.DataTypes; + +/// +/// Raw daily energy totals for one installation and calendar day. +/// Source of truth for weekly and monthly report generation. +/// Populated by DailyIngestionService from xlsx (current) or S3 (future). +/// Retention: 1 year (cleaned up annually on Jan 2). +/// +public class DailyEnergyRecord +{ + [PrimaryKey, AutoIncrement] + public Int64 Id { get; set; } + + [Indexed] + public Int64 InstallationId { get; set; } + + // ISO date string: "YYYY-MM-DD" + public String Date { get; set; } = ""; + + // Energy totals (kWh) — cumulative for the full calendar day + public Double PvProduction { get; set; } + public Double LoadConsumption { get; set; } + public Double GridImport { get; set; } + public Double GridExport { get; set; } + public Double BatteryCharged { get; set; } + public Double BatteryDischarged { get; set; } + + public String CreatedAt { get; set; } = ""; +} diff --git a/csharp/App/Backend/Database/Create.cs b/csharp/App/Backend/Database/Create.cs index 04fe7d631..051fcfb31 100644 --- a/csharp/App/Backend/Database/Create.cs +++ b/csharp/App/Backend/Database/Create.cs @@ -71,6 +71,8 @@ public static partial class Db public static Boolean Create(WeeklyReportSummary report) => Insert(report); public static Boolean Create(MonthlyReportSummary report) => Insert(report); public static Boolean Create(YearlyReportSummary report) => Insert(report); + public static Boolean Create(DailyEnergyRecord record) => Insert(record); + public static Boolean Create(AiInsightCache cache) => Insert(cache); public static void HandleAction(UserAction newAction) { diff --git a/csharp/App/Backend/Database/Db.cs b/csharp/App/Backend/Database/Db.cs index 52c82b53e..3b1233219 100644 --- a/csharp/App/Backend/Database/Db.cs +++ b/csharp/App/Backend/Database/Db.cs @@ -28,6 +28,8 @@ public static partial class Db public static TableQuery WeeklyReports => Connection.Table(); public static TableQuery MonthlyReports => Connection.Table(); public static TableQuery YearlyReports => Connection.Table(); + public static TableQuery DailyRecords => Connection.Table(); + public static TableQuery AiInsightCaches => Connection.Table(); public static void Init() @@ -57,6 +59,8 @@ public static partial class Db Connection.CreateTable(); Connection.CreateTable(); Connection.CreateTable(); + Connection.CreateTable(); + Connection.CreateTable(); }); // One-time migration: normalize legacy long-form language values to ISO codes @@ -97,6 +101,7 @@ public static partial class Db fileConnection.CreateTable(); fileConnection.CreateTable(); fileConnection.CreateTable(); + fileConnection.CreateTable(); return fileConnection; //return CopyDbToMemory(fileConnection); diff --git a/csharp/App/Backend/Database/Delete.cs b/csharp/App/Backend/Database/Delete.cs index f39eb6dd0..9e613743f 100644 --- a/csharp/App/Backend/Database/Delete.cs +++ b/csharp/App/Backend/Database/Delete.cs @@ -180,4 +180,57 @@ public static partial class Db var count = YearlyReports.Delete(r => r.InstallationId == installationId && r.Year == year); if (count > 0) Backup(); } + + /// + /// Deletes all report records older than 1 year. Called annually on Jan 2 + /// after yearly reports are created. Uses fetch-then-delete for string-compared + /// date fields (SQLite-net doesn't support string comparisons in Delete lambdas). + /// + public static void CleanupOldData() + { + var cutoff = DateOnly.FromDateTime(DateTime.UtcNow).AddYears(-1).ToString("yyyy-MM-dd"); + var prevYear = DateTime.UtcNow.Year - 1; + + // Daily records older than 1 year + var oldDailyIds = DailyRecords + .ToList() + .Where(r => String.Compare(r.Date, cutoff, StringComparison.Ordinal) < 0) + .Select(r => r.Id) + .ToList(); + foreach (var id in oldDailyIds) + DailyRecords.Delete(r => r.Id == id); + + // Weekly summaries older than 1 year + var oldWeeklyIds = WeeklyReports + .ToList() + .Where(r => String.Compare(r.PeriodEnd, cutoff, StringComparison.Ordinal) < 0) + .Select(r => r.Id) + .ToList(); + foreach (var id in oldWeeklyIds) + WeeklyReports.Delete(r => r.Id == id); + + // Monthly summaries older than 1 year + var oldMonthlyIds = MonthlyReports + .ToList() + .Where(r => String.Compare(r.PeriodEnd, cutoff, StringComparison.Ordinal) < 0) + .Select(r => r.Id) + .ToList(); + foreach (var id in oldMonthlyIds) + MonthlyReports.Delete(r => r.Id == id); + + // Yearly summaries — keep current and previous year only + YearlyReports.Delete(r => r.Year < prevYear); + + // AI insight cache entries older than 1 year + var oldCacheIds = AiInsightCaches + .ToList() + .Where(c => String.Compare(c.CreatedAt, cutoff, StringComparison.Ordinal) < 0) + .Select(c => c.Id) + .ToList(); + foreach (var id in oldCacheIds) + AiInsightCaches.Delete(c => c.Id == id); + + Backup(); + Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff})."); + } } \ No newline at end of file diff --git a/csharp/App/Backend/Database/Read.cs b/csharp/App/Backend/Database/Read.cs index 68bfa2015..a9eb013c3 100644 --- a/csharp/App/Backend/Database/Read.cs +++ b/csharp/App/Backend/Database/Read.cs @@ -94,4 +94,41 @@ public static partial class Db .Where(r => r.InstallationId == installationId) .OrderByDescending(r => r.Year) .ToList(); + + // ── DailyEnergyRecord Queries ────────────────────────────────────── + + /// + /// Returns daily records for an installation within [from, to] inclusive, ordered by date. + /// + public static List GetDailyRecords(Int64 installationId, DateOnly from, DateOnly to) + { + var fromStr = from.ToString("yyyy-MM-dd"); + var toStr = to.ToString("yyyy-MM-dd"); + return DailyRecords + .Where(r => r.InstallationId == installationId) + .ToList() + .Where(r => String.Compare(r.Date, fromStr, StringComparison.Ordinal) >= 0 + && String.Compare(r.Date, toStr, StringComparison.Ordinal) <= 0) + .OrderBy(r => r.Date) + .ToList(); + } + + /// + /// Returns true if a daily record already exists for this installation+date (idempotency check). + /// + public static Boolean DailyRecordExists(Int64 installationId, String date) + => DailyRecords + .Any(r => r.InstallationId == installationId && r.Date == date); + + // ── AiInsightCache Queries ───────────────────────────────────────── + + /// + /// Returns the cached AI insight text for (reportType, reportId, language), or null on miss. + /// + public static String? GetCachedInsight(String reportType, Int64 reportId, String language) + => AiInsightCaches + .FirstOrDefault(c => c.ReportType == reportType + && c.ReportId == reportId + && c.Language == language) + ?.InsightText; } \ No newline at end of file diff --git a/csharp/App/Backend/Program.cs b/csharp/App/Backend/Program.cs index 73350b51c..499bbdeea 100644 --- a/csharp/App/Backend/Program.cs +++ b/csharp/App/Backend/Program.cs @@ -28,7 +28,8 @@ public static class Program LoadEnvFile(); DiagnosticService.Initialize(); AlarmReviewService.StartDailyScheduler(); - // ReportAggregationService.StartScheduler(); // TODO: uncomment to enable automatic monthly/yearly aggregation + // DailyIngestionService.StartScheduler(); // Phase 2: enable when S3 auto-push is ready + // ReportAggregationService.StartScheduler(); // Phase 2: enable when scheduler should auto-run monthly/yearly var builder = WebApplication.CreateBuilder(args); RabbitMqManager.InitializeEnvironment(); diff --git a/csharp/App/Backend/Services/DailyIngestionService.cs b/csharp/App/Backend/Services/DailyIngestionService.cs new file mode 100644 index 000000000..e8d0f8ae9 --- /dev/null +++ b/csharp/App/Backend/Services/DailyIngestionService.cs @@ -0,0 +1,137 @@ +using InnovEnergy.App.Backend.Database; +using InnovEnergy.App.Backend.DataTypes; + +namespace InnovEnergy.App.Backend.Services; + +/// +/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table. +/// This is the source-of-truth population step for the report pipeline. +/// +/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx +/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor) +/// +/// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the +/// IngestDailyData API endpoint. +/// +public static class DailyIngestionService +{ + private static readonly String TmpReportDir = + Environment.CurrentDirectory + "/tmp_report/"; + + private static Timer? _dailyTimer; + + /// + /// Starts the daily scheduler. Call once on app startup. + /// Ingests xlsx data at 01:00 UTC every day. + /// + public static void StartScheduler() + { + var now = DateTime.UtcNow; + var next = now.Date.AddDays(1).AddHours(1); // 01:00 UTC tomorrow + + _dailyTimer = new Timer( + _ => + { + try + { + IngestAllInstallationsAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[DailyIngestion] Scheduler error: {ex.Message}"); + } + }, + null, next - now, TimeSpan.FromDays(1)); + + Console.WriteLine($"[DailyIngestion] Scheduler started. Next run: {next:yyyy-MM-dd HH:mm} UTC"); + } + + /// + /// Ingests xlsx data for all SodioHome installations. Safe to call manually. + /// + public static async Task IngestAllInstallationsAsync() + { + Console.WriteLine($"[DailyIngestion] Starting ingestion for all SodioHome installations..."); + + var installations = Db.Installations + .Where(i => i.Product == (Int32)ProductType.SodioHome) + .ToList(); + + foreach (var installation in installations) + { + try + { + await IngestInstallationAsync(installation.Id); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[DailyIngestion] Failed for installation {installation.Id}: {ex.Message}"); + } + } + + Console.WriteLine($"[DailyIngestion] Ingestion complete."); + } + + /// + /// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores + /// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per + /// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent. + /// + public static async Task IngestInstallationAsync(Int64 installationId) + { + if (!Directory.Exists(TmpReportDir)) + { + Console.WriteLine($"[DailyIngestion] tmp_report directory not found, skipping."); + return; + } + + var xlsxFiles = Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx"); + if (xlsxFiles.Length == 0) + { + Console.WriteLine($"[DailyIngestion] No xlsx found for installation {installationId}, skipping."); + return; + } + + var newCount = 0; + var totalParsed = 0; + + foreach (var xlsxPath in xlsxFiles.OrderBy(f => f)) + { + List days; + try + { + days = ExcelDataParser.Parse(xlsxPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[DailyIngestion] Failed to parse {Path.GetFileName(xlsxPath)}: {ex.Message}"); + continue; + } + + totalParsed += days.Count; + + foreach (var day in days) + { + if (Db.DailyRecordExists(installationId, day.Date)) + continue; + + Db.Create(new DailyEnergyRecord + { + InstallationId = installationId, + Date = day.Date, + PvProduction = day.PvProduction, + LoadConsumption = day.LoadConsumption, + GridImport = day.GridImport, + GridExport = day.GridExport, + BatteryCharged = day.BatteryCharged, + BatteryDischarged = day.BatteryDischarged, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + newCount++; + } + } + + Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newCount} new day(s) ingested ({totalParsed} total across {xlsxFiles.Length} file(s))."); + await Task.CompletedTask; + } +} diff --git a/csharp/App/Backend/Services/ReportAggregationService.cs b/csharp/App/Backend/Services/ReportAggregationService.cs index 9100bcab0..3dbc40e62 100644 --- a/csharp/App/Backend/Services/ReportAggregationService.cs +++ b/csharp/App/Backend/Services/ReportAggregationService.cs @@ -167,7 +167,7 @@ public static class ReportAggregationService // ── Save Weekly Summary ─────────────────────────────────────────── - public static void SaveWeeklySummary(Int64 installationId, WeeklyReportResponse report) + public static void SaveWeeklySummary(Int64 installationId, WeeklyReportResponse report, String language = "en") { try { @@ -211,6 +211,18 @@ public static class ReportAggregationService }; Db.Create(summary); + + // Seed AiInsightCache so historical reads for this language are free + if (!String.IsNullOrEmpty(summary.AiInsight)) + Db.Create(new AiInsightCache + { + ReportType = "weekly", + ReportId = summary.Id, + Language = language, + InsightText = summary.AiInsight, + CreatedAt = summary.CreatedAt, + }); + Console.WriteLine($"[ReportAggregation] Saved weekly summary for installation {installationId}, period {report.PeriodStart}–{report.PeriodEnd} (replaced {overlapping.Count} overlapping record(s))"); } catch (Exception ex) @@ -272,12 +284,14 @@ public static class ReportAggregationService /// public static async Task TriggerMonthlyAggregationAsync(Int64 installationId, Int32 year, Int32 month, String language = "en") { - var weeklies = Db.GetWeeklyReportsForMonth(installationId, year, month); - if (weeklies.Count == 0) + var first = new DateOnly(year, month, 1); + var last = first.AddMonths(1).AddDays(-1); + var days = Db.GetDailyRecords(installationId, first, last); + if (days.Count == 0) return 0; await AggregateMonthForInstallation(installationId, year, month, language); - return weeklies.Count; + return days.Count; } /// @@ -300,10 +314,15 @@ public static class ReportAggregationService var previousMonth = DateTime.Now.AddMonths(-1); var year = previousMonth.Year; var month = previousMonth.Month; + var first = new DateOnly(year, month, 1); + var last = first.AddMonths(1).AddDays(-1); Console.WriteLine($"[ReportAggregation] Running month-end aggregation for {year}-{month:D2}..."); - var installationIds = Db.WeeklyReports + // Find all installations that have daily records for the previous month + var installationIds = Db.DailyRecords + .Where(r => String.Compare(r.Date, first.ToString("yyyy-MM-dd"), StringComparison.Ordinal) >= 0 + && String.Compare(r.Date, last.ToString("yyyy-MM-dd"), StringComparison.Ordinal) <= 0) .Select(r => r.InstallationId) .ToList() .Distinct() @@ -314,11 +333,6 @@ public static class ReportAggregationService try { await AggregateMonthForInstallation(installationId, year, month); - - // Scheduler mode: clean up weekly records after successful aggregation. - // The monthly report is now the source of truth for this month. - Db.DeleteWeeklyReportsForMonth(installationId, year, month); - Console.WriteLine($"[ReportAggregation] Cleaned up weekly records for installation {installationId}, {year}-{month:D2}."); } catch (Exception ex) { @@ -329,23 +343,25 @@ public static class ReportAggregationService private static async Task AggregateMonthForInstallation(Int64 installationId, Int32 year, Int32 month, String language = "en") { - var weeklies = Db.GetWeeklyReportsForMonth(installationId, year, month); - if (weeklies.Count == 0) + // Compute from daily records for the full calendar month + var first = new DateOnly(year, month, 1); + var last = first.AddMonths(1).AddDays(-1); + var days = Db.GetDailyRecords(installationId, first, last); + if (days.Count == 0) return; - // If monthly report already exists, delete it so we can re-generate with latest weekly data. - // This supports partial months and re-aggregation when new weekly data arrives. + // If monthly report already exists, delete it so we can re-generate Db.DeleteMonthlyReport(installationId, year, month); - // Sum energy totals - var totalPv = Math.Round(weeklies.Sum(w => w.TotalPvProduction), 1); - var totalConsump = Math.Round(weeklies.Sum(w => w.TotalConsumption), 1); - var totalGridIn = Math.Round(weeklies.Sum(w => w.TotalGridImport), 1); - var totalGridOut = Math.Round(weeklies.Sum(w => w.TotalGridExport), 1); - var totalBattChg = Math.Round(weeklies.Sum(w => w.TotalBatteryCharged), 1); - var totalBattDis = Math.Round(weeklies.Sum(w => w.TotalBatteryDischarged), 1); + // Sum energy totals directly from daily records + var totalPv = Math.Round(days.Sum(d => d.PvProduction), 1); + var totalConsump = Math.Round(days.Sum(d => d.LoadConsumption), 1); + var totalGridIn = Math.Round(days.Sum(d => d.GridImport), 1); + var totalGridOut = Math.Round(days.Sum(d => d.GridExport), 1); + var totalBattChg = Math.Round(days.Sum(d => d.BatteryCharged), 1); + var totalBattDis = Math.Round(days.Sum(d => d.BatteryDischarged), 1); - // Re-derive ratios from aggregated totals + // Re-derive ratios var energySaved = Math.Round(totalConsump - totalGridIn, 1); var savingsCHF = Math.Round(energySaved * ElectricityPriceCHF, 0); var selfSufficiency = totalConsump > 0 ? Math.Round((totalConsump - totalGridIn) / totalConsump * 100, 1) : 0; @@ -353,56 +369,59 @@ public static class ReportAggregationService var batteryEff = totalBattChg > 0 ? Math.Round(totalBattDis / totalBattChg * 100, 1) : 0; var gridDependency = totalConsump > 0 ? Math.Round(totalGridIn / totalConsump * 100, 1) : 0; - // Average behavioral highlights - var avgPeakLoad = (Int32)Math.Round(weeklies.Average(w => w.PeakLoadHour)); - var avgPeakSolar = (Int32)Math.Round(weeklies.Average(w => w.PeakSolarHour)); - var avgWeekdayLoad = Math.Round(weeklies.Average(w => w.WeekdayAvgDailyLoad), 1); - var avgWeekendLoad = Math.Round(weeklies.Average(w => w.WeekendAvgDailyLoad), 1); - // Get installation name for AI insight var installation = Db.GetInstallationById(installationId); var installationName = installation?.InstallationName ?? $"Installation {installationId}"; var monthName = new DateTime(year, month, 1).ToString("MMMM yyyy"); var aiInsight = await GenerateMonthlyAiInsightAsync( - installationName, monthName, weeklies.Count, + installationName, monthName, days.Count, totalPv, totalConsump, totalGridIn, totalGridOut, totalBattChg, totalBattDis, energySaved, savingsCHF, selfSufficiency, batteryEff, language); var monthlySummary = new MonthlyReportSummary { - InstallationId = installationId, - Year = year, - Month = month, - PeriodStart = weeklies.Min(w => w.PeriodStart), - PeriodEnd = weeklies.Max(w => w.PeriodEnd), - TotalPvProduction = totalPv, - TotalConsumption = totalConsump, - TotalGridImport = totalGridIn, - TotalGridExport = totalGridOut, - TotalBatteryCharged = totalBattChg, - TotalBatteryDischarged = totalBattDis, - TotalEnergySaved = energySaved, - TotalSavingsCHF = savingsCHF, - SelfSufficiencyPercent = selfSufficiency, - SelfConsumptionPercent = selfConsumption, + InstallationId = installationId, + Year = year, + Month = month, + PeriodStart = first.ToString("yyyy-MM-dd"), + PeriodEnd = last.ToString("yyyy-MM-dd"), + TotalPvProduction = totalPv, + TotalConsumption = totalConsump, + TotalGridImport = totalGridIn, + TotalGridExport = totalGridOut, + TotalBatteryCharged = totalBattChg, + TotalBatteryDischarged = totalBattDis, + TotalEnergySaved = energySaved, + TotalSavingsCHF = savingsCHF, + SelfSufficiencyPercent = selfSufficiency, + SelfConsumptionPercent = selfConsumption, BatteryEfficiencyPercent = batteryEff, - GridDependencyPercent = gridDependency, - AvgPeakLoadHour = avgPeakLoad, - AvgPeakSolarHour = avgPeakSolar, - AvgWeekdayDailyLoad = avgWeekdayLoad, - AvgWeekendDailyLoad = avgWeekendLoad, - WeekCount = weeklies.Count, - AiInsight = aiInsight, - CreatedAt = DateTime.UtcNow.ToString("o"), + GridDependencyPercent = gridDependency, + AvgPeakLoadHour = 0, // Not available without hourly data; Phase 3 will add + AvgPeakSolarHour = 0, + AvgWeekdayDailyLoad = 0, + AvgWeekendDailyLoad = 0, + WeekCount = days.Count, // repurposed as day count + AiInsight = aiInsight, + CreatedAt = DateTime.UtcNow.ToString("o"), }; Db.Create(monthlySummary); - // Weekly records are kept — allows re-generation if new weekly data arrives. - // Cleanup can be done later by the automated scheduler or manually. - Console.WriteLine($"[ReportAggregation] Monthly report created for installation {installationId}, {year}-{month:D2} ({weeklies.Count} weeks aggregated)."); + // Seed AiInsightCache so the generating language is pre-cached + if (!String.IsNullOrEmpty(monthlySummary.AiInsight)) + Db.Create(new AiInsightCache + { + ReportType = "monthly", + ReportId = monthlySummary.Id, + Language = language, + InsightText = monthlySummary.AiInsight, + CreatedAt = monthlySummary.CreatedAt, + }); + + Console.WriteLine($"[ReportAggregation] Monthly report created for installation {installationId}, {year}-{month:D2} ({days.Count} days, {first}–{last})."); } // ── Year-End Aggregation ────────────────────────────────────────── @@ -425,17 +444,15 @@ public static class ReportAggregationService try { await AggregateYearForInstallation(installationId, previousYear); - - // Scheduler mode: clean up monthly records after successful aggregation. - // The yearly report is now the source of truth for this year. - Db.DeleteMonthlyReportsForYear(installationId, previousYear); - Console.WriteLine($"[ReportAggregation] Cleaned up monthly records for installation {installationId}, {previousYear}."); } catch (Exception ex) { Console.Error.WriteLine($"[ReportAggregation] Year aggregation failed for installation {installationId}: {ex.Message}"); } } + + // Time-based cleanup: delete records older than 1 year, runs after yearly report is created + CleanupOldRecords(); } private static async Task AggregateYearForInstallation(Int64 installationId, Int32 year, String language = "en") @@ -505,13 +522,140 @@ public static class ReportAggregationService }; Db.Create(yearlySummary); - // Monthly records are kept — allows re-generation if new monthly data arrives. + + // Seed AiInsightCache so the generating language is pre-cached + if (!String.IsNullOrEmpty(yearlySummary.AiInsight)) + Db.Create(new AiInsightCache + { + ReportType = "yearly", + ReportId = yearlySummary.Id, + Language = language, + InsightText = yearlySummary.AiInsight, + CreatedAt = yearlySummary.CreatedAt, + }); Console.WriteLine($"[ReportAggregation] Yearly report created for installation {installationId}, {year} ({monthlies.Count} months aggregated)."); } + // ── AI Insight Cache ────────────────────────────────────────────── + + /// + /// Returns cached AI insight for (reportType, reportId, language). + /// On cache miss: calls generate(), stores the result, and returns it. + /// This is the single entry-point for all per-language insight reads. + /// + public static async Task GetOrGenerateInsightAsync( + String reportType, Int64 reportId, String language, Func> generate) + { + var cached = Db.GetCachedInsight(reportType, reportId, language); + if (!String.IsNullOrEmpty(cached)) + return cached; + + var insight = await generate(); + Db.Create(new AiInsightCache + { + ReportType = reportType, + ReportId = reportId, + Language = language, + InsightText = insight, + CreatedAt = DateTime.UtcNow.ToString("o"), + }); + return insight; + } + + /// Cached-or-generated AI insight for a stored WeeklyReportSummary. + public static Task GetOrGenerateWeeklyInsightAsync( + WeeklyReportSummary report, String language) + { + var installationName = Db.GetInstallationById(report.InstallationId)?.InstallationName + ?? $"Installation {report.InstallationId}"; + return GetOrGenerateInsightAsync("weekly", report.Id, language, + () => GenerateWeeklySummaryAiInsightAsync(report, installationName, language)); + } + + /// Cached-or-generated AI insight for a stored MonthlyReportSummary. + public static Task GetOrGenerateMonthlyInsightAsync( + MonthlyReportSummary report, String language) + { + var installationName = Db.GetInstallationById(report.InstallationId)?.InstallationName + ?? $"Installation {report.InstallationId}"; + var monthName = new DateTime(report.Year, report.Month, 1).ToString("MMMM yyyy"); + return GetOrGenerateInsightAsync("monthly", report.Id, language, + () => GenerateMonthlyAiInsightAsync( + installationName, monthName, report.WeekCount, + report.TotalPvProduction, report.TotalConsumption, + report.TotalGridImport, report.TotalGridExport, + report.TotalBatteryCharged, report.TotalBatteryDischarged, + report.TotalEnergySaved, report.TotalSavingsCHF, + report.SelfSufficiencyPercent, report.BatteryEfficiencyPercent, language)); + } + + /// Cached-or-generated AI insight for a stored YearlyReportSummary. + public static Task GetOrGenerateYearlyInsightAsync( + YearlyReportSummary report, String language) + { + var installationName = Db.GetInstallationById(report.InstallationId)?.InstallationName + ?? $"Installation {report.InstallationId}"; + return GetOrGenerateInsightAsync("yearly", report.Id, language, + () => GenerateYearlyAiInsightAsync( + installationName, report.Year, report.MonthCount, + report.TotalPvProduction, report.TotalConsumption, + report.TotalGridImport, report.TotalGridExport, + report.TotalBatteryCharged, report.TotalBatteryDischarged, + report.TotalEnergySaved, report.TotalSavingsCHF, + report.SelfSufficiencyPercent, report.BatteryEfficiencyPercent, language)); + } + + // ── Time-Based Cleanup ──────────────────────────────────────────── + + /// + /// Deletes records older than 1 year. Called annually on Jan 2 after + /// yearly reports are created, so monthly summaries are still available + /// when yearly is computed. + /// + private static void CleanupOldRecords() + { + try + { + Db.CleanupOldData(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[ReportAggregation] Cleanup error: {ex.Message}"); + } + } + // ── AI Insight Generation ───────────────────────────────────────── + /// + /// Simplified weekly AI insight generated from the stored WeeklyReportSummary numerical fields. + /// Used for historical weeks where the original hourly data is no longer available. + /// + private static async Task GenerateWeeklySummaryAiInsightAsync( + WeeklyReportSummary summary, String installationName, String language = "en") + { + var apiKey = Environment.GetEnvironmentVariable("MISTRAL_API_KEY"); + if (String.IsNullOrWhiteSpace(apiKey)) + return "AI insight unavailable (API key not configured)."; + + var langName = GetLanguageName(language); + var prompt = $@"You are an energy advisor for a sodistore home installation: ""{installationName}"". + +Write a concise weekly performance summary in {langName} (4 bullet points starting with ""- ""). + +WEEKLY FACTS for {summary.PeriodStart} to {summary.PeriodEnd}: +- PV production: {summary.TotalPvProduction:F1} kWh | Consumption: {summary.TotalConsumption:F1} kWh +- Grid import: {summary.TotalGridImport:F1} kWh | Grid export: {summary.TotalGridExport:F1} kWh +- Battery: {summary.TotalBatteryCharged:F1} kWh charged, {summary.TotalBatteryDischarged:F1} kWh discharged +- Energy saved: {summary.TotalEnergySaved:F1} kWh = ~{summary.TotalSavingsCHF:F0} CHF +- Self-sufficiency: {summary.SelfSufficiencyPercent:F1}% | Grid dependency: {summary.GridDependencyPercent:F1}% + +Rules: Write in {langName}. Write for a homeowner. No asterisks or formatting marks. +Exactly 4 bullet points. Each starts with ""- Title: description"" format."; + + return await CallMistralAsync(apiKey, prompt); + } + private static String GetLanguageName(String code) => code switch { "de" => "German", @@ -533,26 +677,28 @@ public static class ReportAggregationService return "AI insight unavailable (API key not configured)."; var langName = GetLanguageName(language); + + // Determine which metric is weakest so the tip can be targeted + var weakMetric = batteryEff < 80 ? "battery" : selfSufficiency < 40 ? "self-sufficiency" : "general"; + var prompt = $@"You are an energy advisor for a sodistore home installation: ""{installationName}"". Write a concise monthly performance summary in {langName} (4 bullet points, plain text, no markdown). -MONTHLY FACTS for {monthName} ({weekCount} weeks): -- Total PV production: {totalPv:F1} kWh +MONTHLY FACTS for {monthName} ({weekCount} days of data): +- PV production: {totalPv:F1} kWh - Total consumption: {totalConsump:F1} kWh -- Total grid import: {totalGridIn:F1} kWh, export: {totalGridOut:F1} kWh -- Battery: {totalBattChg:F1} kWh charged, {totalBattDis:F1} kWh discharged -- Energy saved from grid: {energySaved:F1} kWh = ~{savingsCHF:F0} CHF (at {ElectricityPriceCHF} CHF/kWh) -- Self-sufficiency: {selfSufficiency:F1}% -- Battery efficiency: {batteryEff:F1}% +- Self-sufficiency: {selfSufficiency:F1}% (powered by solar + battery, not grid) +- Battery: {totalBattChg:F1} kWh charged, {totalBattDis:F1} kWh discharged, efficiency {batteryEff:F1}% +- Energy saved: {energySaved:F1} kWh = ~{savingsCHF:F0} CHF saved (at {ElectricityPriceCHF} CHF/kWh) INSTRUCTIONS: -1. Monthly savings summary: total energy saved and money saved this month. Use the exact numbers provided. -2. Solar & battery performance: comment on PV production and battery utilization. -3. Grid dependency: note grid import relative to total consumption. -4. Recommendation for next month: one actionable suggestion. +1. Savings: state exactly how much energy and money was saved this month. Positive framing. +2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""X% of your home ran on solar + battery""). Do NOT mention raw grid import kWh. +3. Battery: comment on battery utilization and efficiency. If efficiency < 80%, note it may need attention. +4. Tip: one specific actionable suggestion based on the weakest metric: {weakMetric}. If general, suggest the most impactful habit change based on the numbers above. -Rules: Write in {langName}. Write for a homeowner. No asterisks or formatting marks. No closing remarks. Exactly 4 bullet points starting with ""- "". Each bullet must have a short title followed by colon then description."; +Rules: Write in {langName}. Write for a homeowner, not a technician. No asterisks or formatting marks. No closing remarks. Exactly 4 bullet points starting with ""- "". Each bullet must have a short title followed by colon then description."; return await CallMistralAsync(apiKey, prompt); } diff --git a/csharp/App/Backend/Services/ReportEmailService.cs b/csharp/App/Backend/Services/ReportEmailService.cs index a00a74f6a..1dbec7ec4 100644 --- a/csharp/App/Backend/Services/ReportEmailService.cs +++ b/csharp/App/Backend/Services/ReportEmailService.cs @@ -217,10 +217,10 @@ public static class ReportEmailService var cur = r.CurrentWeek; var prev = r.PreviousWeek; - // Parse AI insight into
  • bullet points (split on newlines, strip leading "- " or "1. ") + // Parse AI insight into
  • bullet points (split on newlines, strip leading "- " or "1. ", strip ** markdown) var insightLines = r.AiInsight .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(l => System.Text.RegularExpressions.Regex.Replace(l.Trim(), @"^[\d]+[.)]\s*|^[-*]\s*", "")) + .Select(l => System.Text.RegularExpressions.Regex.Replace(l.Trim(), @"^[\d]+[.)]\s*|^[-*]\s*", "").Replace("**", "")) .Where(l => l.Length > 0) .ToList(); @@ -584,7 +584,7 @@ public static class ReportEmailService { var insightLines = aiInsight .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(l => System.Text.RegularExpressions.Regex.Replace(l.Trim(), @"^[\d]+[.)]\s*|^[-*]\s*", "")) + .Select(l => System.Text.RegularExpressions.Regex.Replace(l.Trim(), @"^[\d]+[.)]\s*|^[-*]\s*", "").Replace("**", "")) .Where(l => l.Length > 0) .ToList(); diff --git a/csharp/App/Backend/Services/WeeklyReportService.cs b/csharp/App/Backend/Services/WeeklyReportService.cs index 9c9e4a3a9..28a1d2833 100644 --- a/csharp/App/Backend/Services/WeeklyReportService.cs +++ b/csharp/App/Backend/Services/WeeklyReportService.cs @@ -1,4 +1,5 @@ using Flurl.Http; +using InnovEnergy.App.Backend.Database; using InnovEnergy.App.Backend.DataTypes; using Newtonsoft.Json; @@ -8,22 +9,73 @@ public static class WeeklyReportService { private static readonly string TmpReportDir = Environment.CurrentDirectory + "/tmp_report/"; + // ── Calendar Week Helpers ────────────────────────────────────────── + + /// + /// Returns the last completed calendar week (Mon–Sun). + /// Example: called on Wednesday 2026-03-11 → returns (2026-03-02, 2026-03-08). + /// + public static (DateOnly Monday, DateOnly Sunday) LastCalendarWeek() + { + var today = DateOnly.FromDateTime(DateTime.UtcNow); + // DayOfWeek: Sun=0, Mon=1, ..., Sat=6 + // daysSinceMonday: Mon=0, Tue=1, ..., Sun=6 + var daysSinceMonday = ((Int32)today.DayOfWeek + 6) % 7; + var thisMonday = today.AddDays(-daysSinceMonday); + var lastMonday = thisMonday.AddDays(-7); + var lastSunday = thisMonday.AddDays(-1); + return (lastMonday, lastSunday); + } + + /// + /// Returns the calendar week before last (for comparison). + /// + private static (DateOnly Monday, DateOnly Sunday) PreviousCalendarWeek() + { + var (lastMon, _) = LastCalendarWeek(); + return (lastMon.AddDays(-7), lastMon.AddDays(-1)); + } + + // ── Report Generation ────────────────────────────────────────────── + /// /// Generates a full weekly report for the given installation. - /// Cache is invalidated automatically when the xlsx file is newer than the cache. - /// To force regeneration (e.g. after a prompt change), simply delete the cache files. + /// Data source priority: + /// 1. DailyEnergyRecord rows in SQLite (populated by DailyIngestionService) + /// 2. xlsx file fallback (if DB not yet populated for the target week) + /// Cache is keyed to the calendar week — invalidated when the week changes. + /// Pass weekStartOverride to generate a report for a specific historical week (disables cache). /// - public static async Task GenerateReportAsync(long installationId, string installationName, string language = "en") + public static async Task GenerateReportAsync( + long installationId, string installationName, string language = "en", + DateOnly? weekStartOverride = null) { - var xlsxPath = TmpReportDir + installationId + ".xlsx"; - var cachePath = TmpReportDir + $"{installationId}_{language}.cache.json"; + DateOnly curMon, curSun, prevMon, prevSun; - // Use cached report if xlsx hasn't changed since cache was written - if (File.Exists(cachePath) && File.Exists(xlsxPath)) + if (weekStartOverride.HasValue) { - var xlsxModified = File.GetLastWriteTimeUtc(xlsxPath); - var cacheModified = File.GetLastWriteTimeUtc(cachePath); - if (cacheModified > xlsxModified) + // Debug/backfill mode: use the provided Monday as the week start + curMon = weekStartOverride.Value; + curSun = curMon.AddDays(6); + prevMon = curMon.AddDays(-7); + prevSun = curMon.AddDays(-1); + } + else + { + (curMon, curSun) = LastCalendarWeek(); + (prevMon, prevSun) = PreviousCalendarWeek(); + } + + // Skip file cache when a specific week is requested (avoid stale or wrong-week hits) + var cachePath = weekStartOverride.HasValue + ? null + : TmpReportDir + $"{installationId}_{language}_{curMon:yyyy-MM-dd}.cache.json"; + + // Use cache if it exists and is less than 6 hours old (skipped in override mode) + if (cachePath != null && File.Exists(cachePath)) + { + var cacheAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(cachePath); + if (cacheAge.TotalHours < 6) { try { @@ -31,7 +83,7 @@ public static class WeeklyReportService await File.ReadAllTextAsync(cachePath)); if (cached != null) { - Console.WriteLine($"[WeeklyReportService] Returning cached report for installation {installationId} ({language})."); + Console.WriteLine($"[WeeklyReportService] Returning cached report for installation {installationId} ({language}), week {curMon:yyyy-MM-dd}."); return cached; } } @@ -42,54 +94,94 @@ public static class WeeklyReportService } } - // Parse both daily summaries and hourly intervals from the same xlsx - var allDays = ExcelDataParser.Parse(xlsxPath); - var allHourly = ExcelDataParser.ParseHourly(xlsxPath); - var report = await GenerateReportFromDataAsync(allDays, allHourly, installationName, language); + // 1. Try to load daily records from SQLite for the calendar weeks + var currentWeekDays = Db.GetDailyRecords(installationId, curMon, curSun) + .Select(r => ToDailyEnergyData(r)).ToList(); + var previousWeekDays = Db.GetDailyRecords(installationId, prevMon, prevSun) + .Select(r => ToDailyEnergyData(r)).ToList(); - // Write cache - try + // 2. Fallback: if DB empty for current week, parse all xlsx files on the fly + var xlsxFiles = Directory.Exists(TmpReportDir) + ? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList() + : new List(); + + if (currentWeekDays.Count == 0 && xlsxFiles.Count > 0) { - await File.WriteAllTextAsync(cachePath, JsonConvert.SerializeObject(report)); + Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx."); + var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList(); + currentWeekDays = allDaysParsed + .Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; }) + .ToList(); + previousWeekDays = allDaysParsed + .Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; }) + .ToList(); } - catch (Exception ex) + + if (currentWeekDays.Count == 0) + throw new InvalidOperationException( + $"No energy data available for week {curMon:yyyy-MM-dd}–{curSun:yyyy-MM-dd}. " + + "Upload an xlsx file or wait for daily ingestion."); + + // 3. Load hourly data from ALL xlsx files for behavioral analysis (current week only). + // Combine all files so any week can find its hourly data regardless of file split. + // Future: replace with S3 hourly fetch. + var allHourly = xlsxFiles + .SelectMany(p => { try { return ExcelDataParser.ParseHourly(p); } catch { return Enumerable.Empty(); } }) + .ToList(); + var curMonDt = curMon.ToDateTime(TimeOnly.MinValue); + var curSunDt = curSun.ToDateTime(TimeOnly.MaxValue); + var currentHourlyData = allHourly + .Where(h => h.DateTime >= curMonDt && h.DateTime <= curSunDt) + .ToList(); + + var report = await GenerateReportFromDataAsync( + currentWeekDays, previousWeekDays, currentHourlyData, installationName, language, + curMon, curSun); + + // Write cache (skipped in override mode) + if (cachePath != null) { - Console.Error.WriteLine($"[WeeklyReportService] Could not write cache: {ex.Message}"); + try + { + await File.WriteAllTextAsync(cachePath, JsonConvert.SerializeObject(report)); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[WeeklyReportService] Could not write cache: {ex.Message}"); + } } return report; } + // ── Conversion helpers ───────────────────────────────────────────── + + private static DailyEnergyData ToDailyEnergyData(DailyEnergyRecord r) => new() + { + Date = r.Date, + PvProduction = r.PvProduction, + LoadConsumption = r.LoadConsumption, + GridImport = r.GridImport, + GridExport = r.GridExport, + BatteryCharged = r.BatteryCharged, + BatteryDischarged = r.BatteryDischarged, + }; + /// - /// Core report generation. Accepts both daily summaries and hourly intervals. + /// Core report generation. Accepts pre-split current/previous week data and hourly intervals. + /// weekStart/weekEnd are used to set PeriodStart/PeriodEnd on the response. /// public static async Task GenerateReportFromDataAsync( - List allDays, - List allHourly, + List currentWeekDays, + List previousWeekDays, + List currentHourlyData, string installationName, - string language = "en") + string language = "en", + DateOnly? weekStart = null, + DateOnly? weekEnd = null) { - // Sort by date - allDays = allDays.OrderBy(d => d.Date).ToList(); - - // Split into previous week and current week (daily) - List previousWeekDays; - List currentWeekDays; - - if (allDays.Count > 7) - { - previousWeekDays = allDays.Take(allDays.Count - 7).ToList(); - currentWeekDays = allDays.Skip(allDays.Count - 7).ToList(); - } - else - { - previousWeekDays = new List(); - currentWeekDays = allDays; - } - - // Restrict hourly data to current week only for behavioral analysis - var currentWeekStart = DateTime.Parse(currentWeekDays.First().Date); - var currentHourlyData = allHourly.Where(h => h.DateTime.Date >= currentWeekStart.Date).ToList(); + currentWeekDays = currentWeekDays.OrderBy(d => d.Date).ToList(); + previousWeekDays = previousWeekDays.OrderBy(d => d.Date).ToList(); var currentSummary = Summarize(currentWeekDays); var previousSummary = previousWeekDays.Count > 0 ? Summarize(previousWeekDays) : null; @@ -149,7 +241,7 @@ public static class WeeklyReportService PvChangePercent = pvChange, ConsumptionChangePercent = consumptionChange, GridImportChangePercent = gridImportChange, - DailyData = allDays, + DailyData = currentWeekDays, Behavior = behavior, AiInsight = aiInsight, }; diff --git a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/WeeklyReport.tsx b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/WeeklyReport.tsx index aeddfcbc2..b166364ea 100644 --- a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/WeeklyReport.tsx +++ b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/WeeklyReport.tsx @@ -224,10 +224,11 @@ function WeeklyReport({ installationId }: WeeklyReportProps) { const [generating, setGenerating] = useState(null); const fetchReportData = () => { - axiosConfig.get('/GetMonthlyReports', { params: { installationId } }) + const lang = intl.locale; + axiosConfig.get('/GetMonthlyReports', { params: { installationId, language: lang } }) .then(res => setMonthlyReports(res.data)) .catch(() => {}); - axiosConfig.get('/GetYearlyReports', { params: { installationId } }) + axiosConfig.get('/GetYearlyReports', { params: { installationId, language: lang } }) .then(res => setYearlyReports(res.data)) .catch(() => {}); axiosConfig.get('/GetPendingMonthlyAggregations', { params: { installationId } }) @@ -238,7 +239,7 @@ function WeeklyReport({ installationId }: WeeklyReportProps) { .catch(() => {}); }; - useEffect(() => { fetchReportData(); }, [installationId]); + useEffect(() => { fetchReportData(); }, [installationId, intl.locale]); const handleGenerateMonthly = async (year: number, month: number) => { setGenerating(`monthly-${year}-${month}`); @@ -277,7 +278,7 @@ function WeeklyReport({ installationId }: WeeklyReportProps) { const safeTab = Math.min(activeTab, tabs.length - 1); return ( - + setActiveTab(v)} @@ -286,10 +287,17 @@ function WeeklyReport({ installationId }: WeeklyReportProps) { {tabs.map(t => )} - {tabs[safeTab]?.key === 'weekly' && ( - - )} - {tabs[safeTab]?.key === 'monthly' && ( + + 0 + ? monthlyReports.reduce((a, b) => a.periodEnd > b.periodEnd ? a : b).periodEnd + : null + } + /> + + - )} - {tabs[safeTab]?.key === 'yearly' && ( + + - )} + ); } // ── Weekly Section (existing weekly report content) ──────────── -function WeeklySection({ installationId }: { installationId: number }) { +function WeeklySection({ installationId, latestMonthlyPeriodEnd }: { installationId: number; latestMonthlyPeriodEnd: string | null }) { const intl = useIntl(); const [report, setReport] = useState(null); const [loading, setLoading] = useState(true); @@ -391,9 +399,13 @@ function WeeklySection({ installationId }: { installationId: number }) { return effective > 0 ? '#27ae60' : effective < 0 ? '#e74c3c' : '#888'; }; + // Compute change % from two raw values; returns 0 (shown as —) if prev is 0 + const calcChange = (curVal: number, prevVal: number) => + prevVal === 0 ? 0 : ((curVal - prevVal) / prevVal) * 100; + const insightBullets = report.aiInsight .split(/\n+/) - .map((line) => line.replace(/^[\d]+[.)]\s*/, '').replace(/^[-*]\s*/, '').trim()) + .map((line) => line.replace(/\*\*/g, '').replace(/^[\d]+[.)]\s*/, '').replace(/^[-*]\s*/, '').trim()) .filter((line) => line.length > 0); const totalEnergySavedKwh = report.totalEnergySaved; @@ -502,13 +514,13 @@ function WeeklySection({ installationId }: { installationId: number }) { {cur.totalGridExport.toFixed(1)} kWh {prev && {prev.totalGridExport.toFixed(1)} kWh} - {prev && —} + {prev && {formatChange(calcChange(cur.totalGridExport, prev.totalGridExport))}} {cur.totalBatteryCharged.toFixed(1)} / {cur.totalBatteryDischarged.toFixed(1)} kWh {prev && {prev.totalBatteryCharged.toFixed(1)} / {prev.totalBatteryDischarged.toFixed(1)} kWh} - {prev && —} + {prev && {formatChange(calcChange(cur.totalBatteryCharged, prev.totalBatteryCharged))}} @@ -557,33 +569,69 @@ function WeeklySection({ installationId }: { installationId: number }) { )} - {/* Weekly History for current month */} - + {/* Weekly History — weeks not yet captured in a monthly report, excluding the current week shown above */} + ); } // ── Weekly History (saved weekly reports for current month) ───── -function WeeklyHistory({ installationId }: { installationId: number }) { +function WeeklyHistory({ installationId, latestMonthlyPeriodEnd, currentReportPeriodEnd }: { + installationId: number; + latestMonthlyPeriodEnd: string | null; + currentReportPeriodEnd: string | null; +}) { const intl = useIntl(); const [records, setRecords] = useState([]); useEffect(() => { - const now = new Date(); - axiosConfig.get('/GetWeeklyReportSummaries', { - params: { installationId, year: now.getFullYear(), month: now.getMonth() + 1 } - }) - .then(res => setRecords(res.data)) - .catch(() => {}); - }, [installationId]); + const now = new Date(); + const year = now.getFullYear(); + const month = now.getMonth() + 1; + const lang = intl.locale; + + // Fetch current month and previous month to cover cross-month weeks + const prevMonth = month === 1 ? 12 : month - 1; + const prevYear = month === 1 ? year - 1 : year; + + Promise.all([ + axiosConfig.get('/GetWeeklyReportSummaries', { params: { installationId, year, month, language: lang } }) + .then(r => r.data as WeeklyReportSummaryRecord[]).catch(() => [] as WeeklyReportSummaryRecord[]), + axiosConfig.get('/GetWeeklyReportSummaries', { params: { installationId, year: prevYear, month: prevMonth, language: lang } }) + .then(r => r.data as WeeklyReportSummaryRecord[]).catch(() => [] as WeeklyReportSummaryRecord[]) + ]).then(([cur, prev]) => { + // Combine, deduplicate by id, sort descending by periodEnd + const all = [...cur, ...prev]; + const unique = Array.from(new Map(all.map(r => [r.id, r])).values()) + .sort((a, b) => b.periodEnd.localeCompare(a.periodEnd)); + + // Exclude the current week already shown as the main report above + const withoutCurrent = currentReportPeriodEnd + ? unique.filter(r => r.periodEnd !== currentReportPeriodEnd) + : unique; + + // Keep only weeks not yet covered by a monthly report + const uncovered = latestMonthlyPeriodEnd + ? withoutCurrent.filter(r => r.periodEnd > latestMonthlyPeriodEnd) + : withoutCurrent; + + // Always show at least the latest non-current week even if all are covered + const toShow = uncovered.length > 0 ? uncovered : withoutCurrent.slice(0, 1); + setRecords(toShow); + }); + }, [installationId, latestMonthlyPeriodEnd, currentReportPeriodEnd, intl.locale]); if (records.length === 0) return null; return ( - + {records.map((rec) => ( @@ -639,15 +687,6 @@ function WeeklyHistory({ installationId }: { installationId: number }) { - {rec.aiInsight && rec.aiInsight.length > 10 && ( - - {rec.aiInsight.split(/\n+/).filter(l => l.trim()).slice(0, 2).map((line, i) => ( - - {line.replace(/^[-*]\s*/, '').replace(/^[\d]+[.)]\s*/, '')} - - ))} - - )} ))} @@ -842,7 +881,7 @@ function AggregatedSection({ const insightBullets = r.aiInsight .split(/\n+/) - .map((line) => line.replace(/^[\d]+[.)]\s*/, '').replace(/^[-*]\s*/, '').trim()) + .map((line) => line.replace(/\*\*/g, '').replace(/^[\d]+[.)]\s*/, '').replace(/^[-*]\s*/, '').trim()) .filter((line) => line.length > 0); const handleSendEmail = async (emailAddress: string) => {