store hourly energy records in SQLite and remove file cache

csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs

@@ -0,0 +1,37 @@
+using SQLite;
+
+namespace InnovEnergy.App.Backend.DataTypes;
+
+public class HourlyEnergyRecord
+{
+    [PrimaryKey, AutoIncrement]
+    public Int64   Id                   { get; set; }
+
+    [Indexed]
+    public Int64   InstallationId       { get; set; }
+
+    // "YYYY-MM-DD" — used for range queries (same pattern as DailyEnergyRecord)
+    [Indexed]
+    public String  Date                 { get; set; } = "";
+
+    // 0–23
+    public Int32   Hour                 { get; set; }
+
+    // "YYYY-MM-DD HH" — used for idempotency check
+    public String  DateHour             { get; set; } = "";
+
+    public String  DayOfWeek            { get; set; } = "";
+    public Boolean IsWeekend            { get; set; }
+
+    // Energy for this hour (kWh)
+    public Double  PvKwh                { get; set; }
+    public Double  LoadKwh              { get; set; }
+    public Double  GridImportKwh        { get; set; }
+    public Double  BatteryChargedKwh    { get; set; }
+    public Double  BatteryDischargedKwh { get; set; }
+
+    // Instantaneous state of charge at snapshot time (%)
+    public Double  BattSoC              { get; set; }
+
+    public String  CreatedAt            { get; set; } = "";
+}

csharp/App/Backend/Database/Create.cs

@@ -72,6 +72,7 @@ public static partial class Db
     public static Boolean Create(MonthlyReportSummary report) => Insert(report);
     public static Boolean Create(YearlyReportSummary report)  => Insert(report);
     public static Boolean Create(DailyEnergyRecord record)    => Insert(record);
+    public static Boolean Create(HourlyEnergyRecord record)   => Insert(record);
     public static Boolean Create(AiInsightCache cache)        => Insert(cache);
     
     public static void HandleAction(UserAction newAction)

csharp/App/Backend/Database/Db.cs

@@ -29,6 +29,7 @@ public static partial class Db
     public static TableQuery<MonthlyReportSummary> MonthlyReports => Connection.Table<MonthlyReportSummary>();
     public static TableQuery<YearlyReportSummary>  YearlyReports  => Connection.Table<YearlyReportSummary>();
     public static TableQuery<DailyEnergyRecord>    DailyRecords      => Connection.Table<DailyEnergyRecord>();
+    public static TableQuery<HourlyEnergyRecord>   HourlyRecords     => Connection.Table<HourlyEnergyRecord>();
     public static TableQuery<AiInsightCache>       AiInsightCaches   => Connection.Table<AiInsightCache>();
 
 
@@ -60,6 +61,7 @@ public static partial class Db
             Connection.CreateTable<MonthlyReportSummary>();
             Connection.CreateTable<YearlyReportSummary>();
             Connection.CreateTable<DailyEnergyRecord>();
+            Connection.CreateTable<HourlyEnergyRecord>();
             Connection.CreateTable<AiInsightCache>();
         });
         
@@ -102,6 +104,8 @@ public static partial class Db
         fileConnection.CreateTable<MonthlyReportSummary>();
         fileConnection.CreateTable<YearlyReportSummary>();
         fileConnection.CreateTable<DailyEnergyRecord>();
+        fileConnection.CreateTable<HourlyEnergyRecord>();
+        fileConnection.CreateTable<AiInsightCache>();
 
         return fileConnection;
         //return CopyDbToMemory(fileConnection);

csharp/App/Backend/Database/Delete.cs

@@ -200,6 +200,16 @@ public static partial class Db
         foreach (var id in oldDailyIds)
             DailyRecords.Delete(r => r.Id == id);
 
+        // Hourly records older than 3 months (sufficient for pattern detection, 24x more rows than daily)
+        var hourlyCutoff = DateOnly.FromDateTime(DateTime.UtcNow).AddMonths(-3).ToString("yyyy-MM-dd");
+        var oldHourlyIds = HourlyRecords
+            .ToList()
+            .Where(r => String.Compare(r.Date, hourlyCutoff, StringComparison.Ordinal) < 0)
+            .Select(r => r.Id)
+            .ToList();
+        foreach (var id in oldHourlyIds)
+            HourlyRecords.Delete(r => r.Id == id);
+
         // Weekly summaries older than 1 year
         var oldWeeklyIds = WeeklyReports
             .ToList()
@@ -231,6 +241,6 @@ public static partial class Db
             AiInsightCaches.Delete(c => c.Id == id);
 
         Backup();
-        Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff}).");
+        Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldHourlyIds.Count} hourly, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff}).");
     }
 }

csharp/App/Backend/Database/Read.cs

@@ -120,6 +120,31 @@ public static partial class Db
         => DailyRecords
             .Any(r => r.InstallationId == installationId && r.Date == date);
 
+    // ── HourlyEnergyRecord Queries ─────────────────────────────────────
+
+    /// <summary>
+    /// Returns hourly records for an installation within [from, to] inclusive, ordered by date+hour.
+    /// </summary>
+    public static List<HourlyEnergyRecord> GetHourlyRecords(Int64 installationId, DateOnly from, DateOnly to)
+    {
+        var fromStr = from.ToString("yyyy-MM-dd");
+        var toStr   = to.ToString("yyyy-MM-dd");
+        return HourlyRecords
+            .Where(r => r.InstallationId == installationId)
+            .ToList()
+            .Where(r => String.Compare(r.Date, fromStr, StringComparison.Ordinal) >= 0
+                     && String.Compare(r.Date, toStr,   StringComparison.Ordinal) <= 0)
+            .OrderBy(r => r.Date).ThenBy(r => r.Hour)
+            .ToList();
+    }
+
+    /// <summary>
+    /// Returns true if an hourly record already exists for this installation+dateHour (idempotency check).
+    /// </summary>
+    public static Boolean HourlyRecordExists(Int64 installationId, String dateHour)
+        => HourlyRecords
+            .Any(r => r.InstallationId == installationId && r.DateHour == dateHour);
+
     // ── AiInsightCache Queries ─────────────────────────────────────────
 
     /// <summary>

csharp/App/Backend/Services/DailyIngestionService.cs

@@ -92,11 +92,13 @@ public static class DailyIngestionService
             return;
         }
 
-        var newCount    = 0;
+        var newDailyCount  = 0;
-        var totalParsed = 0;
+        var newHourlyCount = 0;
+        var totalParsed    = 0;
 
         foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
         {
+            // Ingest daily records
             List<DailyEnergyData> days;
             try
             {
@@ -104,7 +106,7 @@ public static class DailyIngestionService
             }
             catch (Exception ex)
             {
-                Console.Error.WriteLine($"[DailyIngestion] Failed to parse {Path.GetFileName(xlsxPath)}: {ex.Message}");
+                Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
                 continue;
             }
 
@@ -127,11 +129,48 @@ public static class DailyIngestionService
                     BatteryDischarged = day.BatteryDischarged,
                     CreatedAt         = DateTime.UtcNow.ToString("o"),
                 });
-                newCount++;
+                newDailyCount++;
+            }
+
+            // Ingest hourly records
+            List<HourlyEnergyData> hours;
+            try
+            {
+                hours = ExcelDataParser.ParseHourly(xlsxPath);
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
+                continue;
+            }
+
+            foreach (var hour in hours)
+            {
+                var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
+                if (Db.HourlyRecordExists(installationId, dateHour))
+                    continue;
+
+                Db.Create(new HourlyEnergyRecord
+                {
+                    InstallationId       = installationId,
+                    Date                 = hour.DateTime.ToString("yyyy-MM-dd"),
+                    Hour                 = hour.Hour,
+                    DateHour             = dateHour,
+                    DayOfWeek            = hour.DayOfWeek,
+                    IsWeekend            = hour.IsWeekend,
+                    PvKwh                = hour.PvKwh,
+                    LoadKwh              = hour.LoadKwh,
+                    GridImportKwh        = hour.GridImportKwh,
+                    BatteryChargedKwh    = hour.BatteryChargedKwh,
+                    BatteryDischargedKwh = hour.BatteryDischargedKwh,
+                    BattSoC              = hour.BattSoC,
+                    CreatedAt            = DateTime.UtcNow.ToString("o"),
+                });
+                newHourlyCount++;
             }
         }
 
-        Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newCount} new day(s) ingested ({totalParsed} total across {xlsxFiles.Length} file(s)).");
+        Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
         await Task.CompletedTask;
     }
 }

csharp/App/Backend/Services/ReportAggregationService.cs

@@ -688,13 +688,13 @@ Write a concise monthly performance summary in {langName} (4 bullet points, plai
 MONTHLY FACTS for {monthName} ({weekCount} days of data):
 - PV production: {totalPv:F1} kWh
 - Total consumption: {totalConsump:F1} kWh
-- Self-sufficiency: {selfSufficiency:F1}% (powered by solar + battery, not grid)
+- Self-sufficiency: {selfSufficiency:F1}% (share of energy covered by solar, without drawing from grid)
 - Battery: {totalBattChg:F1} kWh charged, {totalBattDis:F1} kWh discharged, efficiency {batteryEff:F1}%
 - Energy saved: {energySaved:F1} kWh = ~{savingsCHF:F0} CHF saved (at {ElectricityPriceCHF} CHF/kWh)
 
 INSTRUCTIONS:
 1. Savings: state exactly how much energy and money was saved this month. Positive framing.
-2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""X% of your home ran on solar + battery""). Do NOT mention raw grid import kWh.
+2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""Your solar system covered X% of your home's energy needs""). Do NOT mention battery here. Do NOT mention raw grid import kWh.
 3. Battery: comment on battery utilization and efficiency. If efficiency < 80%, note it may need attention.
 4. Tip: one specific actionable suggestion based on the weakest metric: {weakMetric}. If general, suggest the most impactful habit change based on the numbers above.
 

csharp/App/Backend/Services/WeeklyReportService.cs

@@ -66,55 +66,30 @@ public static class WeeklyReportService
             (prevMon, prevSun) = PreviousCalendarWeek();
         }
 
-        // Skip file cache when a specific week is requested (avoid stale or wrong-week hits)
+        // 1. Load daily records from SQLite
-        var cachePath = weekStartOverride.HasValue
-            ? null
-            : TmpReportDir + $"{installationId}_{language}_{curMon:yyyy-MM-dd}.cache.json";
-
-        // Use cache if it exists and is less than 6 hours old (skipped in override mode)
-        if (cachePath != null && File.Exists(cachePath))
-        {
-            var cacheAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(cachePath);
-            if (cacheAge.TotalHours < 6)
-            {
-                try
-                {
-                    var cached = JsonConvert.DeserializeObject<WeeklyReportResponse>(
-                        await File.ReadAllTextAsync(cachePath));
-                    if (cached != null)
-                    {
-                        Console.WriteLine($"[WeeklyReportService] Returning cached report for installation {installationId} ({language}), week {curMon:yyyy-MM-dd}.");
-                        return cached;
-                    }
-                }
-                catch
-                {
-                    // Cache corrupt — regenerate
-                }
-            }
-        }
-
-        // 1. Try to load daily records from SQLite for the calendar weeks
         var currentWeekDays  = Db.GetDailyRecords(installationId, curMon, curSun)
-            .Select(r => ToDailyEnergyData(r)).ToList();
+            .Select(ToDailyEnergyData).ToList();
         var previousWeekDays = Db.GetDailyRecords(installationId, prevMon, prevSun)
-            .Select(r => ToDailyEnergyData(r)).ToList();
+            .Select(ToDailyEnergyData).ToList();
 
-        // 2. Fallback: if DB empty for current week, parse all xlsx files on the fly
+        // 2. Fallback: if DB empty, parse xlsx on the fly (pre-ingestion scenario)
-        var xlsxFiles = Directory.Exists(TmpReportDir)
+        if (currentWeekDays.Count == 0)
-            ? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList()
-            : new List<String>();
-
-        if (currentWeekDays.Count == 0 && xlsxFiles.Count > 0)
         {
-            Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx.");
+            var xlsxFiles = Directory.Exists(TmpReportDir)
-            var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList();
+                ? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList()
-            currentWeekDays  = allDaysParsed
+                : new List<String>();
-                .Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; })
+
-                .ToList();
+            if (xlsxFiles.Count > 0)
-            previousWeekDays = allDaysParsed
+            {
-                .Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; })
+                Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx.");
-                .ToList();
+                var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList();
+                currentWeekDays  = allDaysParsed
+                    .Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; })
+                    .ToList();
+                previousWeekDays = allDaysParsed
+                    .Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; })
+                    .ToList();
+            }
         }
 
         if (currentWeekDays.Count == 0)
@@ -122,36 +97,13 @@ public static class WeeklyReportService
                 $"No energy data available for week {curMon:yyyy-MM-dd}–{curSun:yyyy-MM-dd}. " +
                 "Upload an xlsx file or wait for daily ingestion.");
 
-        // 3. Load hourly data from ALL xlsx files for behavioral analysis (current week only).
+        // 3. Load hourly records from SQLite for behavioral analysis
-        //    Combine all files so any week can find its hourly data regardless of file split.
+        var currentHourlyData = Db.GetHourlyRecords(installationId, curMon, curSun)
-        //    Future: replace with S3 hourly fetch.
+            .Select(ToHourlyEnergyData).ToList();
-        var allHourly = xlsxFiles
-            .SelectMany(p => { try { return ExcelDataParser.ParseHourly(p); } catch { return Enumerable.Empty<HourlyEnergyData>(); } })
-            .ToList();
-        var curMonDt          = curMon.ToDateTime(TimeOnly.MinValue);
-        var curSunDt          = curSun.ToDateTime(TimeOnly.MaxValue);
-        var currentHourlyData = allHourly
-            .Where(h => h.DateTime >= curMonDt && h.DateTime <= curSunDt)
-            .ToList();
 
-        var report = await GenerateReportFromDataAsync(
+        return await GenerateReportFromDataAsync(
             currentWeekDays, previousWeekDays, currentHourlyData, installationName, language,
             curMon, curSun);
-
-        // Write cache (skipped in override mode)
-        if (cachePath != null)
-        {
-            try
-            {
-                await File.WriteAllTextAsync(cachePath, JsonConvert.SerializeObject(report));
-            }
-            catch (Exception ex)
-            {
-                Console.Error.WriteLine($"[WeeklyReportService] Could not write cache: {ex.Message}");
-            }
-        }
-
-        return report;
     }
 
     // ── Conversion helpers ─────────────────────────────────────────────
@@ -167,6 +119,20 @@ public static class WeeklyReportService
         BatteryDischarged = r.BatteryDischarged,
     };
 
+    private static HourlyEnergyData ToHourlyEnergyData(HourlyEnergyRecord r) => new()
+    {
+        DateTime             = DateTime.ParseExact(r.DateHour, "yyyy-MM-dd HH", null),
+        Hour                 = r.Hour,
+        DayOfWeek            = r.DayOfWeek,
+        IsWeekend            = r.IsWeekend,
+        PvKwh                = r.PvKwh,
+        LoadKwh              = r.LoadKwh,
+        GridImportKwh        = r.GridImportKwh,
+        BatteryChargedKwh    = r.BatteryChargedKwh,
+        BatteryDischargedKwh = r.BatteryDischargedKwh,
+        BattSoC              = r.BattSoC,
+    };
+
     /// <summary>
     /// Core report generation. Accepts pre-split current/previous week data and hourly intervals.
     /// weekStart/weekEnd are used to set PeriodStart/PeriodEnd on the response.