Innovenergy
/
Innovenergy_trunk
3
1
Fork 1
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

store hourly energy records in SQLite and remove file cache

This commit is contained in:
Yinyin Liu 2026-03-02 18:52:36 +01:00
parent 1761914f24
commit 35b64c3318
8 changed files with 162 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
csharp/App/Backend/DataTypes/HourlyEnergyRecord.cs Normal file
View File

@ -0,0 +1,37 @@
using SQLite;
namespace InnovEnergy.App.Backend.DataTypes;
public class HourlyEnergyRecord
{
[PrimaryKey, AutoIncrement]
public Int64 Id { get; set; }
[Indexed]
public Int64 InstallationId { get; set; }
// "YYYY-MM-DD" — used for range queries (same pattern as DailyEnergyRecord)
[Indexed]
public String Date { get; set; } = "";
// 0–23
public Int32 Hour { get; set; }
// "YYYY-MM-DD HH" — used for idempotency check
public String DateHour { get; set; } = "";
public String DayOfWeek { get; set; } = "";
public Boolean IsWeekend { get; set; }
// Energy for this hour (kWh)
public Double PvKwh { get; set; }
public Double LoadKwh { get; set; }
public Double GridImportKwh { get; set; }
public Double BatteryChargedKwh { get; set; }
public Double BatteryDischargedKwh { get; set; }
// Instantaneous state of charge at snapshot time (%)
public Double BattSoC { get; set; }
public String CreatedAt { get; set; } = "";
}

1
csharp/App/Backend/Database/Create.cs
View File

@ -72,6 +72,7 @@ public static partial class Db
public static Boolean Create(MonthlyReportSummary report) => Insert(report);
public static Boolean Create(YearlyReportSummary report) => Insert(report);
public static Boolean Create(DailyEnergyRecord record) => Insert(record);
public static Boolean Create(HourlyEnergyRecord record) => Insert(record);
public static Boolean Create(AiInsightCache cache) => Insert(cache);
public static void HandleAction(UserAction newAction)

4
csharp/App/Backend/Database/Db.cs
View File

@ -29,6 +29,7 @@ public static partial class Db
public static TableQuery<MonthlyReportSummary> MonthlyReports => Connection.Table<MonthlyReportSummary>();
public static TableQuery<YearlyReportSummary> YearlyReports => Connection.Table<YearlyReportSummary>();
public static TableQuery<DailyEnergyRecord> DailyRecords => Connection.Table<DailyEnergyRecord>();
public static TableQuery<HourlyEnergyRecord> HourlyRecords => Connection.Table<HourlyEnergyRecord>();
public static TableQuery<AiInsightCache> AiInsightCaches => Connection.Table<AiInsightCache>();
@ -60,6 +61,7 @@ public static partial class Db
Connection.CreateTable<MonthlyReportSummary>();
Connection.CreateTable<YearlyReportSummary>();
Connection.CreateTable<DailyEnergyRecord>();
Connection.CreateTable<HourlyEnergyRecord>();
Connection.CreateTable<AiInsightCache>();
});
@ -102,6 +104,8 @@ public static partial class Db
fileConnection.CreateTable<MonthlyReportSummary>();
fileConnection.CreateTable<YearlyReportSummary>();
fileConnection.CreateTable<DailyEnergyRecord>();
fileConnection.CreateTable<HourlyEnergyRecord>();
fileConnection.CreateTable<AiInsightCache>();
return fileConnection;
//return CopyDbToMemory(fileConnection);

12
csharp/App/Backend/Database/Delete.cs
View File

@ -200,6 +200,16 @@ public static partial class Db
foreach (var id in oldDailyIds)
DailyRecords.Delete(r => r.Id == id);
// Hourly records older than 3 months (sufficient for pattern detection, 24x more rows than daily)
var hourlyCutoff = DateOnly.FromDateTime(DateTime.UtcNow).AddMonths(-3).ToString("yyyy-MM-dd");
var oldHourlyIds = HourlyRecords
.ToList()
.Where(r => String.Compare(r.Date, hourlyCutoff, StringComparison.Ordinal) < 0)
.Select(r => r.Id)
.ToList();
foreach (var id in oldHourlyIds)
HourlyRecords.Delete(r => r.Id == id);
// Weekly summaries older than 1 year
var oldWeeklyIds = WeeklyReports
.ToList()
@ -231,6 +241,6 @@ public static partial class Db
AiInsightCaches.Delete(c => c.Id == id);
Backup();
Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff}).");
Console.WriteLine($"[Db] Cleanup: {oldDailyIds.Count} daily, {oldHourlyIds.Count} hourly, {oldWeeklyIds.Count} weekly, {oldMonthlyIds.Count} monthly, {oldCacheIds.Count} insight cache records deleted (cutoff {cutoff}).");
}
}

25
csharp/App/Backend/Database/Read.cs
View File

@ -120,6 +120,31 @@ public static partial class Db
=> DailyRecords
.Any(r => r.InstallationId == installationId && r.Date == date);
// ── HourlyEnergyRecord Queries ─────────────────────────────────────
/// <summary>
/// Returns hourly records for an installation within [from, to] inclusive, ordered by date+hour.
/// </summary>
public static List<HourlyEnergyRecord> GetHourlyRecords(Int64 installationId, DateOnly from, DateOnly to)
{
var fromStr = from.ToString("yyyy-MM-dd");
var toStr = to.ToString("yyyy-MM-dd");
return HourlyRecords
.Where(r => r.InstallationId == installationId)
.ToList()
.Where(r => String.Compare(r.Date, fromStr, StringComparison.Ordinal) >= 0
&& String.Compare(r.Date, toStr, StringComparison.Ordinal) <= 0)
.OrderBy(r => r.Date).ThenBy(r => r.Hour)
.ToList();
}
/// <summary>
/// Returns true if an hourly record already exists for this installation+dateHour (idempotency check).
/// </summary>
public static Boolean HourlyRecordExists(Int64 installationId, String dateHour)
=> HourlyRecords
.Any(r => r.InstallationId == installationId && r.DateHour == dateHour);
// ── AiInsightCache Queries ─────────────────────────────────────────
/// <summary>

49
csharp/App/Backend/Services/DailyIngestionService.cs
View File

@ -92,11 +92,13 @@ public static class DailyIngestionService
return;
}
var newCount = 0;
var totalParsed = 0;
var newDailyCount = 0;
var newHourlyCount = 0;
var totalParsed = 0;
foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
{
// Ingest daily records
List<DailyEnergyData> days;
try
{
@ -104,7 +106,7 @@ public static class DailyIngestionService
}
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse {Path.GetFileName(xlsxPath)}: {ex.Message}");
Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
continue;
}
@ -127,11 +129,48 @@ public static class DailyIngestionService
BatteryDischarged = day.BatteryDischarged,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newCount++;
newDailyCount++;
}
// Ingest hourly records
List<HourlyEnergyData> hours;
try
{
hours = ExcelDataParser.ParseHourly(xlsxPath);
}
catch (Exception ex)
{
Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
continue;
}
foreach (var hour in hours)
{
var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
if (Db.HourlyRecordExists(installationId, dateHour))
continue;
Db.Create(new HourlyEnergyRecord
{
InstallationId = installationId,
Date = hour.DateTime.ToString("yyyy-MM-dd"),
Hour = hour.Hour,
DateHour = dateHour,
DayOfWeek = hour.DayOfWeek,
IsWeekend = hour.IsWeekend,
PvKwh = hour.PvKwh,
LoadKwh = hour.LoadKwh,
GridImportKwh = hour.GridImportKwh,
BatteryChargedKwh = hour.BatteryChargedKwh,
BatteryDischargedKwh = hour.BatteryDischargedKwh,
BattSoC = hour.BattSoC,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newHourlyCount++;
}
}
Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newCount} new day(s) ingested ({totalParsed} total across {xlsxFiles.Length} file(s)).");
Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
await Task.CompletedTask;
}
}

4
csharp/App/Backend/Services/ReportAggregationService.cs
View File

@ -688,13 +688,13 @@ Write a concise monthly performance summary in {langName} (4 bullet points, plai
MONTHLY FACTS for {monthName} ({weekCount} days of data):
- PV production: {totalPv:F1} kWh
- Total consumption: {totalConsump:F1} kWh
- Self-sufficiency: {selfSufficiency:F1}% (powered by solar + battery, not grid)
- Self-sufficiency: {selfSufficiency:F1}% (share of energy covered by solar, without drawing from grid)
- Battery: {totalBattChg:F1} kWh charged, {totalBattDis:F1} kWh discharged, efficiency {batteryEff:F1}%
- Energy saved: {energySaved:F1} kWh = ~{savingsCHF:F0} CHF saved (at {ElectricityPriceCHF} CHF/kWh)
INSTRUCTIONS:
1. Savings: state exactly how much energy and money was saved this month. Positive framing.
2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""X% of your home ran on solar + battery""). Do NOT mention raw grid import kWh.
2. Solar performance: how much the solar system produced and what self-sufficiency % means for the homeowner (e.g. ""Your solar system covered X% of your home's energy needs""). Do NOT mention battery here. Do NOT mention raw grid import kWh.
3. Battery: comment on battery utilization and efficiency. If efficiency < 80%, note it may need attention.
4. Tip: one specific actionable suggestion based on the weakest metric: {weakMetric}. If general, suggest the most impactful habit change based on the numbers above.

110
csharp/App/Backend/Services/WeeklyReportService.cs
View File

@ -66,55 +66,30 @@ public static class WeeklyReportService
(prevMon, prevSun) = PreviousCalendarWeek();
}
// Skip file cache when a specific week is requested (avoid stale or wrong-week hits)
var cachePath = weekStartOverride.HasValue
? null
: TmpReportDir + $"{installationId}_{language}_{curMon:yyyy-MM-dd}.cache.json";
// Use cache if it exists and is less than 6 hours old (skipped in override mode)
if (cachePath != null && File.Exists(cachePath))
{
var cacheAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(cachePath);
if (cacheAge.TotalHours < 6)
{
try
{
var cached = JsonConvert.DeserializeObject<WeeklyReportResponse>(
await File.ReadAllTextAsync(cachePath));
if (cached != null)
{
Console.WriteLine($"[WeeklyReportService] Returning cached report for installation {installationId} ({language}), week {curMon:yyyy-MM-dd}.");
return cached;
}
}
catch
{
// Cache corrupt — regenerate
}
}
}
// 1. Try to load daily records from SQLite for the calendar weeks
// 1. Load daily records from SQLite
var currentWeekDays = Db.GetDailyRecords(installationId, curMon, curSun)
.Select(r => ToDailyEnergyData(r)).ToList();
.Select(ToDailyEnergyData).ToList();
var previousWeekDays = Db.GetDailyRecords(installationId, prevMon, prevSun)
.Select(r => ToDailyEnergyData(r)).ToList();
.Select(ToDailyEnergyData).ToList();
// 2. Fallback: if DB empty for current week, parse all xlsx files on the fly
var xlsxFiles = Directory.Exists(TmpReportDir)
? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList()
: new List<String>();
if (currentWeekDays.Count == 0 && xlsxFiles.Count > 0)
// 2. Fallback: if DB empty, parse xlsx on the fly (pre-ingestion scenario)
if (currentWeekDays.Count == 0)
{
Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx.");
var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList();
currentWeekDays = allDaysParsed
.Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; })
.ToList();
previousWeekDays = allDaysParsed
.Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; })
.ToList();
var xlsxFiles = Directory.Exists(TmpReportDir)
? Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx").OrderBy(f => f).ToList()
: new List<String>();
if (xlsxFiles.Count > 0)
{
Console.WriteLine($"[WeeklyReportService] DB empty for week {curMon:yyyy-MM-dd}, falling back to xlsx.");
var allDaysParsed = xlsxFiles.SelectMany(p => ExcelDataParser.Parse(p)).ToList();
currentWeekDays = allDaysParsed
.Where(d => { var date = DateOnly.Parse(d.Date); return date >= curMon && date <= curSun; })
.ToList();
previousWeekDays = allDaysParsed
.Where(d => { var date = DateOnly.Parse(d.Date); return date >= prevMon && date <= prevSun; })
.ToList();
}
}
if (currentWeekDays.Count == 0)
@ -122,36 +97,13 @@ public static class WeeklyReportService
$"No energy data available for week {curMon:yyyy-MM-dd}–{curSun:yyyy-MM-dd}. " +
"Upload an xlsx file or wait for daily ingestion.");
// 3. Load hourly data from ALL xlsx files for behavioral analysis (current week only).
// Combine all files so any week can find its hourly data regardless of file split.
// Future: replace with S3 hourly fetch.
var allHourly = xlsxFiles
.SelectMany(p => { try { return ExcelDataParser.ParseHourly(p); } catch { return Enumerable.Empty<HourlyEnergyData>(); } })
.ToList();
var curMonDt = curMon.ToDateTime(TimeOnly.MinValue);
var curSunDt = curSun.ToDateTime(TimeOnly.MaxValue);
var currentHourlyData = allHourly
.Where(h => h.DateTime >= curMonDt && h.DateTime <= curSunDt)
.ToList();
// 3. Load hourly records from SQLite for behavioral analysis
var currentHourlyData = Db.GetHourlyRecords(installationId, curMon, curSun)
.Select(ToHourlyEnergyData).ToList();
var report = await GenerateReportFromDataAsync(
return await GenerateReportFromDataAsync(
currentWeekDays, previousWeekDays, currentHourlyData, installationName, language,
curMon, curSun);
// Write cache (skipped in override mode)
if (cachePath != null)
{
try
{
await File.WriteAllTextAsync(cachePath, JsonConvert.SerializeObject(report));
}
catch (Exception ex)
{
Console.Error.WriteLine($"[WeeklyReportService] Could not write cache: {ex.Message}");
}
}
return report;
}
// ── Conversion helpers ─────────────────────────────────────────────
@ -167,6 +119,20 @@ public static class WeeklyReportService
BatteryDischarged = r.BatteryDischarged,
};
private static HourlyEnergyData ToHourlyEnergyData(HourlyEnergyRecord r) => new()
{
DateTime = DateTime.ParseExact(r.DateHour, "yyyy-MM-dd HH", null),
Hour = r.Hour,
DayOfWeek = r.DayOfWeek,
IsWeekend = r.IsWeekend,
PvKwh = r.PvKwh,
LoadKwh = r.LoadKwh,
GridImportKwh = r.GridImportKwh,
BatteryChargedKwh = r.BatteryChargedKwh,
BatteryDischargedKwh = r.BatteryDischargedKwh,
BattSoC = r.BattSoC,
};
/// <summary>
/// Core report generation. Accepts pre-split current/previous week data and hourly intervals.
/// weekStart/weekEnd are used to set PeriodStart/PeriodEnd on the response.