Innovenergy
/
Innovenergy_trunk
3
1
Fork 1
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

digested hourly and daily data from S3 for Sinexcel

This commit is contained in:
Yinyin Liu 2026-03-11 11:43:55 +01:00
parent 27b84a0d46
commit 1306ae6b9f
3 changed files with 377 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
csharp/App/Backend/Controller.cs
View File

@ -1257,7 +1257,7 @@ public class Controller : ControllerBase
/// <summary> /// <summary>
/// Returns daily + hourly records for a date range. /// Returns daily + hourly records for a date range.
/// DB first; if empty, falls back to xlsx parsing and caches results for future calls. /// Fallback chain: DB → JSON (local + S3) → xlsx. Caches to DB on first parse.
/// </summary> /// </summary>
[HttpGet(nameof(GetDailyDetailRecords))] [HttpGet(nameof(GetDailyDetailRecords))]
public ActionResult GetDailyDetailRecords( public ActionResult GetDailyDetailRecords(
@ -1279,73 +1279,107 @@ public class Controller : ControllerBase
var dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate); var dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
var hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate); var hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
// 2. Fallback: parse xlsx + cache to DB if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
if (dailyRecords.Count == 0 || hourlyRecords.Count == 0) return Ok(FormatResult(dailyRecords, hourlyRecords));
// 2. Fallback: try JSON (local files + S3)
TryIngestFromJson(installationId, installation, fromDate, toDate);
dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
if (dailyRecords.Count > 0 && hourlyRecords.Count > 0)
return Ok(FormatResult(dailyRecords, hourlyRecords));
// 3. Fallback: parse xlsx + cache to DB
TryIngestFromXlsx(installationId, fromDate, toDate);
dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
return Ok(FormatResult(dailyRecords, hourlyRecords));
}
private static Object FormatResult(
List<DailyEnergyRecord> daily, List<HourlyEnergyRecord> hourly) => new
{
dailyRecords = new { count = daily.Count, records = daily },
hourlyRecords = new { count = hourly.Count, records = hourly },
};
private static void TryIngestFromJson(
Int64 installationId, Installation installation,
DateOnly fromDate, DateOnly toDate)
{
var jsonDir = Path.Combine(
Environment.CurrentDirectory, "tmp_report", "aggregated", installationId.ToString());
for (var date = fromDate; date <= toDate; date = date.AddDays(1))
{ {
var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate); var isoDate = date.ToString("yyyy-MM-dd");
if (xlsxFiles.Count > 0) var fileName = AggregatedJsonParser.ToJsonFileName(date);
// Try local file first
var localPath = Path.Combine(jsonDir, fileName);
String? content = System.IO.File.Exists(localPath) ? System.IO.File.ReadAllText(localPath) : null;
// Try S3 if no local file
content ??= AggregatedJsonParser.TryReadFromS3(installation, isoDate)
.GetAwaiter().GetResult();
if (content is null) continue;
DailyIngestionService.IngestJsonContent(installationId, content);
}
}
private static void TryIngestFromXlsx(
Int64 installationId, DateOnly fromDate, DateOnly toDate)
{
var xlsxFiles = WeeklyReportService.GetRelevantXlsxFiles(installationId, fromDate, toDate);
if (xlsxFiles.Count == 0) return;
foreach (var xlsxPath in xlsxFiles)
{
foreach (var day in ExcelDataParser.Parse(xlsxPath))
{ {
foreach (var xlsxPath in xlsxFiles) if (Db.DailyRecordExists(installationId, day.Date))
continue;
Db.Create(new DailyEnergyRecord
{ {
if (dailyRecords.Count == 0) InstallationId = installationId,
{ Date = day.Date,
foreach (var day in ExcelDataParser.Parse(xlsxPath)) PvProduction = day.PvProduction,
{ LoadConsumption = day.LoadConsumption,
if (Db.DailyRecordExists(installationId, day.Date)) GridImport = day.GridImport,
continue; GridExport = day.GridExport,
Db.Create(new DailyEnergyRecord BatteryCharged = day.BatteryCharged,
{ BatteryDischarged = day.BatteryDischarged,
InstallationId = installationId, CreatedAt = DateTime.UtcNow.ToString("o"),
Date = day.Date, });
PvProduction = day.PvProduction, }
LoadConsumption = day.LoadConsumption,
GridImport = day.GridImport,
GridExport = day.GridExport,
BatteryCharged = day.BatteryCharged,
BatteryDischarged = day.BatteryDischarged,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
}
}
if (hourlyRecords.Count == 0) foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath))
{ {
foreach (var hour in ExcelDataParser.ParseHourly(xlsxPath)) var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
{ if (Db.HourlyRecordExists(installationId, dateHour))
var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}"; continue;
if (Db.HourlyRecordExists(installationId, dateHour)) Db.Create(new HourlyEnergyRecord
continue; {
Db.Create(new HourlyEnergyRecord InstallationId = installationId,
{ Date = hour.DateTime.ToString("yyyy-MM-dd"),
InstallationId = installationId, Hour = hour.Hour,
Date = hour.DateTime.ToString("yyyy-MM-dd"), DateHour = dateHour,
Hour = hour.Hour, DayOfWeek = hour.DayOfWeek,
DateHour = dateHour, IsWeekend = hour.IsWeekend,
DayOfWeek = hour.DayOfWeek, PvKwh = hour.PvKwh,
IsWeekend = hour.IsWeekend, LoadKwh = hour.LoadKwh,
PvKwh = hour.PvKwh, GridImportKwh = hour.GridImportKwh,
LoadKwh = hour.LoadKwh, BatteryChargedKwh = hour.BatteryChargedKwh,
GridImportKwh = hour.GridImportKwh, BatteryDischargedKwh = hour.BatteryDischargedKwh,
BatteryChargedKwh = hour.BatteryChargedKwh, BattSoC = hour.BattSoC,
BatteryDischargedKwh = hour.BatteryDischargedKwh, CreatedAt = DateTime.UtcNow.ToString("o"),
BattSoC = hour.BattSoC, });
CreatedAt = DateTime.UtcNow.ToString("o"),
});
}
}
}
// Re-read from DB (now cached)
dailyRecords = Db.GetDailyRecords(installationId, fromDate, toDate);
hourlyRecords = Db.GetHourlyRecords(installationId, fromDate, toDate);
} }
} }
return Ok(new
{
dailyRecords = new { count = dailyRecords.Count, records = dailyRecords },
hourlyRecords = new { count = hourlyRecords.Count, records = hourlyRecords },
});
} }
/// <summary> /// <summary>

162
csharp/App/Backend/Services/AggregatedJsonParser.cs Normal file
View File

@ -0,0 +1,162 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using InnovEnergy.App.Backend.DataTypes;
using InnovEnergy.App.Backend.DataTypes.Methods;
using InnovEnergy.Lib.S3Utils;
using InnovEnergy.Lib.S3Utils.DataTypes;
using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
namespace InnovEnergy.App.Backend.Services;
/// <summary>
/// Parses NDJSON aggregated data files generated by SodistoreHome devices.
/// Each file (DDMMYYYY.json) contains one JSON object per line:
/// - Type "Hourly": per-hour kWh values (already computed, no diffing needed)
/// - Type "Daily": daily totals
/// </summary>
public static class AggregatedJsonParser
{
private static readonly JsonSerializerOptions JsonOpts = new()
{
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString,
};
public static List<DailyEnergyData> ParseDaily(String ndjsonContent)
{
var dailyByDate = new SortedDictionary<String, DailyEnergyData>();
foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
{
if (!line.Contains("\"Type\":\"Daily\""))
continue;
try
{
var raw = JsonSerializer.Deserialize<DailyJsonDto>(line, JsonOpts);
if (raw is null) continue;
var date = raw.Timestamp.ToString("yyyy-MM-dd");
dailyByDate[date] = new DailyEnergyData
{
Date = date,
PvProduction = Math.Round(raw.DailySelfGeneratedElectricity, 4),
GridImport = Math.Round(raw.DailyElectricityPurchased, 4),
GridExport = Math.Round(raw.DailyElectricityFed, 4),
BatteryCharged = Math.Round(raw.BatteryDailyChargeEnergy, 4),
BatteryDischarged = Math.Round(raw.BatteryDailyDischargeEnergy, 4),
LoadConsumption = Math.Round(raw.DailyLoadPowerConsumption, 4),
};
}
catch (Exception ex)
{
Console.Error.WriteLine($"[AggregatedJsonParser] Skipping daily line: {ex.Message}");
}
}
Console.WriteLine($"[AggregatedJsonParser] Parsed {dailyByDate.Count} daily record(s)");
return dailyByDate.Values.ToList();
}
public static List<HourlyEnergyData> ParseHourly(String ndjsonContent)
{
var result = new List<HourlyEnergyData>();
foreach (var line in ndjsonContent.Split('\n', StringSplitOptions.RemoveEmptyEntries))
{
if (!line.Contains("\"Type\":\"Hourly\""))
continue;
try
{
var raw = JsonSerializer.Deserialize<HourlyJsonDto>(line, JsonOpts);
if (raw is null) continue;
var dt = new DateTime(
raw.Timestamp.Year, raw.Timestamp.Month, raw.Timestamp.Day,
raw.Timestamp.Hour, 0, 0);
result.Add(new HourlyEnergyData
{
DateTime = dt,
Hour = dt.Hour,
DayOfWeek = dt.DayOfWeek.ToString(),
IsWeekend = dt.DayOfWeek is System.DayOfWeek.Saturday or System.DayOfWeek.Sunday,
PvKwh = Math.Round(raw.SelfGeneratedElectricity, 4),
GridImportKwh = Math.Round(raw.ElectricityPurchased, 4),
BatteryChargedKwh = Math.Round(raw.BatteryChargeEnergy, 4),
BatteryDischargedKwh = Math.Round(raw.BatteryDischargeEnergy, 4),
LoadKwh = Math.Round(raw.LoadPowerConsumption, 4),
});
}
catch (Exception ex)
{
Console.Error.WriteLine($"[AggregatedJsonParser] Skipping hourly line: {ex.Message}");
}
}
Console.WriteLine($"[AggregatedJsonParser] Parsed {result.Count} hourly record(s)");
return result;
}
/// <summary>
/// Converts ISO date "yyyy-MM-dd" to device filename format "ddMMyyyy".
/// </summary>
public static String ToJsonFileName(String isoDate)
{
var d = DateOnly.ParseExact(isoDate, "yyyy-MM-dd");
return d.ToString("ddMMyyyy") + ".json";
}
public static String ToJsonFileName(DateOnly date) => date.ToString("ddMMyyyy") + ".json";
/// <summary>
/// Tries to read an aggregated JSON file from the installation's S3 bucket.
/// S3 key: DDMMYYYY.json (directly in bucket root).
/// Returns file content or null if not found / error.
/// </summary>
public static async Task<String?> TryReadFromS3(Installation installation, String isoDate)
{
try
{
var fileName = ToJsonFileName(isoDate);
var region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!);
var bucket = region.Bucket(installation.BucketName());
var s3Url = bucket.Path(fileName);
return await s3Url.GetObjectAsString();
}
catch (Exception ex)
{
Console.Error.WriteLine($"[AggregatedJsonParser] S3 read failed for {isoDate}: {ex.Message}");
return null;
}
}
// --- JSON DTOs ---
private sealed class HourlyJsonDto
{
public String Type { get; set; } = "";
public DateTime Timestamp { get; set; }
public Double SelfGeneratedElectricity { get; set; }
public Double ElectricityPurchased { get; set; }
public Double ElectricityFed { get; set; }
public Double BatteryChargeEnergy { get; set; }
public Double BatteryDischargeEnergy { get; set; }
public Double LoadPowerConsumption { get; set; }
}
private sealed class DailyJsonDto
{
public String Type { get; set; } = "";
public DateTime Timestamp { get; set; }
public Double DailySelfGeneratedElectricity { get; set; }
public Double DailyElectricityPurchased { get; set; }
public Double DailyElectricityFed { get; set; }
public Double BatteryDailyChargeEnergy { get; set; }
public Double BatteryDailyDischargeEnergy { get; set; }
public Double DailyLoadPowerConsumption { get; set; }
}
}

140
csharp/App/Backend/Services/DailyIngestionService.cs
View File

@ -4,12 +4,8 @@ using InnovEnergy.App.Backend.DataTypes;
namespace InnovEnergy.App.Backend.Services; namespace InnovEnergy.App.Backend.Services;
/// <summary> /// <summary>
/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table. /// Ingests daily energy totals into the DailyEnergyRecord SQLite table.
/// This is the source-of-truth population step for the report pipeline. /// Data source priority: JSON (local) → JSON (S3) → xlsx fallback.
///
/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx
/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor)
///
/// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the /// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the
/// IngestDailyData API endpoint. /// IngestDailyData API endpoint.
/// </summary> /// </summary>
@ -18,6 +14,9 @@ public static class DailyIngestionService
private static readonly String TmpReportDir = private static readonly String TmpReportDir =
Environment.CurrentDirectory + "/tmp_report/"; Environment.CurrentDirectory + "/tmp_report/";
private static readonly String JsonAggregatedDir =
Environment.CurrentDirectory + "/tmp_report/aggregated/";
private static Timer? _dailyTimer; private static Timer? _dailyTimer;
/// <summary> /// <summary>
@ -73,11 +72,119 @@ public static class DailyIngestionService
} }
/// <summary> /// <summary>
/// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores /// Ingests data for one installation. Tries JSON (local + S3) and xlsx.
/// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per /// Both sources are tried — idempotency checks prevent duplicates.
/// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent. /// JSON provides recent data; xlsx provides historical data.
/// </summary> /// </summary>
public static async Task IngestInstallationAsync(Int64 installationId) public static async Task IngestInstallationAsync(Int64 installationId)
{
await TryIngestFromJson(installationId);
IngestFromXlsx(installationId);
}
private static async Task<Boolean> TryIngestFromJson(Int64 installationId)
{
var newDaily = 0;
var newHourly = 0;
var jsonDir = Path.Combine(JsonAggregatedDir, installationId.ToString());
// Collect JSON content from local files
var jsonFiles = Directory.Exists(jsonDir)
? Directory.GetFiles(jsonDir, "*.json")
: Array.Empty<String>();
foreach (var jsonPath in jsonFiles.OrderBy(f => f))
{
var content = File.ReadAllText(jsonPath);
var (d, h) = IngestJsonContent(installationId, content);
newDaily += d;
newHourly += h;
}
// Also try S3 for recent days (yesterday + today) if no local files found
if (jsonFiles.Length == 0)
{
var installation = Db.GetInstallationById(installationId);
if (installation is not null)
{
for (var daysBack = 0; daysBack <= 1; daysBack++)
{
var date = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-daysBack));
var isoDate = date.ToString("yyyy-MM-dd");
if (Db.DailyRecordExists(installationId, isoDate))
continue;
var content = await AggregatedJsonParser.TryReadFromS3(installation, isoDate);
if (content is null) continue;
var (d, h) = IngestJsonContent(installationId, content);
newDaily += d;
newHourly += h;
}
}
}
if (newDaily > 0 || newHourly > 0)
Console.WriteLine($"[DailyIngestion] Installation {installationId} (JSON): {newDaily} day(s), {newHourly} hour(s) ingested.");
return newDaily > 0 || newHourly > 0;
}
public static (Int32 daily, Int32 hourly) IngestJsonContent(Int64 installationId, String content)
{
var newDaily = 0;
var newHourly = 0;
foreach (var day in AggregatedJsonParser.ParseDaily(content))
{
if (Db.DailyRecordExists(installationId, day.Date))
continue;
Db.Create(new DailyEnergyRecord
{
InstallationId = installationId,
Date = day.Date,
PvProduction = day.PvProduction,
LoadConsumption = day.LoadConsumption,
GridImport = day.GridImport,
GridExport = day.GridExport,
BatteryCharged = day.BatteryCharged,
BatteryDischarged = day.BatteryDischarged,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newDaily++;
}
foreach (var hour in AggregatedJsonParser.ParseHourly(content))
{
var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
if (Db.HourlyRecordExists(installationId, dateHour))
continue;
Db.Create(new HourlyEnergyRecord
{
InstallationId = installationId,
Date = hour.DateTime.ToString("yyyy-MM-dd"),
Hour = hour.Hour,
DateHour = dateHour,
DayOfWeek = hour.DayOfWeek,
IsWeekend = hour.IsWeekend,
PvKwh = hour.PvKwh,
LoadKwh = hour.LoadKwh,
GridImportKwh = hour.GridImportKwh,
BatteryChargedKwh = hour.BatteryChargedKwh,
BatteryDischargedKwh = hour.BatteryDischargedKwh,
BattSoC = 0,
CreatedAt = DateTime.UtcNow.ToString("o"),
});
newHourly++;
}
return (newDaily, newHourly);
}
private static void IngestFromXlsx(Int64 installationId)
{ {
if (!Directory.Exists(TmpReportDir)) if (!Directory.Exists(TmpReportDir))
{ {
@ -98,12 +205,8 @@ public static class DailyIngestionService
foreach (var xlsxPath in xlsxFiles.OrderBy(f => f)) foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
{ {
// Ingest daily records
List<DailyEnergyData> days; List<DailyEnergyData> days;
try try { days = ExcelDataParser.Parse(xlsxPath); }
{
days = ExcelDataParser.Parse(xlsxPath);
}
catch (Exception ex) catch (Exception ex)
{ {
Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}"); Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
@ -132,12 +235,8 @@ public static class DailyIngestionService
newDailyCount++; newDailyCount++;
} }
// Ingest hourly records
List<HourlyEnergyData> hours; List<HourlyEnergyData> hours;
try try { hours = ExcelDataParser.ParseHourly(xlsxPath); }
{
hours = ExcelDataParser.ParseHourly(xlsxPath);
}
catch (Exception ex) catch (Exception ex)
{ {
Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}"); Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
@ -170,7 +269,6 @@ public static class DailyIngestionService
} }
} }
Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s))."); Console.WriteLine($"[DailyIngestion] Installation {installationId} (xlsx): {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
await Task.CompletedTask;
} }
} }