From 807882b96019b37e3f92f744a3fcd416001ff202 Mon Sep 17 00:00:00 2001 From: Yinyin Liu Date: Fri, 12 Jun 2026 13:05:43 +0200 Subject: [PATCH] get price data point every 15 min --- csharp/App/Backend/Controller.cs | 28 ++ .../Services/CurrentPriceHistoryService.cs | 245 ++++++++++++++++++ .../CurrentPriceChart.tsx | 4 +- .../SodiohomeInstallations/Installation.tsx | 1 - .../SodistoreHomeConfigurationV2.tsx | 7 +- .../currentPriceData.ts | 81 ++---- 6 files changed, 295 insertions(+), 71 deletions(-) create mode 100644 csharp/App/Backend/Services/CurrentPriceHistoryService.cs diff --git a/csharp/App/Backend/Controller.cs b/csharp/App/Backend/Controller.cs index 79f8e228b..67934b147 100644 --- a/csharp/App/Backend/Controller.cs +++ b/csharp/App/Backend/Controller.cs @@ -415,6 +415,34 @@ public class Controller : ControllerBase return sampledTimestamps; } + // Dynamic-pricing "Current Price" history (CHF/kWh) for the Configuration tab chart. + // Only Growatt (device 3) and Sinexcel (device 4) carry Config.CurrentPrice. + [HttpGet(nameof(GetCurrentPriceHistory))] + public async Task>> GetCurrentPriceHistory(Int64 id, Int64 start, Int64 end, Token authToken) + { + var user = Db.GetSession(authToken)?.User; + if (user == null) + return Unauthorized(); + + var installation = Db.GetInstallationById(id); + if (installation is null || !user.HasAccessTo(installation)) + return Unauthorized(); + + if (installation.Device != 3 && installation.Device != 4) + return Ok(new List()); + + // Clamp the range: each day spawns s3cmd processes + S3 fetches, so an + // unbounded range from a crafted request could exhaust the single prod backend. + const Int64 maxRangeSeconds = 31L * 86400; + if (start < 0 || end <= start) + return Ok(new List()); + if (end - start > maxRangeSeconds) + start = end - maxRangeSeconds; + + var history = await CurrentPriceHistoryService.GetHistory(installation, start, end); + return Ok(history); + } + [HttpGet(nameof(GetUserById))] public ActionResult GetUserById(Int64 id, Token authToken) { diff --git a/csharp/App/Backend/Services/CurrentPriceHistoryService.cs b/csharp/App/Backend/Services/CurrentPriceHistoryService.cs new file mode 100644 index 000000000..d95f2de26 --- /dev/null +++ b/csharp/App/Backend/Services/CurrentPriceHistoryService.cs @@ -0,0 +1,245 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.IO.Compression; +using System.Text.Json; +using System.Text.RegularExpressions; +using InnovEnergy.App.Backend.DataTypes; +using InnovEnergy.App.Backend.DataTypes.Methods; +using InnovEnergy.Lib.S3Utils; +using S3Bucket = InnovEnergy.Lib.S3Utils.DataTypes.S3Bucket; +using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region; + +namespace InnovEnergy.App.Backend.Services; + +/// +/// One data point of the dynamic-pricing "Current Price" history (CHF/kWh). +/// +public class PricePoint +{ + public Int64 Timestamp { get; set; } // unix seconds + public Double Price { get; set; } // CHF/kWh +} + +/// +/// Builds the "Current Price" history shown on the Configuration tab for +/// Growatt (device 3) and Sinexcel (device 4) installations. +/// +/// CurrentPrice only lives inside the per-10-second chunk files (Config.CurrentPrice); +/// there is no pre-aggregated source. To keep load bounded we sample ONE chunk per +/// 15-minute slot, fetch those in parallel, and cache each fully-past day (immutable). +/// +public static class CurrentPriceHistoryService +{ + private const Int64 BucketSeconds = 900; // 15-minute resolution + private const Int32 MaxParallelFetches = 24; + private const Int32 CacheRetentionDays = 14; // bound the in-memory day cache + private const String S3CfgPath = "/home/ubuntu/.s3cfg"; + + private static readonly Regex ChunkFileRegex = new(@"/([0-9]+)\.(csv|json)$", RegexOptions.Compiled); + + // Immutable past days cached as "{installationId}:{yyyyMMdd}". + private static readonly ConcurrentDictionary> DayCache = new(); + + public static async Task> GetHistory(Installation installation, Int64 startSec, Int64 endSec) + { + var todayUtc = DateTime.UtcNow.Date; + var firstDay = DateTimeOffset.FromUnixTimeSeconds(startSec).UtcDateTime.Date; + var lastDay = DateTimeOffset.FromUnixTimeSeconds(endSec).UtcDateTime.Date; + + var points = new List(); + for (var day = firstDay; day <= lastDay; day = day.AddDays(1)) + points.AddRange(await GetDay(installation, day, cacheable: day < todayUtc)); + + return points + .Where(p => p.Timestamp >= startSec && p.Timestamp <= endSec) + .OrderBy(p => p.Timestamp) + .ToList(); + } + + private static async Task> GetDay(Installation installation, DateTime dayUtc, Boolean cacheable) + { + var key = $"{installation.Id}:{dayUtc:yyyyMMdd}"; + if (cacheable && DayCache.TryGetValue(key, out var cached)) + return cached; + + var dayStart = new DateTimeOffset(dayUtc, TimeSpan.Zero).ToUnixTimeSeconds(); + var dayEnd = dayStart + 86400 - 1; + + var timestamps = SampleByBucket(ListChunkTimestamps(installation, dayStart, dayEnd)); + var points = await FetchPrices(installation, timestamps); + + // Only cache non-empty past days: an empty result can mean a transient s3cmd/S3 + // failure, and caching that would serve "no data" forever until restart. + if (cacheable && points.Count > 0) + CacheDay(key, points); + + return points; + } + + private static void CacheDay(String key, List points) + { + DayCache[key] = points; + + // Prune entries older than the retention window to bound memory growth. + var cutoff = DateTime.UtcNow.Date.AddDays(-CacheRetentionDays); + foreach (var existingKey in DayCache.Keys) + { + var datePart = existingKey.Substring(existingKey.IndexOf(':') + 1); + if (DateTime.TryParseExact(datePart, "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var day) && day < cutoff) + DayCache.TryRemove(existingKey, out _); + } + } + + // Keep the first chunk in each 15-minute slot. + private static List SampleByBucket(List timestamps) + { + var seenBuckets = new HashSet(); + var picked = new List(); + foreach (var t in timestamps.OrderBy(x => x)) + if (seenBuckets.Add(t / BucketSeconds)) + picked.Add(t); + return picked; + } + + // List every chunk filename in range via `s3cmd ls` over the 5-digit timestamp prefixes + // (same listing approach as Controller.GetCsvTimestampsForInstallation). + private static List ListChunkTimestamps(Installation installation, Int64 start, Int64 end) + { + var all = new List(); + var startPrefix = Int64.Parse(start.ToString().Substring(0, 5)); + var endPrefix = Int64.Parse(end.ToString().Substring(0, 5)); + + for (var prefix = startPrefix; prefix <= endPrefix; prefix++) + { + var output = RunS3cmdLs("s3://" + installation.BucketName() + "/" + prefix); + foreach (var line in output.Split('\n')) + { + var match = ChunkFileRegex.Match(line); + if (match.Success && Int64.TryParse(match.Groups[1].Value, out var t) && t >= start && t <= end) + all.Add(t); + } + } + return all; + } + + private static String RunS3cmdLs(String bucketPath) + { + try + { + var startInfo = new ProcessStartInfo + { + FileName = "s3cmd", + Arguments = $"--config {S3CfgPath} ls {bucketPath}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + using var process = new Process { StartInfo = startInfo }; + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + process.StandardError.ReadToEnd(); + process.WaitForExit(); + return process.ExitCode == 0 ? output : ""; + } + catch (Exception e) + { + Console.WriteLine($"[CurrentPriceHistory] s3cmd ls failed for {bucketPath}: {e.Message}"); + return ""; + } + } + + private static async Task> FetchPrices(Installation installation, List timestamps) + { + var region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!); + var bucket = region.Bucket(installation.BucketName()); + + using var gate = new SemaphoreSlim(MaxParallelFetches); + + var tasks = timestamps.Select(async ts => + { + await gate.WaitAsync(); + try + { + var price = await FetchPriceAt(bucket, ts); + return price.HasValue ? new PricePoint { Timestamp = ts, Price = price.Value } : null; + } + finally + { + gate.Release(); + } + }); + + var results = await Task.WhenAll(tasks); + return results.Where(p => p != null).Select(p => p!).OrderBy(p => p.Timestamp).ToList(); + } + + private static async Task FetchPriceAt(S3Bucket bucket, Int64 ts) + { + try + { + var raw = await bucket.Path($"{ts}.json").GetObjectAsString(); + var json = DecodeChunk(raw); + return json == null ? null : ExtractCurrentPrice(json); + } + catch + { + return null; // missing chunk / decode error -> just skip this slot + } + } + + // Chunk objects are Base64-encoded ZIP archives whose inner "data.json" holds the record. + private static String? DecodeChunk(String raw) + { + try + { + var trimmed = raw.Trim(); + if (trimmed.StartsWith('{')) + return raw; // defensive: already plain JSON + + var bytes = Convert.FromBase64String(trimmed); + using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read); + var entry = zip.GetEntry("data.json"); + if (entry == null) + return null; + + using var reader = new StreamReader(entry.Open()); + return reader.ReadToEnd(); + } + catch + { + return null; + } + } + + // data.json is an object keyed by timestamp(s); read Config.CurrentPrice of the last record. + private static Double? ExtractCurrentPrice(String json) + { + using var doc = JsonDocument.Parse(json); + if (doc.RootElement.ValueKind != JsonValueKind.Object) + return null; + + var found = false; + var record = default(JsonElement); + foreach (var prop in doc.RootElement.EnumerateObject()) + { + record = prop.Value; + found = true; + } + + if (!found + || !record.TryGetProperty("Config", out var config) + || !config.TryGetProperty("CurrentPrice", out var currentPrice)) + return null; + + return currentPrice.ValueKind switch + { + JsonValueKind.Number => currentPrice.GetDouble(), + JsonValueKind.String when Double.TryParse(currentPrice.GetString(), + NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed) => parsed, + _ => null + }; + } +} diff --git a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/CurrentPriceChart.tsx b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/CurrentPriceChart.tsx index 6c0be17c9..fb3ada3b3 100644 --- a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/CurrentPriceChart.tsx +++ b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/CurrentPriceChart.tsx @@ -5,7 +5,6 @@ import RestartAltIcon from '@mui/icons-material/RestartAlt'; import { FormattedMessage, useIntl } from 'react-intl'; import ReactApexChart from 'react-apexcharts'; import ApexCharts, { ApexOptions } from 'apexcharts'; -import { I_S3Credentials } from 'src/interfaces/S3Types'; import { UnixTime, TimeSpan } from 'src/dataCache/time'; import { CurrentPriceSeries, @@ -13,7 +12,6 @@ import { } from './currentPriceData'; interface CurrentPriceChartProps { - s3Credentials: I_S3Credentials; id: number; } @@ -32,7 +30,7 @@ function CurrentPriceChart(props: CurrentPriceChartProps) { const end = UnixTime.now(); const start = end.earlier(TimeSpan.fromDays(HISTORY_DAYS)); - fetchCurrentPriceHistory(props.s3Credentials, props.id, start, end) + fetchCurrentPriceHistory(props.id, start, end) .then((result) => { if (!cancelled) { setSeries(result); diff --git a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/Installation.tsx b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/Installation.tsx index e5f06decc..73705dfcd 100644 --- a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/Installation.tsx +++ b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/Installation.tsx @@ -605,7 +605,6 @@ function SodioHomeInstallation(props: singleInstallationProps) { values={values} id={props.current_installation.id} installation={props.current_installation} - s3Credentials={s3Credentials} /> } /> diff --git a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/SodistoreHomeConfigurationV2.tsx b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/SodistoreHomeConfigurationV2.tsx index 08bd76078..86cbbeb9c 100644 --- a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/SodistoreHomeConfigurationV2.tsx +++ b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/SodistoreHomeConfigurationV2.tsx @@ -43,14 +43,12 @@ import { DateTimePicker, TimePicker } from '@mui/x-date-pickers'; import dayjs from 'dayjs'; import Switch from '@mui/material/Switch'; import FormControlLabel from '@mui/material/FormControlLabel'; -import { I_S3Credentials } from 'src/interfaces/S3Types'; import CurrentPriceChart from './CurrentPriceChart'; interface SodistoreHomeConfigurationProps { values: JSONRecordData; id: number; installation: I_Installation; - s3Credentials: I_S3Credentials; } function SodistoreHomeConfigurationV2(props: SodistoreHomeConfigurationProps) { @@ -965,10 +963,7 @@ function SodistoreHomeConfigurationV2(props: SodistoreHomeConfigurationProps) {
- +
diff --git a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/currentPriceData.ts b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/currentPriceData.ts index fa755c5bb..17db791f9 100644 --- a/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/currentPriceData.ts +++ b/typescript/frontend-marios2/src/content/dashboards/SodiohomeInstallations/currentPriceData.ts @@ -1,14 +1,10 @@ import { AxiosResponse } from 'axios'; import axiosConfig from 'src/Resources/axiosConfig'; -import { I_S3Credentials } from 'src/interfaces/S3Types'; import { UnixTime } from 'src/dataCache/time'; -import { FetchResult } from 'src/dataCache/dataCache'; -import { fetchDataJson } from 'src/content/dashboards/Installations/fetchData'; // History of the `/Config/CurrentPrice` value (CHF/kWh) for the Configuration tab chart. -// Reuses the same plumbing as the battery charts: the backend -// (GetCsvTimestampsForInstallation) already downsamples to ~100 timestamps for any -// range, so a week-long fetch stays at ~100 S3 chunk reads. +// The backend GetCurrentPriceHistory endpoint reads the per-15-minute price from S3 +// chunks server-side and returns ready-to-plot points, so the browser makes one call. export interface CurrentPriceSeries { // [timestampMs, price] — timestampMs is local-time-shifted like the other charts @@ -17,47 +13,15 @@ export interface CurrentPriceSeries { max: number; } +interface PricePointDto { + timestamp: number; // unix seconds (UTC) + price: number; // CHF/kWh +} + const EMPTY: CurrentPriceSeries = { data: [], min: 0, max: 0 }; -// Fetch one chunk, retrying the next second (chunks land on even/odd seconds). -const fetchChunk = async ( - ticks: number, - s3Credentials: I_S3Credentials -): Promise | null> => { - for (let i = 0; i < 2; i++) { - try { - const res = await fetchDataJson( - UnixTime.fromTicks(ticks + i, true), - s3Credentials - ); - if ( - res && - res !== FetchResult.notAvailable && - res !== FetchResult.tryLater - ) { - return res as Record; - } - } catch { - // ignore and try next second / next timestamp - } - } - return null; -}; - -const extractPrice = (chunk: Record): number | null => { - const keys = Object.keys(chunk); - if (keys.length === 0) return null; - const record = chunk[keys[keys.length - 1]]; - const price = record?.Config?.CurrentPrice; - if (price === undefined || price === null || Number.isNaN(Number(price))) { - return null; - } - return Number(price); -}; - -// Growatt (device 3 / product 2) and Sinexcel (device 4 / product 5) both store -// 10-digit second timestamps, so the chart axis uses `ticks * 1000`, shifted into -// the browser's local zone to match the rest of the dashboards. +// Match the other dashboards: render unix-second timestamps shifted into the +// browser's local zone so the x-axis reads as local time. const toLocalMs = (ticks: number): number => { const d = new Date(ticks * 1000); d.setHours(d.getHours() - d.getTimezoneOffset() / 60); @@ -65,36 +29,31 @@ const toLocalMs = (ticks: number): number => { }; export const fetchCurrentPriceHistory = async ( - s3Credentials: I_S3Credentials, id: number, start: UnixTime, end: UnixTime ): Promise => { - let timestamps: number[] = []; + let points: PricePointDto[] = []; try { - const res: AxiosResponse = await axiosConfig.get( - `/GetCsvTimestampsForInstallation?id=${id}&start=${start.ticks}&end=${end.ticks}` + const res: AxiosResponse = await axiosConfig.get( + `/GetCurrentPriceHistory?id=${id}&start=${start.ticks}&end=${end.ticks}` ); - timestamps = res.data ?? []; + points = res.data ?? []; } catch { return EMPTY; } - const chunks = await Promise.all( - timestamps.map((t) => fetchChunk(t, s3Credentials)) - ); - const data: [number, number][] = []; let min = Number.POSITIVE_INFINITY; let max = Number.NEGATIVE_INFINITY; - chunks.forEach((chunk, i) => { - if (!chunk) return; - const price = extractPrice(chunk); - if (price === null) return; - data.push([toLocalMs(timestamps[i]), price]); - if (price < min) min = price; - if (price > max) max = price; + points.forEach((p) => { + if (!p || p.price === undefined || p.price === null || Number.isNaN(p.price)) { + return; + } + data.push([toLocalMs(p.timestamp), p.price]); + if (p.price < min) min = p.price; + if (p.price > max) max = p.price; }); if (data.length === 0) return EMPTY;