get price data point every 15 min
This commit is contained in:
parent
c0fa353a17
commit
807882b960
|
|
@ -415,6 +415,34 @@ public class Controller : ControllerBase
|
|||
return sampledTimestamps;
|
||||
}
|
||||
|
||||
// Dynamic-pricing "Current Price" history (CHF/kWh) for the Configuration tab chart.
|
||||
// Only Growatt (device 3) and Sinexcel (device 4) carry Config.CurrentPrice.
|
||||
[HttpGet(nameof(GetCurrentPriceHistory))]
|
||||
public async Task<ActionResult<IEnumerable<PricePoint>>> GetCurrentPriceHistory(Int64 id, Int64 start, Int64 end, Token authToken)
|
||||
{
|
||||
var user = Db.GetSession(authToken)?.User;
|
||||
if (user == null)
|
||||
return Unauthorized();
|
||||
|
||||
var installation = Db.GetInstallationById(id);
|
||||
if (installation is null || !user.HasAccessTo(installation))
|
||||
return Unauthorized();
|
||||
|
||||
if (installation.Device != 3 && installation.Device != 4)
|
||||
return Ok(new List<PricePoint>());
|
||||
|
||||
// Clamp the range: each day spawns s3cmd processes + S3 fetches, so an
|
||||
// unbounded range from a crafted request could exhaust the single prod backend.
|
||||
const Int64 maxRangeSeconds = 31L * 86400;
|
||||
if (start < 0 || end <= start)
|
||||
return Ok(new List<PricePoint>());
|
||||
if (end - start > maxRangeSeconds)
|
||||
start = end - maxRangeSeconds;
|
||||
|
||||
var history = await CurrentPriceHistoryService.GetHistory(installation, start, end);
|
||||
return Ok(history);
|
||||
}
|
||||
|
||||
[HttpGet(nameof(GetUserById))]
|
||||
public ActionResult<User> GetUserById(Int64 id, Token authToken)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,245 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using InnovEnergy.App.Backend.DataTypes;
|
||||
using InnovEnergy.App.Backend.DataTypes.Methods;
|
||||
using InnovEnergy.Lib.S3Utils;
|
||||
using S3Bucket = InnovEnergy.Lib.S3Utils.DataTypes.S3Bucket;
|
||||
using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region;
|
||||
|
||||
namespace InnovEnergy.App.Backend.Services;
|
||||
|
||||
/// <summary>
|
||||
/// One data point of the dynamic-pricing "Current Price" history (CHF/kWh).
|
||||
/// </summary>
|
||||
public class PricePoint
|
||||
{
|
||||
public Int64 Timestamp { get; set; } // unix seconds
|
||||
public Double Price { get; set; } // CHF/kWh
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the "Current Price" history shown on the Configuration tab for
|
||||
/// Growatt (device 3) and Sinexcel (device 4) installations.
|
||||
///
|
||||
/// CurrentPrice only lives inside the per-10-second chunk files (Config.CurrentPrice);
|
||||
/// there is no pre-aggregated source. To keep load bounded we sample ONE chunk per
|
||||
/// 15-minute slot, fetch those in parallel, and cache each fully-past day (immutable).
|
||||
/// </summary>
|
||||
public static class CurrentPriceHistoryService
|
||||
{
|
||||
private const Int64 BucketSeconds = 900; // 15-minute resolution
|
||||
private const Int32 MaxParallelFetches = 24;
|
||||
private const Int32 CacheRetentionDays = 14; // bound the in-memory day cache
|
||||
private const String S3CfgPath = "/home/ubuntu/.s3cfg";
|
||||
|
||||
private static readonly Regex ChunkFileRegex = new(@"/([0-9]+)\.(csv|json)$", RegexOptions.Compiled);
|
||||
|
||||
// Immutable past days cached as "{installationId}:{yyyyMMdd}".
|
||||
private static readonly ConcurrentDictionary<String, List<PricePoint>> DayCache = new();
|
||||
|
||||
public static async Task<List<PricePoint>> GetHistory(Installation installation, Int64 startSec, Int64 endSec)
|
||||
{
|
||||
var todayUtc = DateTime.UtcNow.Date;
|
||||
var firstDay = DateTimeOffset.FromUnixTimeSeconds(startSec).UtcDateTime.Date;
|
||||
var lastDay = DateTimeOffset.FromUnixTimeSeconds(endSec).UtcDateTime.Date;
|
||||
|
||||
var points = new List<PricePoint>();
|
||||
for (var day = firstDay; day <= lastDay; day = day.AddDays(1))
|
||||
points.AddRange(await GetDay(installation, day, cacheable: day < todayUtc));
|
||||
|
||||
return points
|
||||
.Where(p => p.Timestamp >= startSec && p.Timestamp <= endSec)
|
||||
.OrderBy(p => p.Timestamp)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static async Task<List<PricePoint>> GetDay(Installation installation, DateTime dayUtc, Boolean cacheable)
|
||||
{
|
||||
var key = $"{installation.Id}:{dayUtc:yyyyMMdd}";
|
||||
if (cacheable && DayCache.TryGetValue(key, out var cached))
|
||||
return cached;
|
||||
|
||||
var dayStart = new DateTimeOffset(dayUtc, TimeSpan.Zero).ToUnixTimeSeconds();
|
||||
var dayEnd = dayStart + 86400 - 1;
|
||||
|
||||
var timestamps = SampleByBucket(ListChunkTimestamps(installation, dayStart, dayEnd));
|
||||
var points = await FetchPrices(installation, timestamps);
|
||||
|
||||
// Only cache non-empty past days: an empty result can mean a transient s3cmd/S3
|
||||
// failure, and caching that would serve "no data" forever until restart.
|
||||
if (cacheable && points.Count > 0)
|
||||
CacheDay(key, points);
|
||||
|
||||
return points;
|
||||
}
|
||||
|
||||
private static void CacheDay(String key, List<PricePoint> points)
|
||||
{
|
||||
DayCache[key] = points;
|
||||
|
||||
// Prune entries older than the retention window to bound memory growth.
|
||||
var cutoff = DateTime.UtcNow.Date.AddDays(-CacheRetentionDays);
|
||||
foreach (var existingKey in DayCache.Keys)
|
||||
{
|
||||
var datePart = existingKey.Substring(existingKey.IndexOf(':') + 1);
|
||||
if (DateTime.TryParseExact(datePart, "yyyyMMdd", CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.None, out var day) && day < cutoff)
|
||||
DayCache.TryRemove(existingKey, out _);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the first chunk in each 15-minute slot.
|
||||
private static List<Int64> SampleByBucket(List<Int64> timestamps)
|
||||
{
|
||||
var seenBuckets = new HashSet<Int64>();
|
||||
var picked = new List<Int64>();
|
||||
foreach (var t in timestamps.OrderBy(x => x))
|
||||
if (seenBuckets.Add(t / BucketSeconds))
|
||||
picked.Add(t);
|
||||
return picked;
|
||||
}
|
||||
|
||||
// List every chunk filename in range via `s3cmd ls` over the 5-digit timestamp prefixes
|
||||
// (same listing approach as Controller.GetCsvTimestampsForInstallation).
|
||||
private static List<Int64> ListChunkTimestamps(Installation installation, Int64 start, Int64 end)
|
||||
{
|
||||
var all = new List<Int64>();
|
||||
var startPrefix = Int64.Parse(start.ToString().Substring(0, 5));
|
||||
var endPrefix = Int64.Parse(end.ToString().Substring(0, 5));
|
||||
|
||||
for (var prefix = startPrefix; prefix <= endPrefix; prefix++)
|
||||
{
|
||||
var output = RunS3cmdLs("s3://" + installation.BucketName() + "/" + prefix);
|
||||
foreach (var line in output.Split('\n'))
|
||||
{
|
||||
var match = ChunkFileRegex.Match(line);
|
||||
if (match.Success && Int64.TryParse(match.Groups[1].Value, out var t) && t >= start && t <= end)
|
||||
all.Add(t);
|
||||
}
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
private static String RunS3cmdLs(String bucketPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = "s3cmd",
|
||||
Arguments = $"--config {S3CfgPath} ls {bucketPath}",
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
process.Start();
|
||||
var output = process.StandardOutput.ReadToEnd();
|
||||
process.StandardError.ReadToEnd();
|
||||
process.WaitForExit();
|
||||
return process.ExitCode == 0 ? output : "";
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"[CurrentPriceHistory] s3cmd ls failed for {bucketPath}: {e.Message}");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<List<PricePoint>> FetchPrices(Installation installation, List<Int64> timestamps)
|
||||
{
|
||||
var region = new S3Region($"https://{installation.S3Region}.{installation.S3Provider}", ExoCmd.S3Credentials!);
|
||||
var bucket = region.Bucket(installation.BucketName());
|
||||
|
||||
using var gate = new SemaphoreSlim(MaxParallelFetches);
|
||||
|
||||
var tasks = timestamps.Select(async ts =>
|
||||
{
|
||||
await gate.WaitAsync();
|
||||
try
|
||||
{
|
||||
var price = await FetchPriceAt(bucket, ts);
|
||||
return price.HasValue ? new PricePoint { Timestamp = ts, Price = price.Value } : null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
});
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
return results.Where(p => p != null).Select(p => p!).OrderBy(p => p.Timestamp).ToList();
|
||||
}
|
||||
|
||||
private static async Task<Double?> FetchPriceAt(S3Bucket bucket, Int64 ts)
|
||||
{
|
||||
try
|
||||
{
|
||||
var raw = await bucket.Path($"{ts}.json").GetObjectAsString();
|
||||
var json = DecodeChunk(raw);
|
||||
return json == null ? null : ExtractCurrentPrice(json);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null; // missing chunk / decode error -> just skip this slot
|
||||
}
|
||||
}
|
||||
|
||||
// Chunk objects are Base64-encoded ZIP archives whose inner "data.json" holds the record.
|
||||
private static String? DecodeChunk(String raw)
|
||||
{
|
||||
try
|
||||
{
|
||||
var trimmed = raw.Trim();
|
||||
if (trimmed.StartsWith('{'))
|
||||
return raw; // defensive: already plain JSON
|
||||
|
||||
var bytes = Convert.FromBase64String(trimmed);
|
||||
using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read);
|
||||
var entry = zip.GetEntry("data.json");
|
||||
if (entry == null)
|
||||
return null;
|
||||
|
||||
using var reader = new StreamReader(entry.Open());
|
||||
return reader.ReadToEnd();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// data.json is an object keyed by timestamp(s); read Config.CurrentPrice of the last record.
|
||||
private static Double? ExtractCurrentPrice(String json)
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
if (doc.RootElement.ValueKind != JsonValueKind.Object)
|
||||
return null;
|
||||
|
||||
var found = false;
|
||||
var record = default(JsonElement);
|
||||
foreach (var prop in doc.RootElement.EnumerateObject())
|
||||
{
|
||||
record = prop.Value;
|
||||
found = true;
|
||||
}
|
||||
|
||||
if (!found
|
||||
|| !record.TryGetProperty("Config", out var config)
|
||||
|| !config.TryGetProperty("CurrentPrice", out var currentPrice))
|
||||
return null;
|
||||
|
||||
return currentPrice.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Number => currentPrice.GetDouble(),
|
||||
JsonValueKind.String when Double.TryParse(currentPrice.GetString(),
|
||||
NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed) => parsed,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ import RestartAltIcon from '@mui/icons-material/RestartAlt';
|
|||
import { FormattedMessage, useIntl } from 'react-intl';
|
||||
import ReactApexChart from 'react-apexcharts';
|
||||
import ApexCharts, { ApexOptions } from 'apexcharts';
|
||||
import { I_S3Credentials } from 'src/interfaces/S3Types';
|
||||
import { UnixTime, TimeSpan } from 'src/dataCache/time';
|
||||
import {
|
||||
CurrentPriceSeries,
|
||||
|
|
@ -13,7 +12,6 @@ import {
|
|||
} from './currentPriceData';
|
||||
|
||||
interface CurrentPriceChartProps {
|
||||
s3Credentials: I_S3Credentials;
|
||||
id: number;
|
||||
}
|
||||
|
||||
|
|
@ -32,7 +30,7 @@ function CurrentPriceChart(props: CurrentPriceChartProps) {
|
|||
const end = UnixTime.now();
|
||||
const start = end.earlier(TimeSpan.fromDays(HISTORY_DAYS));
|
||||
|
||||
fetchCurrentPriceHistory(props.s3Credentials, props.id, start, end)
|
||||
fetchCurrentPriceHistory(props.id, start, end)
|
||||
.then((result) => {
|
||||
if (!cancelled) {
|
||||
setSeries(result);
|
||||
|
|
|
|||
|
|
@ -605,7 +605,6 @@ function SodioHomeInstallation(props: singleInstallationProps) {
|
|||
values={values}
|
||||
id={props.current_installation.id}
|
||||
installation={props.current_installation}
|
||||
s3Credentials={s3Credentials}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -43,14 +43,12 @@ import { DateTimePicker, TimePicker } from '@mui/x-date-pickers';
|
|||
import dayjs from 'dayjs';
|
||||
import Switch from '@mui/material/Switch';
|
||||
import FormControlLabel from '@mui/material/FormControlLabel';
|
||||
import { I_S3Credentials } from 'src/interfaces/S3Types';
|
||||
import CurrentPriceChart from './CurrentPriceChart';
|
||||
|
||||
interface SodistoreHomeConfigurationProps {
|
||||
values: JSONRecordData;
|
||||
id: number;
|
||||
installation: I_Installation;
|
||||
s3Credentials: I_S3Credentials;
|
||||
}
|
||||
|
||||
function SodistoreHomeConfigurationV2(props: SodistoreHomeConfigurationProps) {
|
||||
|
|
@ -965,10 +963,7 @@ function SodistoreHomeConfigurationV2(props: SodistoreHomeConfigurationProps) {
|
|||
</div>
|
||||
|
||||
<div style={{ marginBottom: '15px', marginTop: '10px' }}>
|
||||
<CurrentPriceChart
|
||||
s3Credentials={props.s3Credentials}
|
||||
id={props.id}
|
||||
/>
|
||||
<CurrentPriceChart id={props.id} />
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: '5px' }}>
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
import { AxiosResponse } from 'axios';
|
||||
import axiosConfig from 'src/Resources/axiosConfig';
|
||||
import { I_S3Credentials } from 'src/interfaces/S3Types';
|
||||
import { UnixTime } from 'src/dataCache/time';
|
||||
import { FetchResult } from 'src/dataCache/dataCache';
|
||||
import { fetchDataJson } from 'src/content/dashboards/Installations/fetchData';
|
||||
|
||||
// History of the `/Config/CurrentPrice` value (CHF/kWh) for the Configuration tab chart.
|
||||
// Reuses the same plumbing as the battery charts: the backend
|
||||
// (GetCsvTimestampsForInstallation) already downsamples to ~100 timestamps for any
|
||||
// range, so a week-long fetch stays at ~100 S3 chunk reads.
|
||||
// The backend GetCurrentPriceHistory endpoint reads the per-15-minute price from S3
|
||||
// chunks server-side and returns ready-to-plot points, so the browser makes one call.
|
||||
|
||||
export interface CurrentPriceSeries {
|
||||
// [timestampMs, price] — timestampMs is local-time-shifted like the other charts
|
||||
|
|
@ -17,47 +13,15 @@ export interface CurrentPriceSeries {
|
|||
max: number;
|
||||
}
|
||||
|
||||
interface PricePointDto {
|
||||
timestamp: number; // unix seconds (UTC)
|
||||
price: number; // CHF/kWh
|
||||
}
|
||||
|
||||
const EMPTY: CurrentPriceSeries = { data: [], min: 0, max: 0 };
|
||||
|
||||
// Fetch one chunk, retrying the next second (chunks land on even/odd seconds).
|
||||
const fetchChunk = async (
|
||||
ticks: number,
|
||||
s3Credentials: I_S3Credentials
|
||||
): Promise<Record<string, any> | null> => {
|
||||
for (let i = 0; i < 2; i++) {
|
||||
try {
|
||||
const res = await fetchDataJson(
|
||||
UnixTime.fromTicks(ticks + i, true),
|
||||
s3Credentials
|
||||
);
|
||||
if (
|
||||
res &&
|
||||
res !== FetchResult.notAvailable &&
|
||||
res !== FetchResult.tryLater
|
||||
) {
|
||||
return res as Record<string, any>;
|
||||
}
|
||||
} catch {
|
||||
// ignore and try next second / next timestamp
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const extractPrice = (chunk: Record<string, any>): number | null => {
|
||||
const keys = Object.keys(chunk);
|
||||
if (keys.length === 0) return null;
|
||||
const record = chunk[keys[keys.length - 1]];
|
||||
const price = record?.Config?.CurrentPrice;
|
||||
if (price === undefined || price === null || Number.isNaN(Number(price))) {
|
||||
return null;
|
||||
}
|
||||
return Number(price);
|
||||
};
|
||||
|
||||
// Growatt (device 3 / product 2) and Sinexcel (device 4 / product 5) both store
|
||||
// 10-digit second timestamps, so the chart axis uses `ticks * 1000`, shifted into
|
||||
// the browser's local zone to match the rest of the dashboards.
|
||||
// Match the other dashboards: render unix-second timestamps shifted into the
|
||||
// browser's local zone so the x-axis reads as local time.
|
||||
const toLocalMs = (ticks: number): number => {
|
||||
const d = new Date(ticks * 1000);
|
||||
d.setHours(d.getHours() - d.getTimezoneOffset() / 60);
|
||||
|
|
@ -65,36 +29,31 @@ const toLocalMs = (ticks: number): number => {
|
|||
};
|
||||
|
||||
export const fetchCurrentPriceHistory = async (
|
||||
s3Credentials: I_S3Credentials,
|
||||
id: number,
|
||||
start: UnixTime,
|
||||
end: UnixTime
|
||||
): Promise<CurrentPriceSeries> => {
|
||||
let timestamps: number[] = [];
|
||||
let points: PricePointDto[] = [];
|
||||
try {
|
||||
const res: AxiosResponse<number[]> = await axiosConfig.get(
|
||||
`/GetCsvTimestampsForInstallation?id=${id}&start=${start.ticks}&end=${end.ticks}`
|
||||
const res: AxiosResponse<PricePointDto[]> = await axiosConfig.get(
|
||||
`/GetCurrentPriceHistory?id=${id}&start=${start.ticks}&end=${end.ticks}`
|
||||
);
|
||||
timestamps = res.data ?? [];
|
||||
points = res.data ?? [];
|
||||
} catch {
|
||||
return EMPTY;
|
||||
}
|
||||
|
||||
const chunks = await Promise.all(
|
||||
timestamps.map((t) => fetchChunk(t, s3Credentials))
|
||||
);
|
||||
|
||||
const data: [number, number][] = [];
|
||||
let min = Number.POSITIVE_INFINITY;
|
||||
let max = Number.NEGATIVE_INFINITY;
|
||||
|
||||
chunks.forEach((chunk, i) => {
|
||||
if (!chunk) return;
|
||||
const price = extractPrice(chunk);
|
||||
if (price === null) return;
|
||||
data.push([toLocalMs(timestamps[i]), price]);
|
||||
if (price < min) min = price;
|
||||
if (price > max) max = price;
|
||||
points.forEach((p) => {
|
||||
if (!p || p.price === undefined || p.price === null || Number.isNaN(p.price)) {
|
||||
return;
|
||||
}
|
||||
data.push([toLocalMs(p.timestamp), p.price]);
|
||||
if (p.price < min) min = p.price;
|
||||
if (p.price > max) max = p.price;
|
||||
});
|
||||
|
||||
if (data.length === 0) return EMPTY;
|
||||
|
|
|
|||
Loading…
Reference in New Issue