Innovenergy_trunk/csharp/App/Backend/Services/DailyIngestionService.cs

using InnovEnergy.App.Backend.Database;
using InnovEnergy.App.Backend.DataTypes;

namespace InnovEnergy.App.Backend.Services;

/// <summary>
/// Ingests daily energy totals from xlsx files into the DailyEnergyRecord SQLite table.
/// This is the source-of-truth population step for the report pipeline.
///
/// Current data source: xlsx files placed in tmp_report/{installationId}.xlsx
/// Future data source: S3 raw records (replace ExcelDataParser call with S3DailyExtractor)
///
/// Runs automatically at 01:00 UTC daily. Can also be triggered manually via the
/// IngestDailyData API endpoint.
/// </summary>
public static class DailyIngestionService
{
    private static readonly String TmpReportDir =
        Environment.CurrentDirectory + "/tmp_report/";

    private static Timer? _dailyTimer;

    /// <summary>
    /// Starts the daily scheduler. Call once on app startup.
    /// Ingests xlsx data at 01:00 UTC every day.
    /// </summary>
    public static void StartScheduler()
    {
        var now  = DateTime.UtcNow;
        var next = now.Date.AddDays(1).AddHours(1); // 01:00 UTC tomorrow

        _dailyTimer = new Timer(
            _ =>
            {
                try
                {
                    IngestAllInstallationsAsync().GetAwaiter().GetResult();
                }
                catch (Exception ex)
                {
                    Console.Error.WriteLine($"[DailyIngestion] Scheduler error: {ex.Message}");
                }
            },
            null, next - now, TimeSpan.FromDays(1));

        Console.WriteLine($"[DailyIngestion] Scheduler started. Next run: {next:yyyy-MM-dd HH:mm} UTC");
    }

    /// <summary>
    /// Ingests xlsx data for all SodioHome installations. Safe to call manually.
    /// </summary>
    public static async Task IngestAllInstallationsAsync()
    {
        Console.WriteLine($"[DailyIngestion] Starting ingestion for all SodioHome installations...");

        var installations = Db.Installations
            .Where(i => i.Product == (Int32)ProductType.SodioHome)
            .ToList();

        foreach (var installation in installations)
        {
            try
            {
                await IngestInstallationAsync(installation.Id);
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine($"[DailyIngestion] Failed for installation {installation.Id}: {ex.Message}");
            }
        }

        Console.WriteLine($"[DailyIngestion] Ingestion complete.");
    }

    /// <summary>
    /// Parses all xlsx files matching {installationId}*.xlsx in tmp_report/ and stores
    /// any new days as DailyEnergyRecord rows. Supports multiple time-ranged files per
    /// installation (e.g. 123_0203_0803.xlsx, 123_0901_1231.xlsx). Idempotent.
    /// </summary>
    public static async Task IngestInstallationAsync(Int64 installationId)
    {
        if (!Directory.Exists(TmpReportDir))
        {
            Console.WriteLine($"[DailyIngestion] tmp_report directory not found, skipping.");
            return;
        }

        var xlsxFiles = Directory.GetFiles(TmpReportDir, $"{installationId}*.xlsx");
        if (xlsxFiles.Length == 0)
        {
            Console.WriteLine($"[DailyIngestion] No xlsx found for installation {installationId}, skipping.");
            return;
        }

        var newDailyCount  = 0;
        var newHourlyCount = 0;
        var totalParsed    = 0;

        foreach (var xlsxPath in xlsxFiles.OrderBy(f => f))
        {
            // Ingest daily records
            List<DailyEnergyData> days;
            try
            {
                days = ExcelDataParser.Parse(xlsxPath);
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine($"[DailyIngestion] Failed to parse daily {Path.GetFileName(xlsxPath)}: {ex.Message}");
                continue;
            }

            totalParsed += days.Count;

            foreach (var day in days)
            {
                if (Db.DailyRecordExists(installationId, day.Date))
                    continue;

                Db.Create(new DailyEnergyRecord
                {
                    InstallationId    = installationId,
                    Date              = day.Date,
                    PvProduction      = day.PvProduction,
                    LoadConsumption   = day.LoadConsumption,
                    GridImport        = day.GridImport,
                    GridExport        = day.GridExport,
                    BatteryCharged    = day.BatteryCharged,
                    BatteryDischarged = day.BatteryDischarged,
                    CreatedAt         = DateTime.UtcNow.ToString("o"),
                });
                newDailyCount++;
            }

            // Ingest hourly records
            List<HourlyEnergyData> hours;
            try
            {
                hours = ExcelDataParser.ParseHourly(xlsxPath);
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine($"[DailyIngestion] Failed to parse hourly {Path.GetFileName(xlsxPath)}: {ex.Message}");
                continue;
            }

            foreach (var hour in hours)
            {
                var dateHour = $"{hour.DateTime:yyyy-MM-dd HH}";
                if (Db.HourlyRecordExists(installationId, dateHour))
                    continue;

                Db.Create(new HourlyEnergyRecord
                {
                    InstallationId       = installationId,
                    Date                 = hour.DateTime.ToString("yyyy-MM-dd"),
                    Hour                 = hour.Hour,
                    DateHour             = dateHour,
                    DayOfWeek            = hour.DayOfWeek,
                    IsWeekend            = hour.IsWeekend,
                    PvKwh                = hour.PvKwh,
                    LoadKwh              = hour.LoadKwh,
                    GridImportKwh        = hour.GridImportKwh,
                    BatteryChargedKwh    = hour.BatteryChargedKwh,
                    BatteryDischargedKwh = hour.BatteryDischargedKwh,
                    BattSoC              = hour.BattSoC,
                    CreatedAt            = DateTime.UtcNow.ToString("o"),
                });
                newHourlyCount++;
            }
        }

        Console.WriteLine($"[DailyIngestion] Installation {installationId}: {newDailyCount} new day(s), {newHourlyCount} new hour(s) ingested ({totalParsed} days across {xlsxFiles.Length} file(s)).");
        await Task.CompletedTask;
    }
}