Innovenergy_trunk/csharp/App/SinexcelCommunication/Program.cs

using System.IO.Compression;
using System.IO.Ports;
using System.Text;
using System.Text.Json;

using Flurl.Http;

using System.Reactive.Linq;
using System.Reactive.Threading.Tasks;

using InnovEnergy.Lib.Devices.Sinexcel_12K_TL;
using InnovEnergy.App.SinexcelCommunication.DataLogging;
using InnovEnergy.App.SinexcelCommunication.ESS;
using InnovEnergy.App.SinexcelCommunication.MiddlewareClasses;
using InnovEnergy.App.SinexcelCommunication.SystemConfig;
using InnovEnergy.Lib.Protocols.Modbus.Channels;
using InnovEnergy.App.SinexcelCommunication.DataTypes;

using InnovEnergy.Lib.Units;
using InnovEnergy.Lib.Utils;

using Newtonsoft.Json;
using Formatting = Newtonsoft.Json.Formatting;
using JsonSerializer = System.Text.Json.JsonSerializer;
using static InnovEnergy.App.SinexcelCommunication.MiddlewareClasses.MiddlewareAgent;
using System.Diagnostics.CodeAnalysis;
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.

namespace InnovEnergy.App.SinexcelCommunication;

[SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")]
internal static class Program
{
    private static readonly TimeSpan UpdateInterval = TimeSpan.FromSeconds(4);
    private const  UInt16   NbrOfFileToConcatenate  = 15; // add this to config file
    private static UInt16   _fileCounter            = 0; 
    private static Channel  _sinexcelChannel;

    private const String                SwVersionNumber                 =" V1.00.131025 beta";
    private const  String               VpnServerIp                     = "10.2.0.11";
    private static Boolean             _subscribedToQueue               = false;
    private static Boolean             _subscribeToQueueForTheFirstTime = false;
    private static SodistoreAlarmState _prevSodiohomeAlarmState         = SodistoreAlarmState.Green;
    private static SodistoreAlarmState _sodiohomeAlarmState             = SodistoreAlarmState.Green;
    
    // move all this to config file
    private const String Port     = "/dev/ttyUSB0"; 
    private const Byte   SlaveId  = 1;
    private const Parity Parity   = 0; //none
    private const Int32  StopBits = 1;
    private const Int32  BaudRate = 115200;
    private const Int32  DataBits = 8;

    public static async Task Main(String[] args)
    {
        _sinexcelChannel = new SerialPortChannel(Port, BaudRate, Parity, DataBits, StopBits);

        while (true)
        {
            try
            {
                await Run();
            }
            catch (Exception e)
            {
                e.WriteLine();
            }
        }     
        // ReSharper disable once FunctionNeverReturns
    }
    
    private static async Task Run()
    {
        Watchdog.NotifyReady();

        Console.WriteLine("Starting Sinexcel Communication");

        var sinexcelDevice = new SinexcelDevice(_sinexcelChannel, SlaveId);
       // var sinexcelDevice = new sinexcelDevices(new List<sinexcelDevice> { growattDeviceT415K });

        StatusRecord? ReadStatus()
        {
            var config         = Config.Load();
            var sinexcelRecord = sinexcelDevice.Read();

            return new StatusRecord
            {
                SinexcelRecord = sinexcelRecord,
                Config         = config // load from disk every iteration, so config can be changed while running
            };
        }

        while (true)
        {
            await Observable
                .Interval(UpdateInterval)
                .Select(_ => RunIteration())
                .SelectMany(status =>
                    DataLogging(status, DateTime.Now.Round(UpdateInterval))
                        .ContinueWith(_ => status)) // back to StatusRecord
                .SelectMany(SaveModbusTcpFile)
                .SelectError()
                .ToTask();
        }
        
        StatusRecord? RunIteration()
        {
            try
            {
                Watchdog.NotifyAlive();
                
                var startTime = DateTime.Now;
                Console.WriteLine("***************************** Reading  Battery Data  *********************************************");
                Console.WriteLine(startTime.ToString("HH:mm:ss.fff"));
                                
                // the order matter of the next three lines
                var statusrecord = ReadStatus();
                if (statusrecord == null)
                    return null;
                
                statusrecord.SinexcelRecord.GridAPhaseVoltage.WriteLine(" = Grid A PhaseVoltage");
                statusrecord.SinexcelRecord.GridBPhaseVoltage.WriteLine(" = Grid B PhaseVoltage");
                statusrecord.SinexcelRecord.GridCPhaseVoltage.WriteLine(" = Grid C PhaseVoltage");

                statusrecord.SinexcelRecord.GridAPhaseCurrent.WriteLine(" = Grid A PhaseCurrent");
                statusrecord.SinexcelRecord.GridBPhaseCurrent.WriteLine(" = Grid B PhaseCurrent");
                statusrecord.SinexcelRecord.GridCPhaseCurrent.WriteLine(" = Grid C PhaseCurrent");

                
                statusrecord.SinexcelRecord.GridAPhaseActivePower.WriteLine(" =  A Grid Active Power");
                statusrecord.SinexcelRecord.GridBPhaseActivePower.WriteLine(" =  B Grid Active Power");
                statusrecord.SinexcelRecord.GridCPhaseActivePower.WriteLine(" =  C Grid Active Power");

                statusrecord.SinexcelRecord.GridVoltageFrequency.WriteLine(" = Frequency");

                statusrecord.SinexcelRecord.ElectricMeterCPhaseActivePower.WriteLine(" Meter Phase C Power");
                statusrecord.SinexcelRecord.ElectricMeterBPhaseActivePower.WriteLine(" Meter Phase B Power");
                statusrecord.SinexcelRecord.ElectricMeterAPhaseActivePower.WriteLine(" Meter Phase A Power");
                
                statusrecord.SinexcelRecord.ElectricMeterAPhaseCurrent.WriteLine(" Meter Phase C Current ");
                statusrecord.SinexcelRecord.ElectricMeterBPhaseCurrent.WriteLine(" Meter Phase B Current ");
                statusrecord.SinexcelRecord.ElectricMeterBPhaseCurrent.WriteLine(" Meter Phase A Current ");
                statusrecord.SinexcelRecord.WorkingMode.WriteLine(" workingmode");
                
                statusrecord.SinexcelRecord.BatteryVoltage.WriteLine(" BatteryVoltage");
                statusrecord.SinexcelRecord.BatteryVoltage1.WriteLine(" BatteryVoltage1");
                statusrecord.SinexcelRecord.BatteryVoltage2.WriteLine(" BatteryVoltage2");
                
                statusrecord.SinexcelRecord.BatteryPower1.WriteLine(" BatteryPower1");
                statusrecord.SinexcelRecord.BatteryPower2.WriteLine(" BatteryPower2");
                statusrecord.SinexcelRecord.TotalBatteryPower.WriteLine(" TotalBatteryPower");
                
                statusrecord.SinexcelRecord.EnableBattery1.WriteLine(" EnableBattery1");
                statusrecord.SinexcelRecord.EnableBattery2.WriteLine(" EnableBattery2");
                
                statusrecord.SinexcelRecord.MaxChargingCurrentBattery1.WriteLine(" MaxChargingCurrentBattery1");
                statusrecord.SinexcelRecord.MaxChargingCurrentBattery2.WriteLine(" MaxChargingCurrentBattery2");
                
                statusrecord.SinexcelRecord.MaxDischargingCurrentBattery1.WriteLine(" MaxChargingCurrentBattery1");
                statusrecord.SinexcelRecord.MaxDischargingCurrentBattery2.WriteLine(" MaxChargingCurrentBattery2");
                
                //EssModeControl(statusrecord);
               // statusrecord.ApplyDefaultSettings();
                Console.WriteLine( " ************************************ We are writing ************************************");
                statusrecord?.Config.Save(); // save the config file
                //sinexcelDevice.Write(statusrecord?.SinexcelRecord);
                var stopTime            = DateTime.Now;
                Console.WriteLine(stopTime.ToString("HH:mm:ss.fff"));
                return statusrecord;
            }
            catch (Exception e)
            {
                // Handle exception and print the error
                Console.WriteLine(e );
                return null;
            }

        }
    }
    
    private static Int32 GetInstallationId(String s3Bucket)
    {
        var part = s3Bucket.Split('-').FirstOrDefault();
        return int.TryParse(part, out var id) ? id : 0; // is 0 a default safe value? check with Marios
    }
    
    private static void SendSalimaxStateAlarm(StatusMessage currentSalimaxState, StatusRecord? record)
    {
        var s3Bucket = Config.Load().S3?.Bucket;
        var subscribedNow = false;
        
        //When the controller boots, it tries to subscribe to the queue
        if (_subscribeToQueueForTheFirstTime == false)
        {
            subscribedNow = true;
            _subscribeToQueueForTheFirstTime = true;
            _prevSodiohomeAlarmState = currentSalimaxState.Status;
            _subscribedToQueue = RabbitMqManager.SubscribeToQueue(currentSalimaxState, s3Bucket, VpnServerIp);
        }

        //If already subscribed to the queue and the status has been changed, update the queue
        if (!subscribedNow && _subscribedToQueue && currentSalimaxState.Status != _prevSodiohomeAlarmState)
        {
            _prevSodiohomeAlarmState = currentSalimaxState.Status;
            if (s3Bucket != null)
                RabbitMqManager.InformMiddleware(currentSalimaxState);
        }

        //If there is an available message from the RabbitMQ Broker, apply the configuration file
        Configuration? config = SetConfigurationFile();
        if (config != null)
        {
            record.ApplyConfigFile(config);
        }
    }
    
    private static void ApplyConfigFile(this StatusRecord? status, Configuration? config)
    {
        if (config == null) return;
        if (status == null) return;

        status.Config.MinSoc                    = config.MinimumSoC;
        status.Config.GridSetPoint              = config.GridSetPoint * 1000; // converted from kW to W
        //status.Config.MaximumChargingCurrent    = config.MaximumChargingCurrent;
        //status.Config.MaximumDischargingCurrent = config.MaximumDischargingCurrent; 
        //status.Config.OperatingPriority         = config.OperatingPriority;
        //status.Config.BatteriesCount            = config.BatteriesCount;
    }
    
      private static async Task<Boolean> SaveModbusTcpFile(StatusRecord status)
    {
        var modbusData = new Dictionary<String, UInt16>();
       
       // SYSTEM DATA
        var result1 = ConvertToModbusRegisters((status.Config.MinSoc * 10), "UInt16", 30001); // this to be updated to modbusTCP version
        var result2 = ConvertToModbusRegisters(status.SinexcelRecord.GridAPhaseVoltage, "UInt32", 30002);
       
        // Merge all results into one dictionary
        var allResults = new[]
        {
            result1,result2
        };
        
        foreach (var result in allResults)
        {
            foreach (var entry in result)
            {
                modbusData[entry.Key] = entry.Value;
            }
        }
        // Write to JSON
        var json = JsonSerializer.Serialize(modbusData, new JsonSerializerOptions { WriteIndented = true });
        await File.WriteAllTextAsync("/home/inesco/SodiStoreHome/ModbusTCP/modbus_tcp_data.json", json);

        //Console.WriteLine("JSON file written successfully.");
        //Console.WriteLine(json);
        return true;
    }
      
    private static Dictionary<String, UInt16> ConvertToModbusRegisters(Object value, String outputType, Int32 startingAddress)
    {
        var registers = new Dictionary<String, UInt16>();
    
        switch (outputType)
        {
            case "UInt16":
                registers[startingAddress.ToString()] = Convert.ToUInt16(value);
                break;
    
            case "Int16":
                var int16Val = Convert.ToInt16(value);
                registers[startingAddress.ToString()] = (UInt16)int16Val;  // reinterpret signed as ushort
                break;
    
            case "UInt32":
                var uint32Val = Convert.ToUInt32(value);
                registers[startingAddress.ToString()] = (UInt16)(uint32Val & 0xFFFF);         // Low word
                registers[(startingAddress + 1).ToString()] = (UInt16)(uint32Val >> 16);      // High word
                break;
    
            case "Int32":
                var int32Val = Convert.ToInt32(value);
                var raw = unchecked((UInt32)int32Val); // reinterprets signed int as unsigned
                registers[startingAddress.ToString()] = (UInt16)(raw & 0xFFFF);
                registers[(startingAddress + 1).ToString()] = (UInt16)(raw >> 16);
                break;
    
            default:
                throw new ArgumentException("Unsupported output type: " + outputType);
        }
        return registers;
    }
    
    private static async Task<Boolean> DataLogging(StatusRecord status, DateTime timeStamp)
    {
        var csv = status.ToCsv();
        
        // for debug, only to be deleted. 
        //foreach (var item in csv.SplitLines())
        //{
        //    Console.WriteLine(item + "");
        //}
        
        await SavingLocalCsvFile(timeStamp.ToUnixTime(), csv);

        var jsonData = new Dictionary<String, Object>();

        ConvertToJson(csv, jsonData).LogInfo();
        
        var s3Config = status.Config.S3;
        
        if (s3Config is null)
            return false;

        //Concatenating 15 files in one file 
        return await ConcatinatingAndCompressingFiles(timeStamp.ToUnixTime(), s3Config);
    }
    
    private static void InsertIntoJson(Dictionary<String, Object> jsonDict, String[] keys, String value)
    {
        var currentDict = jsonDict;
        for (Int16 i = 1; i < keys.Length; i++) // Start at 1 to skip empty root
        {
            var key = keys[i];
            if (!currentDict.ContainsKey(key))
            {
                currentDict[key] = new Dictionary<String, Object>();
            }

            if (i == keys.Length - 1) // Last key, store the value
            {
                
                if (!value.Contains(",") && double.TryParse(value, out Double doubleValue)) // Try to parse value as a number
                {
                    currentDict[key] = Math.Round(doubleValue, 2); // Round to 2 decimal places
                   
                }
                else
                {
                    currentDict[key] = value; // Store as string if not a number
                }
            }
            else
            {
                currentDict = (Dictionary<String, Object>)currentDict[key];
            }
        }
    }
    
    private static String ConvertToJson(String csv, Dictionary<String, Object> jsonData)
    {
        foreach (var line in csv.Split('\n'))
        {
            if (string.IsNullOrWhiteSpace(line)) continue;
              
            var  parts   = line.Split(';');
            var  keyPath = parts[0];
            var  value   = parts[1];
            var  unit    = parts.Length > 2 ? parts[2].Trim() : "";
            InsertIntoJson(jsonData, keyPath.Split('/'), value);
        }

        var jsonOutput = JsonConvert.SerializeObject(jsonData, Formatting.None);
        return jsonOutput;
    }
    
    private static async Task SavingLocalCsvFile(Int64 timestamp, String csv)
    {
        const String directoryPath = "/home/inesco/SodiStoreHome/csvFile";

        // Ensure directory exists
        if (!Directory.Exists(directoryPath))
        {
            Directory.CreateDirectory(directoryPath);
        }

        // Get all .csv files ordered by creation time (oldest first)
        var csvFiles = new DirectoryInfo(directoryPath)
            .GetFiles("*.csv")
            .OrderBy(f => f.CreationTimeUtc)
            .ToList();

        // If more than 5000 files, delete the oldest
        if (csvFiles.Count >= 5000)
        {
            var oldestFile = csvFiles.First();
            try
            {
                oldestFile.Delete();
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Failed to delete file: {oldestFile.FullName}, Error: {ex.Message}");
            }
        }

        // Prepare the filtered CSV content
        var filteredCsv = csv
            .SplitLines()
            .Where(l => !l.Contains("Secret"))
            .JoinLines();

        // Save the new CSV file
        var filePath = Path.Combine(directoryPath, timestamp + ".csv");
        await File.WriteAllTextAsync(filePath, filteredCsv);
    }

    private static async Task<Boolean> ConcatinatingAndCompressingFiles(Int64 timeStamp, S3Config s3Config)
    {
         if (_fileCounter >= NbrOfFileToConcatenate)
         {
             _fileCounter = 0;

             var logFileConcatenator = new LogFileConcatenator();
             var jsontoSend = logFileConcatenator.ConcatenateFiles(NbrOfFileToConcatenate);

             var fileNameWithoutExtension = timeStamp.ToString(); // used for both S3 and local
             var s3Path = fileNameWithoutExtension + ".json";

             var request = s3Config.CreatePutRequest(s3Path);

             var compressedBytes = CompresseBytes(jsontoSend);
             var base64String = Convert.ToBase64String(compressedBytes);
             var stringContent = new StringContent(base64String, Encoding.UTF8, "application/base64");

             var uploadSucceeded = false;

             try
             {
                 var response = await request.PutAsync(stringContent);

                 if (response.StatusCode != 200)
                 {
                     Console.WriteLine("ERROR: PUT");
                     var error = await response.GetStringAsync();
                     Console.WriteLine(error);

                     await SaveToLocalCompressedFallback(compressedBytes, fileNameWithoutExtension);
                     Heartbit();
                     return false;
                 }

                 uploadSucceeded = true;
                 Console.WriteLine("✅ File uploaded to S3 successfully.");

                 Console.WriteLine("---------------------------------------- Resending FailedUploadedFiles----------------------------------------");
                 Heartbit();

                 await ResendLocalFailedFilesAsync(s3Config); // retry any pending failed files
             }
             catch (Exception ex)
             {
                 Console.WriteLine("Upload exception: " + ex.Message);

                 if (!uploadSucceeded)
                 {
                     await SaveToLocalCompressedFallback(compressedBytes, fileNameWithoutExtension);
                 }

                 Heartbit();
                 return false;
             }
         }

         _fileCounter++;
         return true;
    }
    
    private static void Heartbit()
    { 
        var s3Bucket = Config.Load().S3?.Bucket;
        var tryParse = int.TryParse(s3Bucket?.Split("-")[0], out var installationId);

        if (tryParse)
        {
            var returnedStatus = new StatusMessage
            {
                InstallationId = installationId,
                Product        = 2,
                Status         = _sodiohomeAlarmState,
                Type           = MessageType.Heartbit,
            };
            if (s3Bucket != null)
                RabbitMqManager.InformMiddleware(returnedStatus);
        }
    }
    
    private static async Task SaveToLocalCompressedFallback(Byte[] compressedData, String fileNameWithoutExtension)
    {
        try
        {
            var fallbackDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "FailedUploads");
            Directory.CreateDirectory(fallbackDir);

            var fileName = fileNameWithoutExtension + ".json"; // Save as .json, but still compressed
            var fullPath = Path.Combine(fallbackDir, fileName);

            await File.WriteAllBytesAsync(fullPath, compressedData); // Compressed data
            Console.WriteLine($"Saved compressed failed upload to: {fullPath}");
        }
        catch (Exception ex)
        {
            Console.WriteLine("Failed to save compressed file locally: " + ex.Message);
        }
    }
    
    private static async Task ResendLocalFailedFilesAsync(S3Config s3Config)
    {
        var fallbackDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "FailedUploads");

        if (!Directory.Exists(fallbackDir))
            return;

        var files = Directory.GetFiles(fallbackDir, "*.json");
        files.Length.WriteLine(" Number of failed files, to upload");

        foreach (var filePath in files)
        {
            var fileName = Path.GetFileName(filePath); // e.g., "1720023600.json"

            try
            {
                byte[] compressedBytes = await File.ReadAllBytesAsync(filePath);
                var base64String = Convert.ToBase64String(compressedBytes);
                var stringContent = new StringContent(base64String, Encoding.UTF8, "application/base64");

                var request = s3Config.CreatePutRequest(fileName);
                var response = await request.PutAsync(stringContent);

                if (response.StatusCode == 200)
                {
                    File.Delete(filePath);
                    Console.WriteLine($"✅ Successfully resent and deleted: {fileName}");
                }
                else
                {
                    Console.WriteLine($"❌ Failed to resend {fileName}, status: {response.StatusCode}");
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"⚠️ Exception while resending {fileName}: {ex.Message}");
            }
        }
    }
    private static Byte[] CompresseBytes(String jsonToSend)
    {
        //Compress JSON data to a byte array
        using var memoryStream = new MemoryStream();
        //Create a zip directory and put the compressed file inside
        using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
        {
            var entry = archive.CreateEntry("data.json", CompressionLevel.SmallestSize); // Add JSON data to the ZIP archive
            using (var entryStream = entry.Open())
            using (var writer = new StreamWriter(entryStream))
            {
                writer.Write(jsonToSend);
            }
        }
        
        var compressedBytes = memoryStream.ToArray();

        return compressedBytes;
    }

}