From 857f220f43a4ddd76a9645c6d57ea1d1464f9b13 Mon Sep 17 00:00:00 2001 From: Noe Date: Thu, 5 Jun 2025 10:28:09 +0200 Subject: [PATCH] Add S3ExtractTool to git --- S3ExtractingTool/ExtractS3README.txt | 90 +++ S3ExtractingTool/example_file.json | 937 +++++++++++++++++++++++++++ S3ExtractingTool/extractS3data.py | 256 ++++++++ 3 files changed, 1283 insertions(+) create mode 100644 S3ExtractingTool/ExtractS3README.txt create mode 100644 S3ExtractingTool/example_file.json create mode 100644 S3ExtractingTool/extractS3data.py diff --git a/S3ExtractingTool/ExtractS3README.txt b/S3ExtractingTool/ExtractS3README.txt new file mode 100644 index 000000000..c4cd1cb04 --- /dev/null +++ b/S3ExtractingTool/ExtractS3README.txt @@ -0,0 +1,90 @@ +This README file provides a comprehensive guide to utilizing a Python script for interacting with S3 storage, +specifically designed for downloading and processing data files based on a specified time range and key parameters. +The script requires Python3 installed on your system and makes use of the s3cmd tool for accessing data in cloud storage. +It also illustrates the process of configuring s3cmd by creating a .s3cfg file with your access credentials. Nice + + +############ Create the .s3cfg file in home directory ################ + +nano .s3cfg + +Copy this lines inside the file. + +[default] +host_base = sos-ch-dk-2.exo.io +host_bucket = %(bucket)s.sos-ch-dk-2.exo.io +access_key = EXO4d838d1360ba9fb7d51648b0 +secret_key = _bmrp6ewWAvNwdAQoeJuC-9y02Lsx7NV6zD-WjljzCU +use_https = True + + +############ S3cmd instalation ################ + +Please install s3cmd for retrieving data from our Cloud storage. + +sudo apt install s3cmd + +############ Python3 instalation ################ + +To check if you have already have python3, run this command + + python3 --version + + +To install you can use this command: + +1) sudo apt update + +2) sudo apt install python3 + +3) python3 --version (to check if pyhton3 installed correctly) + + +############ Run extractS3data.py ################ + +usage: extractRange.py [-h] --key KEY --bucket-number BUCKET_NUMBER start_timestamp end_timestamp + +KEY: the key can be a one word or a path + + for example: /DcDc/Devices/2/Status/Dc/Battery/voltage ==> this will provide us a Dc battery Voltage of the DcDc device 2. + example : Dc/Battery/voltage ==> This will provide all DcDc Device voltage (including the avg voltage of all DcDc device) + example : voltage ==> This will provide all voltage of all devices in the Salimax + +BUCKET_NUMBER: This a number of bucket name for the instalation + +start_timestamp end_timestamp: this must be a correct timestamp of 10 digits. +The start_timestamp must be smaller than the end_timestamp. + +PS: The data will be downloaded to a folder named S3cmdData_{Bucket_Number}. If this folder does not exist, it will be created. +If the folder exist, it will try to download data between the requested timestamps if they files are not already existing. + +Example command: + +python3 extractS3data.py 1749062721 1749106001 --keys GridMeter/Ac/Power/Active --bucket-number 12 --product_name=SodistoreMax + + +################################ EXTENDED FEATURES FOR MORE ADVANCED USAGE ################################ + +1) Multiple Keys Support: + +The script supports the extraction of data using multiple keys. Users can specify one or multiple keys separated by commas with the --keys parameter. +This feature allows for more granular data extraction, catering to diverse data analysis requirements. For example, users can extract data for different +metrics or parameters from the same or different CSV files within the specified range. + +3) Dynamic Header Generation: + +The script dynamically generates headers for the output CSV file based on the keys provided. This ensures that the output file accurately reflects the +extracted data, providing a clear and understandable format for subsequent analysis. The headers correspond to the keys used for data extraction, making +it easy to identify and analyze the extracted data. + +4)Advanced Data Processing Capabilities: + +Booleans as Numbers: The --booleans_as_numbers flag allows users to convert boolean values (True/False) into numeric representations (1/0). This feature +is particularly useful for analytical tasks that require numerical data processing. + +Example Command: + +python3 extractS3data.py 1749062721 1749106001 --keys AcDc/SystemControl/ResetAlarmsAndWarnings,AcDc/Devices/1/Status/Ac/L1/Voltage --bucket-number 12 --product_name=SodistoreMax + +This command extracts data for AcDc/SystemControl/ResetAlarmsAndWarnings and AcDc/Devices/1/Status/Ac/L1/Voltage keys from bucket number 12, between the specified timestamps, with boolean values converted to numbers. + diff --git a/S3ExtractingTool/example_file.json b/S3ExtractingTool/example_file.json new file mode 100644 index 000000000..ee447e9f8 --- /dev/null +++ b/S3ExtractingTool/example_file.json @@ -0,0 +1,937 @@ +{ + "AcDc": { + "SystemControl": { + "Alarms": "", + "CommunicationTimeout": "00:00:20", + "SystemConfig": "AcDcAndDcDc", + "ResetAlarmsAndWarnings": "True", + "TargetSlave": 0.0, + "UseSlaveIdForAddressing": "True", + "ReferenceFrame": "Consumer", + "SlaveErrorHandling": "Relaxed", + "SubSlaveErrorHandling": "Off", + "PowerSetPointActivation": "Immediate", + "PowerSetPointTrigger": "Wait", + "DeviceState": "Operation", + "NumberOfConnectedSlaves": 2.0, + "NumberOfConnectedSubSlaves": 0.0, + "Warnings": "" + }, + "Devices": { + "1": { + "Status": { + "Ac": { + "L1": { + "Voltage": 238.8, + "Current": 1.45, + "Phi": 3.09, + "Power": { + "Active": -345.73, + "Reactive": 19.21, + "Apparent": 346.26 + } + }, + "L2": { + "Voltage": 239.2, + "Current": 1.53, + "Phi": -3.09, + "Power": { + "Active": -365.41, + "Reactive": -20.3, + "Apparent": 365.98 + } + }, + "L3": { + "Voltage": 239.9, + "Current": 1.31, + "Phi": 3.14, + "Power": { + "Active": -314.26, + "Reactive": 1.97, + "Apparent": 314.27 + } + }, + "Frequency": 50.0, + "Power": { + "Active": -1025.4, + "Reactive": 0.88, + "Apparent": 1025.4 + } + }, + "PowerLimitedBy": "DcLink", + "InverterState": { + "Current": "AcConnected", + "OnLastAlarm": "Alarm" + }, + "ActiveGridType": "GridTied400V50Hz", + "DcVoltages": { + "Intern": { + "DcUpperHalf": 389.0, + "DcLowerHalf": 390.0, + "NToPe": 6553.5 + }, + "Extern": { + "UpperHalf": 389.0, + "LowerHalf": 390.0, + "NToPe": 0.7 + }, + "Active": { + "ActiveUpperVoltage": 780.0, + "ActiveLowerVoltage": 720.0, + "ActiveRefVoltage": 750.0 + } + }, + "Temperature": { + "InletAir": 23.9, + "IgbtL1": 49.7, + "IgbtL2": 47.6, + "IgbtL3": 51.8, + "IgbtBalancer": 50.6 + }, + "OverloadCapacity": { + "L1": 100.0, + "L2": 100.0, + "L3": 100.0 + }, + "Nominal": { + "AcFrequency": 50.0, + "AcVoltage": 400.0, + "Power": 25000.0 + }, + "Alarms": "", + "Warnings": "" + }, + "Control": { + "Ac": { + "Power": { + "L1": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + }, + "L2": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + }, + "L3": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + } + }, + "PhaseControl": "Asymmetric", + "GridType": "GridTied400V50Hz", + "IslandMode": { + "FrequencyOffset": 0.0, + "VoltageAdjustmentFactor": 100.0 + } + }, + "Dc": { + "ReferenceVoltage": 750.0, + "MinVoltage": 720.0, + "MaxVoltage": 780.0, + "PrechargeConfig": "PrechargeDcWithInternal" + }, + "PowerStageEnable": "True", + "ResetAlarmsAndWarnings": "False" + } + }, + "2": { + "Status": { + "Ac": { + "L1": { + "Voltage": 239.0, + "Current": 0.74, + "Phi": -0.1, + "Power": { + "Active": 175.89, + "Reactive": -18.51, + "Apparent": 176.86 + } + }, + "L2": { + "Voltage": 239.5, + "Current": 0.62, + "Phi": -0.77, + "Power": { + "Active": 106.89, + "Reactive": -103.07, + "Apparent": 148.49 + } + }, + "L3": { + "Voltage": 239.8, + "Current": 0.47, + "Phi": -0.07, + "Power": { + "Active": 112.42, + "Reactive": -8.03, + "Apparent": 112.71 + } + }, + "Frequency": 50.01, + "Power": { + "Active": 395.2, + "Reactive": -129.62, + "Apparent": 415.91 + } + }, + "PowerLimitedBy": "Nothing", + "InverterState": { + "Current": "AcConnected", + "OnLastAlarm": "Idle" + }, + "ActiveGridType": "GridTied400V50Hz", + "DcVoltages": { + "Intern": { + "DcUpperHalf": 388.0, + "DcLowerHalf": 389.0, + "NToPe": 0.1 + }, + "Extern": { + "UpperHalf": 388.0, + "LowerHalf": 388.0, + "NToPe": 6553.4 + }, + "Active": { + "ActiveUpperVoltage": 780.0, + "ActiveLowerVoltage": 720.0, + "ActiveRefVoltage": 750.0 + } + }, + "Temperature": { + "InletAir": 26.0, + "IgbtL1": 49.0, + "IgbtL2": 47.0, + "IgbtL3": 50.0, + "IgbtBalancer": 48.0 + }, + "OverloadCapacity": { + "L1": 100.0, + "L2": 100.0, + "L3": 100.0 + }, + "Nominal": { + "AcFrequency": 50.0, + "AcVoltage": 400.0, + "Power": 25000.0 + }, + "Alarms": "", + "Warnings": "" + }, + "Control": { + "Ac": { + "Power": { + "L1": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + }, + "L2": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + }, + "L3": { + "Active": 3.81, + "Reactive": 0.0, + "Apparent": 3.81 + } + }, + "PhaseControl": "Asymmetric", + "GridType": "GridTied400V50Hz", + "IslandMode": { + "FrequencyOffset": 0.0, + "VoltageAdjustmentFactor": 100.0 + } + }, + "Dc": { + "ReferenceVoltage": 750.0, + "MinVoltage": 720.0, + "MaxVoltage": 780.0, + "PrechargeConfig": "PrechargeDcWithInternal" + }, + "PowerStageEnable": "True", + "ResetAlarmsAndWarnings": "False" + } + } + }, + "Alarms": "", + "Warnings": "", + "Ac": { + "L1": { + "Voltage": 238.9, + "Current": 2.19, + "Phi": 3.14, + "Power": { + "Active": -523.19, + "Reactive": 2.13, + "Apparent": 523.19 + } + }, + "L2": { + "Voltage": 239.35, + "Current": 2.15, + "Phi": -2.7, + "Power": { + "Active": -464.43, + "Reactive": -221.64, + "Apparent": 514.6 + } + }, + "L3": { + "Voltage": 239.85, + "Current": 1.78, + "Phi": -3.11, + "Power": { + "Active": -426.74, + "Reactive": -12.81, + "Apparent": 426.93 + } + }, + "Frequency": 50.0, + "Power": { + "Active": -1414.36, + "Reactive": -232.31, + "Apparent": 1433.31 + } + }, + "Dc": { + "Voltage": 777.5, + "Current": -1.82, + "Power": -1414.36 + } + }, + "DcDc": { + "Dc": { + "Link": { + "Voltage": 775.5, + "Current": 0.0, + "Power": 0.0 + }, + "Battery": { + "Voltage": 61.2, + "Current": 0.0, + "Power": 0.0 + } + }, + "SystemControl": { + "Alarms": "", + "CommunicationTimeout": "00:00:20", + "SystemConfig": "DcDcOnly", + "ResetAlarmsAndWarnings": "True", + "TargetSlave": 2.0, + "UseSlaveIdForAddressing": "True", + "ReferenceFrame": "Producer", + "SlaveErrorHandling": "Relaxed", + "SubSlaveErrorHandling": "Off", + "PowerSetPointActivation": "Immediate", + "PowerSetPointTrigger": "Wait", + "DeviceState": "Operation", + "NumberOfConnectedSlaves": 2.0, + "NumberOfConnectedSubSlaves": 0.0, + "Warnings": "" + }, + "Devices": { + "1": { + "Status": { + "Dc": { + "Link": { + "Voltage": 775.0, + "Current": -0.0, + "Power": -0.0 + }, + "Battery": { + "Voltage": 61.2, + "Current": -0.0, + "Power": -0.0 + } + }, + "OverloadCapacity": 0.0, + "Temperature": { + "InletAir": 25.0, + "HighVoltageModule": 30.0, + "LowVoltageModule": 28.0 + }, + "PowerLimitedBy": "MaxChargeCurrent", + "Alarms": "", + "Warnings": "" + }, + "Control": { + "Vcc": { + "EndPointCurrent": 50.0, + "EndPointVoltage": 50.0, + "StartPointCurrent": 5.0 + }, + "VoltageLimits": { + "MinBatteryVoltageAlarm": 0.0, + "MaxBatteryVoltageAlarm": 64.0, + "MinBatteryVoltage": 32.0, + "MaxBatteryVoltage": 63.0 + }, + "DroopControl": { + "ReferenceVoltage": 750.0, + "LowerVoltage": 20.0, + "UpperVoltage": 20.0, + "VoltageDeadband": 0.0 + }, + "CurrentControl": { + "CurrentSetpoint": 0.0, + "MaxCurrentChangePerMs": 100.0, + "MaxBatteryChargingCurrent": 0.0, + "MaxBatteryDischargingCurrent": 210.0 + }, + "MaxDcPower": 10000.0, + "ControlMode": "VoltageDroop", + "ResetAlarmsAndWarnings": "False", + "PowerStageEnable": "True" + } + }, + "2": { + "Status": { + "Dc": { + "Link": { + "Voltage": 776.0, + "Current": -0.0, + "Power": -0.0 + }, + "Battery": { + "Voltage": 61.2, + "Current": -0.0, + "Power": -0.0 + } + }, + "OverloadCapacity": 0.0, + "Temperature": { + "InletAir": 23.0, + "HighVoltageModule": 30.0, + "LowVoltageModule": 29.0 + }, + "PowerLimitedBy": "MaxChargeCurrent", + "Alarms": "", + "Warnings": "" + }, + "Control": { + "Vcc": { + "EndPointCurrent": 50.0, + "EndPointVoltage": 50.0, + "StartPointCurrent": 5.0 + }, + "VoltageLimits": { + "MinBatteryVoltageAlarm": 0.0, + "MaxBatteryVoltageAlarm": 64.0, + "MinBatteryVoltage": 32.0, + "MaxBatteryVoltage": 63.0 + }, + "DroopControl": { + "ReferenceVoltage": 750.0, + "LowerVoltage": 20.0, + "UpperVoltage": 20.0, + "VoltageDeadband": 0.0 + }, + "CurrentControl": { + "CurrentSetpoint": 0.0, + "MaxCurrentChangePerMs": 100.0, + "MaxBatteryChargingCurrent": 0.0, + "MaxBatteryDischargingCurrent": 210.0 + }, + "MaxDcPower": 10000.0, + "ControlMode": "VoltageDroop", + "ResetAlarmsAndWarnings": "False", + "PowerStageEnable": "True" + } + } + }, + "Alarms": "", + "Warnings": "" + }, + "Battery": { + "Current": -1.25, + "Voltage": 61.31, + "Soc": 99.85, + "Soh": 100.0, + "CurrentMinSoc": 99.3, + "TemperatureCell1": 23.35, + "Power": -76.62, + "LowestCellVoltage": 3.51, + "HighestCellVoltage": 3.89, + "MonomerHighVoltageAlarm": "True", + "MonomerLowVoltageAlarm": "False", + "ChargeSwitchState": "False", + "DischargeSwitchState": "True", + "AvailableDischBatteries": 6.0, + "AvailableChBatteries": 0.0, + "ChargeModeBatteries": 0.0, + "DischargeModeBatteries": 1.0, + "StandbyModeBatteries": 5.0, + "ShutDownModeBatteries": 0.0, + "Eoc": "False", + "Eod": "False", + "Devices": { + "1": { + "BatteryDeligreenDataRecord": { + "FwVersion": 2.0, + "BusVoltage": 61.33, + "BusCurrent": 0.0, + "Power": 0.0, + "TotalBatteryVoltage": 58.66, + "ResidualCapacity": 209.99, + "BatteryCapacity": 210.0, + "Soc": 99.9, + "RatedCapacity": 210.0, + "NumberOfCycles": 58.0, + "Soh": 100.0, + "CellVoltage": "3.742,3.564,3.887,3.644,3.599,3.574,3.601,3.555,3.528,3.58,3.823,3.753,3.693,3.771,3.569,3.778", + "TemperaturesList": { + "CellTemperature1": 22.2, + "CellTemperature2": 21.5, + "CellTemperature3": 21.5, + "CellTemperature4": 21.2, + "EnvironmentTemperature": 28.1, + "PowerTemperature": 23.0 + } + }, + "BatteryDeligreenAlarmRecord": { + "CellAlarmList": "Normal-no alarm,Normal-no alarm,Alarm that analog quantity reaches the upper limit,Normal-no alarm,Normal-no alarm,Normal-no alarm,Normal-no alarm,Normal-no alarm,Normal-no alarm,Normal-no alarm,Alarm that analog quantity reaches the upper limit,Normal-no alarm,Normal-no alarm,Alarm that analog quantity reaches the upper limit,Normal-no alarm,Alarm that analog quantity reaches the upper limit", + "CellTemperatureAlarm": "Normal-no alarm,Normal-no alarm,Normal-no alarm,Normal-no alarm", + "EnviTempAlarm": "Normal-no alarm", + "PowerTempAlarm": "Normal-no alarm", + "CurrentAlarm": "Normal-no alarm", + "TotalVoltageAlarm": "Normal-no alarm", + "AlarmEvent1": { + "VoltageSensorFault": "False", + "TemperatureSensorFault": "False", + "CurrentSensorFault": "False", + "KeySwitchFault": "False", + "CellVoltageDropoutFault": "False", + "ChargeSwitchFault": "False", + "DischargeSwitchFault": "False", + "CurrentLimitSwitchFault": "False" + }, + "AlarmEvent2": { + "MonomerHighVoltageAlarm": "False", + "MonomerOvervoltageProtection": "True", + "MonomerLowVoltageAlarm": "False", + "MonomerUnderVoltageProtection": "False", + "HighVoltageAlarmForTotalVoltage": "False", + "OvervoltageProtectionForTotalVoltage": "False", + "LowVoltageAlarmForTotalVoltage": "False", + "UnderVoltageProtectionForTotalVoltage": "False" + }, + "AlarmEvent3": { + "ChargeHighTemperatureAlarm": "False", + "ChargeOverTemperatureProtection": "False", + "ChargeLowTemperatureAlarm": "False", + "ChargeUnderTemperatureProtection": "False", + "DischargeHighTemperatureAlarm": "False", + "DischargeOverTemperatureProtection": "False", + "DischargeLowTemperatureAlarm": "False", + "DischargeUnderTemperatureProtection": "False" + }, + "AlarmEvent4": { + "EnvironmentHighTemperatureAlarm": "False", + "EnvironmentOverTemperatureProtection": "False", + "EnvironmentLowTemperatureAlarm": "False", + "EnvironmentUnderTemperatureProtection": "False", + "PowerOverTemperatureProtection": "False", + "PowerHighTemperatureAlarm": "False", + "CellLowTemperatureHeating": "False", + "ReservationBit": "False" + }, + "AlarmEvent5": { + "ChargeOverCurrentAlarm": "False", + "ChargeOverCurrentProtection": "False", + "DischargeOverCurrentAlarm": "False", + "DischargeOverCurrentProtection": "False", + "TransientOverCurrentProtection": "False", + "OutputShortCircuitProtection": "False", + "TransientOverCurrentLockout": "False", + "OutputShortCircuitLockout": "False" + }, + "AlarmEvent6": { + "ChargeHighVoltageProtection": "False", + "IntermittentRechargeWaiting": "True", + "ResidualCapacityAlarm": "False", + "ResidualCapacityProtection": "False", + "CellLowVoltageChargingProhibition": "False", + "OutputReversePolarityProtection": "False", + "OutputConnectionFault": "False", + "InsideBit": "False" + }, + "AlarmEvent7": { + "InsideBit1": "False", + "InsideBit2": "False", + "InsideBit3": "False", + "InsideBit4": "False", + "AutomaticChargingWaiting": "False", + "ManualChargingWaiting": "False", + "InsideBit5": "False", + "InsideBit6": "False" + }, + "AlarmEvent8": { + "EepStorageFault": "False", + "RtcError": "False", + "VoltageCalibrationNotPerformed": "False", + "CurrentCalibrationNotPerformed": "False", + "ZeroCalibrationNotPerformed": "False", + "InsideBit1": "False", + "InsideBit2": "False", + "InsideBit3": "False" + }, + "DisconnectionState1": { + "Cell01Disconnection": "False", + "Cell02Disconnection": "False", + "Cell03Disconnection": "False", + "Cell04Disconnection": "False", + "Cell05Disconnection": "False", + "Cell06Disconnection": "False", + "Cell07Disconnection": "False", + "Cell08Disconnection": "False" + }, + "DisconnectionState2": { + "Cell09Disconnection": "False", + "Cell10Disconnection": "False", + "Cell11Disconnection": "False", + "Cell12Disconnection": "False", + "Cell13Disconnection": "False", + "Cell14Disconnection": "False", + "Cell15Disconnection": "False", + "Cell16Disconnection": "False" + }, + "EquilibriumState1": { + "Cell01Equilibrium": "False", + "Cell02Equilibrium": "False", + "Cell03Equilibrium": "False", + "Cell04Equilibrium": "False", + "Cell05Equilibrium": "False", + "Cell06Equilibrium": "False", + "Cell07Equilibrium": "False", + "Cell08Equilibrium": "False" + }, + "EquilibriumState2": { + "Cell09Equilibrium": "False", + "Cell10Equilibrium": "False", + "Cell11Equilibrium": "False", + "Cell12Equilibrium": "False", + "Cell13Equilibrium": "False", + "Cell14Equilibrium": "False", + "Cell15Equilibrium": "False", + "Cell16Equilibrium": "False" + }, + "OnOffState": { + "DischargeSwitchState": "True", + "ChargeSwitchState": "False", + "CurrentLimitSwitchStat": "False", + "HeatingSwitchState": "False", + "ReservationBit1": "False", + "ReservationBit2": "False", + "ReservationBit3": "False", + "ReservationBit4": "False" + }, + "SystemState": { + "Discharge": "False", + "Charge": "False", + "FloatingCharge": "False", + "ReservationBit1": "False", + "Standby": "True", + "Shutdown": "False", + "ReservationBit2": "False", + "ReservationBit3": "False" + } + } + }, + "2" :{...} + } + }, + "GridMeter": { + "Ac": { + "L1": { + "Voltage": 239.0, + "Current": 0.36, + "Phi": -1.55, + "Power": { + "Active": 1.99, + "Reactive": -86.59, + "Apparent": 86.61 + } + }, + "L2": { + "Voltage": 239.3, + "Current": 0.32, + "Phi": -1.66, + "Power": { + "Active": -7.02, + "Reactive": -77.2, + "Apparent": 77.52 + } + }, + "L3": { + "Voltage": 239.5, + "Current": 1.14, + "Phi": -1.96, + "Power": { + "Active": -103.28, + "Reactive": -252.68, + "Apparent": 272.97 + } + }, + "Frequency": 50.0, + "Power": { + "Active": -108.3, + "Reactive": -416.47, + "Apparent": 430.32 + } + }, + "ActivePowerImportT1": 28349.0, + "ActivePowerExportT1": 61296.0, + "ActivePowerImportT2": 28349.0, + "ActivePowerExportT2": 61296.0, + "ActivePowerImportT3": 46886.0, + "ActivePowerExportT3": 21032.0, + "ActivePowerImportT4": 0.0, + "ActivePowerExportT4": 0.0 + }, + "LoadOnAcIsland": { + "Ac": { + "L1": { + "Voltage": 239.0, + "Current": 0.0, + "Phi": 0.0, + "Power": { + "Active": 0.0, + "Reactive": 0.0, + "Apparent": 0.0 + } + }, + "L2": { + "Voltage": 239.3, + "Current": 0.0, + "Phi": 0.0, + "Power": { + "Active": 0.0, + "Reactive": 0.0, + "Apparent": 0.0 + } + }, + "L3": { + "Voltage": 239.5, + "Current": 0.0, + "Phi": 0.0, + "Power": { + "Active": 0.0, + "Reactive": 0.0, + "Apparent": 0.0 + } + }, + "Frequency": 50.0, + "Power": { + "Active": 0.0, + "Reactive": 0.0, + "Apparent": 0.0 + } + }, + "ActivePowerImportT1": 58082.0, + "ActivePowerExportT1": 1569.0, + "ActivePowerImportT2": 58082.0, + "ActivePowerExportT2": 1569.0, + "ActivePowerImportT3": 123.0, + "ActivePowerExportT3": 1.0, + "ActivePowerImportT4": 0.0, + "ActivePowerExportT4": 0.0 + }, + "LoadOnAcGrid": { + "Power": { + "Active": 1306.05, + "Reactive": -184.15, + "Apparent": 1318.97 + } + }, + "AcGridToAcIsland": { + "Power": { + "Active": -1414.36, + "Reactive": -232.31, + "Apparent": 1433.31 + } + }, + "AcDcToDcLink": { + "Power": -1414.36 + }, + "LoadOnDc": { + "Power": -321.32 + }, + "Relays": { + "K1GridBusIsConnectedToGrid": "True", + "K2IslandBusIsConnectedToGridBus": "True", + "K3Inverter1IsConnectedToIslandBus": "True", + "K3Inverter2IsConnectedToIslandBus": "True", + "K3Inverter3IsConnectedToIslandBus": "True", + "K3Inverter4IsConnectedToIslandBus": "True", + "FiWarning": "False", + "FiError": "False", + "K2ConnectIslandBusToGridBus": "True" + }, + "PvOnDc": { + "Dc": { + "Voltage": 779.07, + "Current": 1.4, + "Power": 1093.03 + }, + "NbrOfStrings": 3.0, + "Strings": { + "1": { + "Voltage": 779.25, + "Current": 0.5, + "Power": 391.96 + }, + "2": { + "Voltage": 779.1, + "Current": 0.52, + "Power": 407.47 + }, + "3": { + "Voltage": 778.86, + "Current": 0.38, + "Power": 293.63 + } + }, + "DcWh": 137.11 + }, + "Config": { + "MinSoc": 10.0, + "CurtailP": 5.0, + "PvInstalledPower": 0.0, + "ForceCalibrationChargeState": "RepetitivelyEvery", + "DayAndTimeForRepetitiveCalibration": "05/29/2025 09:00:00", + "DayAndTimeForAdditionalCalibration": "05/23/2025 15:46:00", + "ForceCalibrationDischargeState": "RepetitivelyEvery", + "DownDayAndTimeForRepetitiveCalibration": "06/23/2025 19:00:00", + "DownDayAndTimeForAdditionalCalibration": "01/01/0001 00:00:00", + "DisplayIndividualBatteries": "False", + "PConstant": 0.5, + "GridSetPoint": 0.0, + "BatterySelfDischargePower": 200.0, + "HoldSocZone": 1.0, + "IslandMode": { + "AcDc": { + "MaxDcLinkVoltage": 930.0, + "MinDcLinkVoltage": 690.0, + "ReferenceDcLinkVoltage": 750.0 + }, + "DcDc": { + "LowerDcLinkVoltage": 50.0, + "ReferenceDcLinkVoltage": 750.0, + "UpperDcLinkVoltage": 50.0, + "MaxBatteryChargingCurrent": 210.0, + "MaxBatteryDischargingCurrent": 210.0, + "MaxDcPower": 10000.0, + "MaxChargeBatteryVoltage": 63.0, + "MinDischargeBatteryVoltage": 32.0 + } + }, + "GridTie": { + "AcDc": { + "MaxDcLinkVoltage": 780.0, + "MinDcLinkVoltage": 720.0, + "ReferenceDcLinkVoltage": 750.0 + }, + "DcDc": { + "LowerDcLinkVoltage": 20.0, + "ReferenceDcLinkVoltage": 750.0, + "UpperDcLinkVoltage": 20.0, + "MaxBatteryChargingCurrent": 210.0, + "MaxBatteryDischargingCurrent": 210.0, + "MaxDcPower": 10000.0, + "MaxChargeBatteryVoltage": 63.0, + "MinDischargeBatteryVoltage": 32.0 + } + }, + "Devices": { + "RelaysIp": { + "DeviceState": "Measured", + "Host": "10.0.1.1", + "Port": 502.0 + }, + "TsRelaysIp": { + "DeviceState": "Measured", + "Host": "10.0.1.2", + "Port": 502.0 + }, + "GridMeterIp": { + "DeviceState": "Measured", + "Host": "10.0.4.1", + "Port": 502.0 + }, + "PvOnAcGrid": { + "DeviceState": "Disabled", + "Host": "false", + "Port": 0.0 + }, + "LoadOnAcGrid": { + "DeviceState": "Measured", + "Host": "true", + "Port": 0.0 + }, + "PvOnAcIsland": { + "DeviceState": "Disabled", + "Host": "false", + "Port": 0.0 + }, + "IslandBusLoadMeterIp": { + "DeviceState": "Measured", + "Host": "10.0.4.2", + "Port": 502.0 + }, + "TruConvertAcIp": { + "DeviceState": "Measured", + "Host": "10.0.2.1", + "Port": 502.0 + }, + "PvOnDc": { + "DeviceState": "Measured", + "Host": "10.0.5.1", + "Port": 502.0 + }, + "LoadOnDc": { + "DeviceState": "Measured", + "Host": "false", + "Port": 0.0 + }, + "TruConvertDcIp": { + "DeviceState": "Measured", + "Host": "10.0.3.1", + "Port": 502.0 + }, + "BatteryIp": { + "DeviceState": "Measured", + "Host": "localhost", + "Port": 6855.0 + }, + "BatteryNodes": "0,1,2,3,4,5" + }, + "S3": { + "Bucket": "1-3e5b3069-214a-43ee-8d85-57d72000c19d", + "Region": "sos-ch-dk-2", + "Provider": "exo.io", + "Key": "EXO1752677fa3b7a5dc1b4efcb9", + "Secret": "fUTGbI-I29dPdS9KrJxpnrpBcWbcpAZmoYcECfVEiYU", + "ContentType": "text/plain", + "Host": "1-3e5b3069-214a-43ee-8d85-57d72000c19d.sos-ch-dk-2.exo.io", + "Url": "https://1-3e5b3069-214a-43ee-8d85-57d72000c19d.sos-ch-dk-2.exo.io" + } + }, + "Log": { + "SalimaxAlarmState": "Green" + }, + "EssControl": { + "Mode": "UpwardsCalibrationCharge", + "LimitedBy": "ChargeLimitedByMaxDcBusVoltage", + "PowerCorrection": -355.15, + "PowerSetpoint": 22.85 + }, + "StateMachine": { + "Message": "ESS", + "State": 23.0 + } +} + diff --git a/S3ExtractingTool/extractS3data.py b/S3ExtractingTool/extractS3data.py new file mode 100644 index 000000000..a40765351 --- /dev/null +++ b/S3ExtractingTool/extractS3data.py @@ -0,0 +1,256 @@ +import os +import csv +import subprocess +import argparse +import matplotlib.pyplot as plt +from collections import defaultdict +import zipfile +import base64 +import shutil +import json +import sys + +def extract_timestamp(filename): + timestamp_str = filename[:10] + try: + timestamp = int(timestamp_str) + return timestamp + except ValueError: + return 0 + + +def list_files_in_range(start_timestamp, end_timestamp, sampling_stepsize,product_type,bucket_number): + if product_type == "Salimax" or product_type=="SodistoreMax": + hash = "3e5b3069-214a-43ee-8d85-57d72000c19d" + elif product_type == "Salidomo": + hash = "c0436b6a-d276-4cd8-9c44-1eae86cf5d0e" + else: + raise ValueError("Invalid product type option. Use Salimax or Salidomo or SodistoreMax") + + # Find common prefix + common_prefix = "" + for s_char, e_char in zip(str(start_timestamp), str(end_timestamp)): + if s_char == e_char: + common_prefix += s_char + else: + break + + s3_path = f"s3://{bucket_number}-{hash}/{common_prefix}*" + s3cmd_command = f"s3cmd ls {s3_path}" + + print(f"Running: {s3cmd_command}") + try: + output = subprocess.check_output(s3cmd_command, shell=True, text=True) + files = [line.split()[-1] for line in output.strip().split("\n") if line.strip()] + filenames = [] + for f in files: + name = f.split("/")[-1] # e.g., 1748802020.json + timestamp_str = name.split(".")[0] # extract '1748802020' + if timestamp_str.isdigit() and int(timestamp_str) <= int(end_timestamp): + filenames.append(name) + else: + break + + print(filenames) + return filenames + except subprocess.CalledProcessError: + print(f"No files found for prefix {common_prefix}") + return [] + +def get_nested_value(data, key_path): + try: + for key in key_path: + data = data[key] + return data + except (KeyError, TypeError): + return None + +def process_json_files_to_csv(output_directory, json_files, keys, start_timestamp, end_timestamp, bucket_number, booleans_as_numbers): + # Generate output file name from all keys + keypath = '_'.join(get_last_component(k) for k in keys) + output_csv_filename = f"{keypath}_from_{start_timestamp}_to_{end_timestamp}_bucket_{bucket_number}.csv" + + with open(output_csv_filename, 'w', newline='') as csvfile: + csv_writer = csv.writer(csvfile) + + # Write header: 'time' + key names + header = ['time'] + [k.split('/')[-1] for k in keys] + csv_writer.writerow(header) + + for json_file in json_files: + file_path = os.path.join(output_directory, json_file) + with open(file_path, 'r') as f: + lines = f.readlines() + + i = 0 + while i < len(lines) - 1: + timestamp_line = lines[i].strip() + json_line = lines[i + 1].strip() + i += 2 + + if not timestamp_line.startswith("Timestamp;"): + continue + + try: + timestamp = int(timestamp_line.split(';')[1]) + except ValueError: + continue + + if not (start_timestamp <= timestamp <= end_timestamp): + continue + + try: + data = json.loads(json_line) + except json.JSONDecodeError: + print(f"❌ Failed to parse JSON in {json_file}, line {i}") + continue + + row = [timestamp] + + for key in keys: + value = get_nested_value(data, key.split('/')) + + if booleans_as_numbers and isinstance(value, str) and value.lower() in ["true", "false"]: + value = 1 if value.lower() == "true" else 0 + if value is None: + value = "No value provided" + + row.append(value) + + csv_writer.writerow(row) + + print(f"✅ Extracted data saved in '{output_csv_filename}'") + + +def download_files(bucket_number, filenames_to_download, product_type): + if product_type == "Salimax" or product_type=="SodistoreMax": + hash = "3e5b3069-214a-43ee-8d85-57d72000c19d" + elif product_type == "Salidomo": + hash = "c0436b6a-d276-4cd8-9c44-1eae86cf5d0e" + else: + raise ValueError("Invalid product type option. Use Salimax or Salidomo or SodistoreMax") + output_directory = f"S3cmdData_{bucket_number}" + + if not os.path.exists(output_directory): + os.makedirs(output_directory) + print(f"Directory '{output_directory}' created.") + + for filename in filenames_to_download: + print(filename) + + local_path = os.path.join(output_directory, filename) + if not os.path.exists(local_path): + s3cmd_command = f"s3cmd get s3://{bucket_number}-{hash}/{filename} {output_directory}/" + + try: + subprocess.run(s3cmd_command, shell=True, check=True) + downloaded_files = [file for file in os.listdir(output_directory) if file.startswith(filename)] + if not downloaded_files: + print(f"No matching files found for prefix '{filename}'.") + else: + print(f"Files with prefix '{filename}' downloaded successfully.") + decompress_file(os.path.join(output_directory, filename), output_directory) + except subprocess.CalledProcessError as e: + # print(f"Error downloading files: {e}") + continue + else: + print(f"File '{filename}.json' already exists locally. Skipping download.") + +def decompress_file(compressed_file, output_directory): + base_name = os.path.splitext(os.path.basename(compressed_file))[0] + + with open(compressed_file, 'rb') as file: + compressed_data = file.read() + + # Decode the base64 encoded content + decoded_data = base64.b64decode(compressed_data) + + zip_path = os.path.join(output_directory, 'temp.zip') + with open(zip_path, 'wb') as zip_file: + zip_file.write(decoded_data) + + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(output_directory) + + # Rename the extracted data.json file to the original timestamp-based name + extracted_csv_path = os.path.join(output_directory, 'data.json') + if os.path.exists(extracted_csv_path): + new_csv_path = os.path.join(output_directory, f"{base_name}.json") + + os.rename(extracted_csv_path, new_csv_path) + + os.remove(zip_path) + print(f"Decompressed and renamed '{compressed_file}' to '{new_csv_path}'.") + + +def get_last_component(path): + path_without_slashes = path.replace('/', '') + return path_without_slashes + +def download_and_process_files(bucket_number, start_timestamp, end_timestamp, sampling_stepsize, keys, booleans_as_numbers, exact_match, product_type): + output_directory = f"S3cmdData_{bucket_number}" + + #if os.path.exists(output_directory): + # shutil.rmtree(output_directory) + + if not os.path.exists(output_directory): + os.makedirs(output_directory) + print(f"Directory '{output_directory}' created.") + + filenames_to_check = list_files_in_range(start_timestamp, end_timestamp, sampling_stepsize,product_type,bucket_number) + existing_files = [filename for filename in filenames_to_check if os.path.exists(os.path.join(output_directory, f"{filename}.json"))] + files_to_download = set(filenames_to_check) - set(existing_files) + print(files_to_download) + + #if os.listdir(output_directory): + # print("Files already exist in the local folder. Skipping download.") + #else: + if files_to_download: + download_files(bucket_number, files_to_download, product_type) + + json_files = [file for file in os.listdir(output_directory) if file.endswith('.json')] + json_files.sort(key=extract_timestamp) + + process_json_files_to_csv( + output_directory=output_directory, + json_files=json_files, + keys=keys, + start_timestamp=start_timestamp, + end_timestamp=end_timestamp, + bucket_number=bucket_number, + booleans_as_numbers=booleans_as_numbers + ) + +def parse_keys(input_string): + keys = [key.strip() for key in input_string.split(',')] + return keys + +def main(): + parser = argparse.ArgumentParser(description='Download files from S3 using s3cmd and extract specific values from CSV files.') + parser.add_argument('start_timestamp', type=int, help='The start timestamp for the range (even number)') + parser.add_argument('end_timestamp', type=int, help='The end timestamp for the range (even number)') + parser.add_argument('--keys', type=parse_keys, required=True, help='The part to match from each CSV file, can be a single key or a comma-separated list of keys') + parser.add_argument('--bucket-number', type=int, required=True, help='The number of the bucket to download from') + parser.add_argument('--sampling_stepsize', type=int, required=False, default=1, help='The number of 2sec intervals, which define the length of the sampling interval in S3 file retrieval') + parser.add_argument('--booleans_as_numbers', action="store_true", required=False, help='If key used, then booleans are converted to numbers [0/1], if key not used, then booleans maintained as text [False/True]') + parser.add_argument('--exact_match', action="store_true", required=False, help='If key used, then key has to match exactly "=", else it is enough that key is found "in" text') + parser.add_argument('--product_name', required=True, help='Use Salimax, Salidomo or SodistoreMax') + + args = parser.parse_args() + start_timestamp = args.start_timestamp + end_timestamp = args.end_timestamp + keys = args.keys + bucket_number = args.bucket_number + sampling_stepsize = args.sampling_stepsize + booleans_as_numbers = args.booleans_as_numbers + exact_match = args.exact_match + # new arg for product type + product_type = args.product_name + + if start_timestamp >= end_timestamp: + print("Error: start_timestamp must be smaller than end_timestamp.") + return + download_and_process_files(bucket_number, start_timestamp, end_timestamp, sampling_stepsize, keys, booleans_as_numbers, exact_match, product_type) + +if __name__ == "__main__": + main()