Prepare Release 2024.05.03 (merge development into master)

.github/workflows/repo-maintenance.yml

@@ -0,0 +1,54 @@
+name: 'Repository Maintenance'
+
+on:
+    schedule:
+      - cron: '0 4 * * *'
+    workflow_dispatch:
+
+permissions:
+    issues: write
+    pull-requests: write
+    discussions: write
+
+concurrency:
+    group: lock
+
+jobs:
+    stale:
+        name: 'Stale'
+        runs-on: ubuntu-latest
+        steps:
+          - uses: actions/stale@v9
+            with:
+                days-before-stale: 14
+                days-before-close: 60
+                any-of-labels: 'cant-reproduce,not a bug'
+                stale-issue-label: stale
+                stale-pr-label: stale
+                stale-issue-message: >
+                    This issue has been automatically marked as stale because it has not had
+                    recent activity. It will be closed if no further activity occurs. Thank you
+                    for your contributions.
+
+    lock-threads:
+        name: 'Lock Old Threads'
+        runs-on: ubuntu-latest
+        steps:
+          - uses: dessant/lock-threads@v5
+            with:
+                issue-inactive-days: '30'
+                pr-inactive-days: '30'
+                discussion-inactive-days: '30'
+                log-output: true
+                issue-comment: >
+                    This issue has been automatically locked since there
+                    has not been any recent activity after it was closed.
+                    Please open a new discussion or issue for related concerns.
+                pr-comment: >
+                    This pull request has been automatically locked since there
+                    has not been any recent activity after it was closed.
+                    Please open a new discussion or issue for related concerns.
+                discussion-comment: >
+                    This discussion has been automatically locked since there
+                    has not been any recent activity after it was closed.
+                    Please open a new discussion for related concerns.

include/BatteryStats.h

@@ -7,6 +7,7 @@
 #include "Arduino.h"
 #include "JkBmsDataPoints.h"
 #include "VeDirectShuntController.h"
+#include <cfloat>
 
 // mandatory interface for all kinds of batteries
 class BatteryStats {
@@ -37,7 +38,10 @@ class BatteryStats {
 
         // returns true if the battery reached a critically low voltage/SoC,
         // such that it is in need of charging to prevent degredation.
-        virtual bool needsCharging() const { return false; }
+        virtual bool getImmediateChargingRequest() const { return false; };
+
+        virtual float getChargeCurrent() const { return 0; };
+        virtual float getChargeCurrentLimitation() const { return FLT_MAX; };
 
     protected:
         virtual void mqttPublish() const;
@@ -71,7 +75,9 @@ class PylontechBatteryStats : public BatteryStats {
     public:
         void getLiveViewData(JsonVariant& root) const final;
         void mqttPublish() const final;
-        bool needsCharging() const final { return _chargeImmediately; }
+        bool getImmediateChargingRequest() const { return _chargeImmediately; } ;
+        float getChargeCurrent() const { return _current; } ;
+        float getChargeCurrentLimitation() const { return _chargeCurrentLimitation; } ;
 
     private:
         void setManufacturer(String&& m) { _manufacturer = std::move(m); }
@@ -141,7 +147,7 @@ class VictronSmartShuntStats : public BatteryStats {
         void getLiveViewData(JsonVariant& root) const final;
         void mqttPublish() const final;
 
-        void updateFrom(VeDirectShuntController::veShuntStruct const& shuntData);
+        void updateFrom(VeDirectShuntController::data_t const& shuntData);
 
     private:
         float _current;

include/Configuration.h

@@ -5,7 +5,7 @@
 #include <cstdint>
 
 #define CONFIG_FILENAME "/config.json"
-#define CONFIG_VERSION 0x00011b00 // 0.1.27 // make sure to clean all after change
+#define CONFIG_VERSION 0x00011c00 // 0.1.28 // make sure to clean all after change
 
 #define WIFI_MAX_SSID_STRLEN 32
 #define WIFI_MAX_PASSWORD_STRLEN 64
@@ -39,8 +39,6 @@
 #define POWERMETER_MAX_HTTP_JSON_PATH_STRLEN 256
 #define POWERMETER_HTTP_TIMEOUT 1000
 
-#define JSON_BUFFER_SIZE 15360
-
 struct CHANNEL_CONFIG_T {
     uint16_t MaxChannelPower;
     char Name[CHAN_MAX_NAME_STRLEN];
@@ -62,8 +60,9 @@ struct INVERTER_CONFIG_T {
     CHANNEL_CONFIG_T channel[INV_MAX_CHAN_COUNT];
 };
 
-enum Auth { none, basic, digest };
 struct POWERMETER_HTTP_PHASE_CONFIG_T {
+    enum Auth { None, Basic, Digest };
+    enum Unit { Watts = 0, MilliWatts = 1, KiloWatts = 2 };
     bool Enabled;
     char Url[POWERMETER_MAX_HTTP_URL_STRLEN + 1];
     Auth AuthType;
@@ -73,7 +72,10 @@ struct POWERMETER_HTTP_PHASE_CONFIG_T {
     char HeaderValue[POWERMETER_MAX_HTTP_HEADER_VALUE_STRLEN + 1];
     uint16_t Timeout;
     char JsonPath[POWERMETER_MAX_HTTP_JSON_PATH_STRLEN + 1];
+    Unit PowerUnit;
+    bool SignInverted;
 };
+using PowerMeterHttpConfig = struct POWERMETER_HTTP_PHASE_CONFIG_T;
 
 struct CONFIG_T {
     struct {
@@ -196,7 +198,7 @@ struct CONFIG_T {
         uint32_t SdmAddress;
         uint32_t HttpInterval;
         bool HttpIndividualRequests;
-        POWERMETER_HTTP_PHASE_CONFIG_T Http_Phase[POWERMETER_MAX_PHASES];
+        PowerMeterHttpConfig Http_Phase[POWERMETER_MAX_PHASES];
     } PowerMeter;
 
     struct {
@@ -213,6 +215,7 @@ struct CONFIG_T {
         int32_t TargetPowerConsumption;
         int32_t TargetPowerConsumptionHysteresis;
         int32_t LowerPowerLimit;
+        int32_t BaseLoadLimit;
         int32_t UpperPowerLimit;
         bool IgnoreSoc;
         uint32_t BatterySocStartThreshold;
@@ -238,12 +241,17 @@ struct CONFIG_T {
 
     struct {
         bool Enabled;
+        bool VerboseLogging;
         uint32_t CAN_Controller_Frequency;
         bool Auto_Power_Enabled;
+        bool Auto_Power_BatterySoC_Limits_Enabled;
+        bool Emergency_Charge_Enabled;
         float Auto_Power_Voltage_Limit;
         float Auto_Power_Enable_Voltage_Limit;
         float Auto_Power_Lower_Power_Limit;
         float Auto_Power_Upper_Power_Limit;
+        uint8_t Auto_Power_Stop_BatterySoC_Threshold;
+        float Auto_Power_Target_Power_Consumption;
     } Huawei;
 
 

include/HttpPowerMeter.h

@@ -4,6 +4,10 @@
 #include <stdint.h>
 #include <Arduino.h>
 #include <HTTPClient.h>
+#include "Configuration.h"
+
+using Auth_t = PowerMeterHttpConfig::Auth;
+using Unit_t = PowerMeterHttpConfig::Unit;
 
 class HttpPowerMeterClass {
 public:
@@ -11,21 +15,18 @@ public:
     bool updateValues();
     float getPower(int8_t phase);
     char httpPowerMeterError[256];
-    bool queryPhase(int phase, const String& url, Auth authType, const char* username, const char* password, 
+    bool queryPhase(int phase, PowerMeterHttpConfig const& config);
-        const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath);
-
 
 private:
     float power[POWERMETER_MAX_PHASES];
     HTTPClient httpClient;
     String httpResponse;
-    bool httpRequest(int phase, WiFiClient &wifiClient, const String& host, uint16_t port, const String& uri, bool https, Auth authType, const char* username,
+    bool httpRequest(int phase, WiFiClient &wifiClient, const String& host, uint16_t port, const String& uri, bool https, PowerMeterHttpConfig const& config);
-           const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath);
     bool extractUrlComponents(String url, String& _protocol, String& _hostname, String& _uri, uint16_t& uint16_t, String& _base64Authorization);
     String extractParam(String& authReq, const String& param, const char delimit);
     String getcNonce(const int len);
     String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter);
-    bool tryGetFloatValueForPhase(int phase, const char* jsonPath);
+    bool tryGetFloatValueForPhase(int phase, const char* jsonPath, Unit_t unit, bool signInverted);
     void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue);
     String sha256(const String& data);
 };

include/Huawei_can.h

@@ -125,8 +125,9 @@ public:
     void setMode(uint8_t mode);
 
     RectifierParameters_t * get();
-    uint32_t getLastUpdate();
+    uint32_t getLastUpdate() const { return _lastUpdateReceivedMillis; };
-    bool getAutoPowerStatus();
+    bool getAutoPowerStatus() const { return _autoPowerEnabled; };
+    uint8_t getMode() const { return _mode; };
 
 private:
     void loop();
@@ -150,6 +151,7 @@ private:
 
     uint8_t _autoPowerEnabledCounter = 0;
     bool _autoPowerEnabled = false;
+    bool _batteryEmergencyCharging = false;
 };
 
 extern HuaweiCanClass HuaweiCan;

include/JkBmsController.h

@@ -19,7 +19,9 @@ class Controller : public BatteryProvider {
         void deinit() final;
         void loop() final;
         std::shared_ptr<BatteryStats> getStats() const final { return _stats; }
-        bool usesHwPort2() const final { return true; }
+        bool usesHwPort2() const final {
+            return ARDUINO_USB_CDC_ON_BOOT != 1;
+        }
 
     private:
         enum class Status : unsigned {

include/MqttHandleHass.h

@@ -66,10 +66,10 @@ private:
     void publishInverterNumber(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* icon, const char* category, const char* commandTopic, const char* stateTopic, const char* unitOfMeasure, const int16_t min = 1, const int16_t max = 100);
     void publishInverterBinarySensor(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* subTopic, const char* payload_on, const char* payload_off);
 
-    static void createInverterInfo(DynamicJsonDocument& doc, std::shared_ptr<InverterAbstract> inv);
+    static void createInverterInfo(JsonDocument& doc, std::shared_ptr<InverterAbstract> inv);
-    static void createDtuInfo(DynamicJsonDocument& doc);
+    static void createDtuInfo(JsonDocument& doc);
 
-    static void createDeviceInfo(DynamicJsonDocument& doc, const String& name, const String& identifiers, const String& configuration_url, const String& manufacturer, const String& model, const String& sw_version, const String& via_device = "");
+    static void createDeviceInfo(JsonDocument& doc, const String& name, const String& identifiers, const String& configuration_url, const String& manufacturer, const String& model, const String& sw_version, const String& via_device = "");
 
     static String getDtuUniqueId();
     static String getDtuUrl();

include/MqttHandleVedirect.h

@@ -21,7 +21,7 @@ public:
     void forceUpdate();
 private:
     void loop();
-    std::map<std::string, VeDirectMpptController::veMpptStruct> _kvFrames;
+    std::map<std::string, VeDirectMpptController::data_t> _kvFrames;
 
     Task _loopTask;
 
@@ -33,8 +33,8 @@ private:
 
     bool _PublishFull;
 
-    void publish_mppt_data(const VeDirectMpptController::spData_t &spMpptData,
+    void publish_mppt_data(const VeDirectMpptController::data_t &mpptData,
-                           VeDirectMpptController::veMpptStruct &frame) const;
+                           const VeDirectMpptController::data_t &frame) const;
 };
 
 extern MqttHandleVedirectClass MqttHandleVedirect;

include/MqttHandleVedirectHass.h

@@ -16,13 +16,13 @@ private:
     void publish(const String& subtopic, const String& payload);
     void publishBinarySensor(const char *caption, const char *icon, const char *subTopic,
                              const char *payload_on, const char *payload_off,
-                             const VeDirectMpptController::spData_t &spMpptData);
+                             const VeDirectMpptController::data_t &mpptData);
     void publishSensor(const char *caption, const char *icon, const char *subTopic,
                        const char *deviceClass, const char *stateClass,
                        const char *unitOfMeasurement,
-                       const VeDirectMpptController::spData_t &spMpptData);
+                       const VeDirectMpptController::data_t &mpptData);
     void createDeviceInfo(JsonObject &object,
-                          const VeDirectMpptController::spData_t &spMpptData);
+                          const VeDirectMpptController::data_t &mpptData);
 
     Task _loopTask;
 

include/PowerLimiter.h

@@ -16,12 +16,6 @@
 #define PL_UI_STATE_USE_SOLAR_ONLY 2
 #define PL_UI_STATE_USE_SOLAR_AND_BATTERY 3
 
-typedef enum {
-    EMPTY_WHEN_FULL= 0, 
-    EMPTY_AT_NIGHT
-} batDrainStrategy;
-   
-
 class PowerLimiterClass {
 public:
     enum class Status : unsigned {
@@ -29,8 +23,6 @@ public:
         DisabledByConfig,
         DisabledByMqtt,
         WaitingForValidTimestamp,
-        PowerMeterDisabled,
-        PowerMeterTimeout,
         PowerMeterPending,
         InverterInvalid,
         InverterChanged,
@@ -45,11 +37,11 @@ public:
         NoVeDirect,
         NoEnergy,
         HuaweiPsu,
-        Settling,
         Stable,
     };
 
     void init(Scheduler& scheduler);
+    uint8_t getInverterUpdateTimeouts() const { return _inverterUpdateTimeouts; }
     uint8_t getPowerLimiterState();
     int32_t getLastRequestedPowerLimit() { return _lastRequestedPowerLimit; }
 
@@ -70,6 +62,7 @@ private:
 
     int32_t _lastRequestedPowerLimit = 0;
     bool _shutdownPending = false;
+    std::optional<uint32_t> _oInverterStatsMillis = std::nullopt;
     std::optional<uint32_t> _oUpdateStartMillis = std::nullopt;
     std::optional<int32_t> _oTargetPowerLimitWatts = std::nullopt;
     std::optional<bool> _oTargetPowerState = std::nullopt;
@@ -85,6 +78,7 @@ private:
     uint32_t _nextCalculateCheck = 5000; // time in millis for next NTP check to calulate restart
     bool _fullSolarPassThroughEnabled = false;
     bool _verboseLogging = true;
+    uint8_t _inverterUpdateTimeouts = 0;
 
     frozen::string const& getStatusText(Status status);
     void announceStatus(Status status);

include/PowerMeter.h

@@ -31,6 +31,7 @@ public:
     void init(Scheduler& scheduler);
     float getPowerTotal(bool forceUpdate = true);
     uint32_t getLastPowerMeterUpdate();
+    bool isDataValid();
 
 private:
     void loop();

include/Utils.h

@@ -10,7 +10,6 @@ public:
     static uint64_t generateDtuSerial();
     static int getTimezoneOffset();
     static void restartDtu();
-    static bool checkJsonAlloc(const DynamicJsonDocument& doc, const char* function, const uint16_t line);
+    static bool checkJsonAlloc(const JsonDocument& doc, const char* function, const uint16_t line);
-    static bool checkJsonOverflow(const DynamicJsonDocument& doc, const char* function, const uint16_t line);
     static void removeAllFiles();
 };

include/VictronMppt.h

@@ -25,7 +25,7 @@ public:
     uint32_t getDataAgeMillis(size_t idx) const;
 
     size_t controllerAmount() const { return _controllers.size(); }
-    std::optional<VeDirectMpptController::spData_t> getData(size_t idx = 0) const;
+    std::optional<VeDirectMpptController::data_t> getData(size_t idx = 0) const;
 
     // total output of all MPPT charge controllers in Watts
     int32_t getPowerOutputWatts() const;
@@ -34,13 +34,13 @@ public:
     int32_t getPanelPowerWatts() const;
 
     // sum of total yield of all MPPT charge controllers in kWh
-    double getYieldTotal() const;
+    float getYieldTotal() const;
 
     // sum of today's yield of all MPPT charge controllers in kWh
-    double getYieldDay() const;
+    float getYieldDay() const;
 
     // minimum of all MPPT charge controllers' output voltages in V
-    double getOutputVoltage() const;
+    float getOutputVoltage() const;
 
 private:
     void loop();

include/VictronSmartShunt.h

@@ -9,7 +9,9 @@ public:
     void deinit() final { }
     void loop() final;
     std::shared_ptr<BatteryStats> getStats() const final { return _stats; }
-    bool usesHwPort2() const final { return true; }
+    bool usesHwPort2() const final {
+        return ARDUINO_USB_CDC_ON_BOOT != 1;
+    }
 
 private:
     uint32_t _lastUpdate = 0;

include/WebApi.h

@@ -25,6 +25,7 @@
 #include "WebApi_webapp.h"
 #include "WebApi_ws_console.h"
 #include "WebApi_ws_live.h"
+#include <AsyncJson.h>
 #include "WebApi_ws_vedirect_live.h"
 #include "WebApi_vedirect.h"
 #include "WebApi_ws_Huawei.h"
@@ -45,6 +46,10 @@ public:
 
     static void writeConfig(JsonVariant& retMsg, const WebApiError code = WebApiError::GenericSuccess, const String& message = "Settings saved!");
 
+    static bool parseRequestData(AsyncWebServerRequest* request, AsyncJsonResponse* response, JsonDocument& json_document);
+    static uint64_t parseSerialFromRequest(AsyncWebServerRequest* request, String param_name = "inv");
+    static bool sendJsonResponse(AsyncWebServerRequest* request, AsyncJsonResponse* response, const char* function, const uint16_t line);
+
 private:
     AsyncWebServer _server;
 

include/WebApi_errors.h

@@ -5,10 +5,11 @@ enum WebApiError {
     GenericBase = 1000,
     GenericSuccess,
     GenericNoValueFound,
-    GenericDataTooLarge,
+    GenericDataTooLarge, // not used anymore
     GenericParseError,
     GenericValueMissing,
     GenericWriteFailed,
+    GenericInternalServerError,
 
     DtuBase = 2000,
     DtuSerialZero,

include/WebApi_mqtt.h

@@ -4,8 +4,6 @@
 #include <ESPAsyncWebServer.h>
 #include <TaskSchedulerDeclarations.h>
 
-#define MQTT_JSON_DOC_SIZE 10240
-
 class WebApiMqttClass {
 public:
     void init(AsyncWebServer& server, Scheduler& scheduler);

include/WebApi_powermeter.h

@@ -3,7 +3,8 @@
 
 #include <ESPAsyncWebServer.h>
 #include <TaskSchedulerDeclarations.h>
-
+#include <ArduinoJson.h>
+#include "Configuration.h"
 
 class WebApiPowerMeterClass {
 public:
@@ -13,6 +14,7 @@ private:
     void onStatus(AsyncWebServerRequest* request);
     void onAdminGet(AsyncWebServerRequest* request);
     void onAdminPost(AsyncWebServerRequest* request);
+    void decodeJsonPhaseConfig(JsonObject const& json, PowerMeterHttpConfig& config) const;
     void onTestHttpRequest(AsyncWebServerRequest* request);
 
     AsyncWebServer* _server;

include/WebApi_ws_Huawei.h

@@ -12,7 +12,7 @@ public:
     void init(AsyncWebServer& server, Scheduler& scheduler);
 
 private:
-    void generateJsonResponse(JsonVariant& root);
+    void generateCommonJsonResponse(JsonVariant& root);
     void onLivedataStatus(AsyncWebServerRequest* request);
     void onWebsocketEvent(AsyncWebSocket* server, AsyncWebSocketClient* client, AwsEventType type, void* arg, uint8_t* data, size_t len);
 

include/WebApi_ws_battery.h

@@ -12,7 +12,7 @@ public:
     void init(AsyncWebServer& server, Scheduler& scheduler);
 
 private:
-    void generateJsonResponse(JsonVariant& root);
+    void generateCommonJsonResponse(JsonVariant& root);
     void onLivedataStatus(AsyncWebServerRequest* request);
     void onWebsocketEvent(AsyncWebSocket* server, AsyncWebSocketClient* client, AwsEventType type, void* arg, uint8_t* data, size_t len);
 

include/WebApi_ws_vedirect_live.h

@@ -14,8 +14,8 @@ public:
     void init(AsyncWebServer& server, Scheduler& scheduler);
 
 private:
-    void generateJsonResponse(JsonVariant& root, bool fullUpdate);
+    void generateCommonJsonResponse(JsonVariant& root, bool fullUpdate);
-    static void populateJson(const JsonObject &root, const VeDirectMpptController::spData_t &spMpptData);
+    static void populateJson(const JsonObject &root, const VeDirectMpptController::data_t &mpptData);
     void onLivedataStatus(AsyncWebServerRequest* request);
     void onWebsocketEvent(AsyncWebSocket* server, AsyncWebSocketClient* client, AwsEventType type, void* arg, uint8_t* data, size_t len);
     bool hasUpdate(size_t idx);

include/defaults.h

@@ -22,7 +22,8 @@
 
 #define MDNS_ENABLED false
 
-#define NTP_SERVER "pool.ntp.org"
+#define NTP_SERVER_OLD "pool.ntp.org"
+#define NTP_SERVER "opendtu.pool.ntp.org"
 #define NTP_TIMEZONE "CET-1CEST,M3.5.0,M10.5.0/3"
 #define NTP_TIMEZONEDESCR "Europe/Berlin"
 #define NTP_LONGITUDE 10.4515f
@@ -131,6 +132,7 @@
 #define POWERLIMITER_TARGET_POWER_CONSUMPTION 0
 #define POWERLIMITER_TARGET_POWER_CONSUMPTION_HYSTERESIS 0
 #define POWERLIMITER_LOWER_POWER_LIMIT 10
+#define POWERLIMITER_BASE_LOAD_LIMIT 100
 #define POWERLIMITER_UPPER_POWER_LIMIT 800
 #define POWERLIMITER_IGNORE_SOC false
 #define POWERLIMITER_BATTERY_SOC_START_THRESHOLD 80
@@ -154,5 +156,7 @@
 #define HUAWEI_AUTO_POWER_ENABLE_VOLTAGE_LIMIT 42.0
 #define HUAWEI_AUTO_POWER_LOWER_POWER_LIMIT 150
 #define HUAWEI_AUTO_POWER_UPPER_POWER_LIMIT 2000
+#define HUAWEI_AUTO_POWER_STOP_BATTERYSOC_THRESHOLD 95
+#define HUAWEI_AUTO_POWER_TARGET_POWER_CONSUMPTION 0
 
 #define VERBOSE_LOGGING true

lib/Hoymiles/src/Hoymiles.cpp

@@ -114,7 +114,7 @@ void HoymilesClass::loop()
                 }
 
                 // Fetch grid profile
-                if (iv->Statistics()->getLastUpdate() > 0 && iv->GridProfile()->getLastUpdate() == 0) {
+                if (iv->Statistics()->getLastUpdate() > 0 && (iv->GridProfile()->getLastUpdate() == 0 || !iv->GridProfile()->containsValidData())) {
                     iv->sendGridOnProFileParaRequest();
                 }
 

lib/Hoymiles/src/HoymilesRadio.h

@@ -1,11 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 #pragma once
 
-#include "TimeoutHelper.h"
 #include "commands/CommandAbstract.h"
 #include "types.h"
-#include <memory>
 #include <ThreadSafeQueue.h>
+#include <TimeoutHelper.h>
+#include <memory>
 
 class HoymilesRadio {
 public:

lib/Hoymiles/src/inverters/HMT_4CH.cpp

@@ -70,7 +70,7 @@ bool HMT_4CH::isValidSerial(const uint64_t serial)
 
 String HMT_4CH::typeName() const
 {
-    return F("HMT-1600/1800/2000-4T");
+    return "HMT-1600/1800/2000-4T";
 }
 
 const byteAssign_t* HMT_4CH::getByteAssignment() const

lib/Hoymiles/src/inverters/HMT_6CH.cpp

@@ -84,7 +84,7 @@ bool HMT_6CH::isValidSerial(const uint64_t serial)
 
 String HMT_6CH::typeName() const
 {
-    return F("HMT-1800/2250-6T");
+    return "HMT-1800/2250-6T";
 }
 
 const byteAssign_t* HMT_6CH::getByteAssignment() const

lib/Hoymiles/src/parser/GridProfileParser.cpp

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Copyright (C) 2023 Thomas Basler and others
+ * Copyright (C) 2023 - 2024 Thomas Basler and others
  */
 #include "GridProfileParser.h"
 #include "../Hoymiles.h"
@@ -446,6 +446,11 @@ std::list<GridProfileSection_t> GridProfileParser::getProfile() const
     return l;
 }
 
+bool GridProfileParser::containsValidData() const
+{
+    return _gridProfileLength > 6;
+}
+
 uint8_t GridProfileParser::getSectionSize(const uint8_t section_id, const uint8_t section_version)
 {
     uint8_t count = 0;

lib/Hoymiles/src/parser/GridProfileParser.h

@@ -43,6 +43,8 @@ public:
 
     std::list<GridProfileSection_t> getProfile() const;
 
+    bool containsValidData() const;
+
 private:
     static uint8_t getSectionSize(const uint8_t section_id, const uint8_t section_version);
     static int16_t getSectionStart(const uint8_t section_id, const uint8_t section_version);

lib/ThreadSafeQueue/library.json

@@ -0,0 +1,13 @@
+{
+    "name": "ThreadSafeQueue",
+    "keywords": "queue, threadsafe",
+    "description": "An Arduino for ESP32 thread safe queue implementation",
+    "authors": {
+        "name": "Thomas Basler"
+    },
+    "version": "0.0.1",
+    "frameworks": "arduino",
+    "platforms": [
+        "espressif32"
+    ]
+}

lib/TimeoutHelper/library.json

@@ -0,0 +1,13 @@
+{
+    "name": "TimeoutHelper",
+    "keywords": "timeout",
+    "description": "An Arduino for ESP32 timeout helper",
+    "authors": {
+        "name": "Thomas Basler"
+    },
+    "version": "0.0.1",
+    "frameworks": "arduino",
+    "platforms": [
+        "espressif32"
+    ]
+}

lib/VeDirectFrameHandler/VeDirectData.cpp

@@ -0,0 +1,260 @@
+#include "VeDirectData.h"
+
+template<typename T, size_t L>
+static frozen::string const& getAsString(frozen::map<T, frozen::string, L> const& values, T val)
+{
+	auto pos = values.find(val);
+	if (pos == values.end()) {
+		static constexpr frozen::string dummy("???");
+		return dummy;
+	}
+	return pos->second;
+}
+
+/*
+ * This function returns the product id (PID) as readable text.
+ */
+frozen::string const& veStruct::getPidAsString() const
+{
+	/**
+	 * this map is rendered from [1], which is more recent than [2]. Phoenix
+	 * inverters are not included in the map. unfortunately, the documents do
+	 * not fully align. PID 0xA07F is only present in [1]. PIDs 0xA048, 0xA110,
+	 * and 0xA111 are only present in [2]. PIDs 0xA06D and 0xA078 are rev3 in
+	 * [1] but rev2 in [2].
+	 *
+	 * [1] https://www.victronenergy.com/upload/documents/VE.Direct-Protocol-3.33.pdf
+	 * [2] https://www.victronenergy.com/upload/documents/BlueSolar-HEX-protocol.pdf
+	 */
+	static constexpr frozen::map<uint16_t, frozen::string, 105> values = {
+		{ 0x0203, "BMV-700" },
+		{ 0x0204, "BMV-702" },
+		{ 0x0205, "BMV-700H" },
+		{ 0x0300, "BlueSolar MPPT 70|15" },
+		{ 0xA040, "BlueSolar MPPT 75|50" },
+		{ 0xA041, "BlueSolar MPPT 150|35" },
+		{ 0xA042, "BlueSolar MPPT 75|15" },
+		{ 0xA043, "BlueSolar MPPT 100|15" },
+		{ 0xA044, "BlueSolar MPPT 100|30" },
+		{ 0xA045, "BlueSolar MPPT 100|50" },
+		{ 0xA046, "BlueSolar MPPT 150|70" },
+		{ 0xA047, "BlueSolar MPPT 150|100" },
+		{ 0xA048, "BlueSolar MPPT 75|50 rev2" },
+		{ 0xA049, "BlueSolar MPPT 100|50 rev2" },
+		{ 0xA04A, "BlueSolar MPPT 100|30 rev2" },
+		{ 0xA04B, "BlueSolar MPPT 150|35 rev2" },
+		{ 0xA04C, "BlueSolar MPPT 75|10" },
+		{ 0xA04D, "BlueSolar MPPT 150|45" },
+		{ 0xA04E, "BlueSolar MPPT 150|60" },
+		{ 0xA04F, "BlueSolar MPPT 150|85" },
+		{ 0xA050, "SmartSolar MPPT 250|100" },
+		{ 0xA051, "SmartSolar MPPT 150|100" },
+		{ 0xA052, "SmartSolar MPPT 150|85" },
+		{ 0xA053, "SmartSolar MPPT 75|15" },
+		{ 0xA054, "SmartSolar MPPT 75|10" },
+		{ 0xA055, "SmartSolar MPPT 100|15" },
+		{ 0xA056, "SmartSolar MPPT 100|30" },
+		{ 0xA057, "SmartSolar MPPT 100|50" },
+		{ 0xA058, "SmartSolar MPPT 150|35" },
+		{ 0xA059, "SmartSolar MPPT 150|100 rev2" },
+		{ 0xA05A, "SmartSolar MPPT 150|85 rev2" },
+		{ 0xA05B, "SmartSolar MPPT 250|70" },
+		{ 0xA05C, "SmartSolar MPPT 250|85" },
+		{ 0xA05D, "SmartSolar MPPT 250|60" },
+		{ 0xA05E, "SmartSolar MPPT 250|45" },
+		{ 0xA05F, "SmartSolar MPPT 100|20" },
+		{ 0xA060, "SmartSolar MPPT 100|20 48V" },
+		{ 0xA061, "SmartSolar MPPT 150|45" },
+		{ 0xA062, "SmartSolar MPPT 150|60" },
+		{ 0xA063, "SmartSolar MPPT 150|70" },
+		{ 0xA064, "SmartSolar MPPT 250|85 rev2" },
+		{ 0xA065, "SmartSolar MPPT 250|100 rev2" },
+		{ 0xA066, "BlueSolar MPPT 100|20" },
+		{ 0xA067, "BlueSolar MPPT 100|20 48V" },
+		{ 0xA068, "SmartSolar MPPT 250|60 rev2" },
+		{ 0xA069, "SmartSolar MPPT 250|70 rev2" },
+		{ 0xA06A, "SmartSolar MPPT 150|45 rev2" },
+		{ 0xA06B, "SmartSolar MPPT 150|60 rev2" },
+		{ 0xA06C, "SmartSolar MPPT 150|70 rev2" },
+		{ 0xA06D, "SmartSolar MPPT 150|85 rev3" },
+		{ 0xA06E, "SmartSolar MPPT 150|100 rev3" },
+		{ 0xA06F, "BlueSolar MPPT 150|45 rev2" },
+		{ 0xA070, "BlueSolar MPPT 150|60 rev2" },
+		{ 0xA071, "BlueSolar MPPT 150|70 rev2" },
+		{ 0xA072, "BlueSolar MPPT 150|45 rev3" },
+		{ 0xA073, "SmartSolar MPPT 150|45 rev3" },
+		{ 0xA074, "SmartSolar MPPT 75|10 rev2" },
+		{ 0xA075, "SmartSolar MPPT 75|15 rev2" },
+		{ 0xA076, "BlueSolar MPPT 100|30 rev3" },
+		{ 0xA077, "BlueSolar MPPT 100|50 rev3" },
+		{ 0xA078, "BlueSolar MPPT 150|35 rev3" },
+		{ 0xA079, "BlueSolar MPPT 75|10 rev2" },
+		{ 0xA07A, "BlueSolar MPPT 75|15 rev2" },
+		{ 0xA07B, "BlueSolar MPPT 100|15 rev2" },
+		{ 0xA07C, "BlueSolar MPPT 75|10 rev3" },
+		{ 0xA07D, "BlueSolar MPPT 75|15 rev3" },
+		{ 0xA07E, "SmartSolar MPPT 100|30 12V" },
+		{ 0xA07F, "All-In-1 SmartSolar MPPT 75|15 12V" },
+		{ 0xA102, "SmartSolar MPPT VE.Can 150|70" },
+		{ 0xA103, "SmartSolar MPPT VE.Can 150|45" },
+		{ 0xA104, "SmartSolar MPPT VE.Can 150|60" },
+		{ 0xA105, "SmartSolar MPPT VE.Can 150|85" },
+		{ 0xA106, "SmartSolar MPPT VE.Can 150|100" },
+		{ 0xA107, "SmartSolar MPPT VE.Can 250|45" },
+		{ 0xA108, "SmartSolar MPPT VE.Can 250|60" },
+		{ 0xA109, "SmartSolar MPPT VE.Can 250|70" },
+		{ 0xA10A, "SmartSolar MPPT VE.Can 250|85" },
+		{ 0xA10B, "SmartSolar MPPT VE.Can 250|100" },
+		{ 0xA10C, "SmartSolar MPPT VE.Can 150|70 rev2" },
+		{ 0xA10D, "SmartSolar MPPT VE.Can 150|85 rev2" },
+		{ 0xA10E, "SmartSolar MPPT VE.Can 150|100 rev2" },
+		{ 0xA10F, "BlueSolar MPPT VE.Can 150|100" },
+		{ 0xA110, "SmartSolar MPPT RS 450|100" },
+		{ 0xA111, "SmartSolar MPPT RS 450|200" },
+		{ 0xA112, "BlueSolar MPPT VE.Can 250|70" },
+		{ 0xA113, "BlueSolar MPPT VE.Can 250|100" },
+		{ 0xA114, "SmartSolar MPPT VE.Can 250|70 rev2" },
+		{ 0xA115, "SmartSolar MPPT VE.Can 250|100 rev2" },
+		{ 0xA116, "SmartSolar MPPT VE.Can 250|85 rev2" },
+		{ 0xA117, "BlueSolar MPPT VE.Can 150|100 rev2" },
+		{ 0xA340, "Phoenix Smart IP43 Charger 12|50 (1+1)" },
+		{ 0xA341, "Phoenix Smart IP43 Charger 12|50 (3)" },
+		{ 0xA342, "Phoenix Smart IP43 Charger 24|25 (1+1)" },
+		{ 0xA343, "Phoenix Smart IP43 Charger 24|25 (3)" },
+		{ 0xA344, "Phoenix Smart IP43 Charger 12|30 (1+1)" },
+		{ 0xA345, "Phoenix Smart IP43 Charger 12|30 (3)" },
+		{ 0xA346, "Phoenix Smart IP43 Charger 24|16 (1+1)" },
+		{ 0xA347, "Phoenix Smart IP43 Charger 24|16 (3)" },
+		{ 0xA381, "BMV-712 Smart" },
+		{ 0xA382, "BMV-710H Smart" },
+		{ 0xA383, "BMV-712 Smart Rev2" },
+		{ 0xA389, "SmartShunt 500A/50mV" },
+		{ 0xA38A, "SmartShunt 1000A/50mV" },
+		{ 0xA38B, "SmartShunt 2000A/50mV" },
+		{ 0xA3F0, "Smart BuckBoost 12V/12V-50A" },
+	};
+
+	return getAsString(values, productID_PID);
+}
+
+/*
+ * This function returns the state of operations (CS) as readable text.
+ */
+frozen::string const& veMpptStruct::getCsAsString() const
+{
+	static constexpr frozen::map<uint8_t, frozen::string, 9> values = {
+		{ 0,   "OFF" },
+		{ 2,   "Fault" },
+		{ 3,   "Bulk" },
+		{ 4,   "Absorbtion" },
+		{ 5,   "Float" },
+		{ 7,   "Equalize (manual)" },
+		{ 245, "Starting-up" },
+		{ 247, "Auto equalize / Recondition" },
+		{ 252, "External Control" }
+	};
+
+	return getAsString(values, currentState_CS);
+}
+
+/*
+ * This function returns the state of MPPT (MPPT) as readable text.
+ */
+frozen::string const& veMpptStruct::getMpptAsString() const
+{
+	static constexpr frozen::map<uint8_t, frozen::string, 3> values = {
+		{ 0, "OFF" },
+		{ 1, "Voltage or current limited" },
+		{ 2, "MPP Tracker active" }
+	};
+
+	return getAsString(values, stateOfTracker_MPPT);
+}
+
+/*
+ * This function returns error state (ERR) as readable text.
+ */
+frozen::string const& veMpptStruct::getErrAsString() const
+{
+	static constexpr frozen::map<uint8_t, frozen::string, 20> values = {
+		{ 0,   "No error" },
+		{ 2,   "Battery voltage too high" },
+		{ 17,  "Charger temperature too high" },
+		{ 18,  "Charger over current" },
+		{ 19,  "Charger current reversed" },
+		{ 20,  "Bulk time limit exceeded" },
+		{ 21,  "Current sensor issue(sensor bias/sensor broken)" },
+		{ 26,  "Terminals overheated" },
+		{ 28,  "Converter issue (dual converter models only)" },
+		{ 33,  "Input voltage too high (solar panel)" },
+		{ 34,  "Input current too high (solar panel)" },
+		{ 38,  "Input shutdown (due to excessive battery voltage)" },
+		{ 39,  "Input shutdown (due to current flow during off mode)" },
+		{ 40,  "Input" },
+		{ 65,  "Lost communication with one of devices" },
+		{ 67,  "Synchronisedcharging device configuration issue" },
+		{ 68,  "BMS connection lost" },
+		{ 116, "Factory calibration data lost" },
+		{ 117, "Invalid/incompatible firmware" },
+		{ 118, "User settings invalid" }
+	};
+
+	return getAsString(values, errorCode_ERR);
+}
+
+/*
+ * This function returns the off reason (OR) as readable text.
+ */
+frozen::string const& veMpptStruct::getOrAsString() const
+{
+	static constexpr frozen::map<uint32_t, frozen::string, 10> values = {
+		{ 0x00000000, "Not off" },
+		{ 0x00000001, "No input power" },
+		{ 0x00000002, "Switched off (power switch)" },
+		{ 0x00000004, "Switched off (device moderegister)" },
+		{ 0x00000008, "Remote input" },
+		{ 0x00000010, "Protection active" },
+		{ 0x00000020, "Paygo" },
+		{ 0x00000040, "BMS" },
+		{ 0x00000080, "Engine shutdown detection" },
+		{ 0x00000100, "Analysing input voltage" }
+	};
+
+	return getAsString(values, offReason_OR);
+}
+
+frozen::string const& VeDirectHexData::getResponseAsString() const
+{
+	using Response = VeDirectHexResponse;
+	static constexpr frozen::map<Response, frozen::string, 7> values = {
+		{ Response::DONE, "Done" },
+		{ Response::UNKNOWN, "Unknown" },
+		{ Response::ERROR, "Error" },
+		{ Response::PING, "Ping" },
+		{ Response::GET, "Get" },
+		{ Response::SET, "Set" },
+		{ Response::ASYNC, "Async" }
+	};
+
+	return getAsString(values, rsp);
+}
+
+frozen::string const& VeDirectHexData::getRegisterAsString() const
+{
+	using Register = VeDirectHexRegister;
+	static constexpr frozen::map<Register, frozen::string, 11> values = {
+		{ Register::DeviceMode, "Device Mode" },
+		{ Register::DeviceState, "Device State" },
+		{ Register::RemoteControlUsed, "Remote Control Used" },
+		{ Register::PanelVoltage, "Panel Voltage" },
+		{ Register::ChargerVoltage, "Charger Voltage" },
+		{ Register::NetworkTotalDcInputPower, "Network Total DC Input Power" },
+		{ Register::ChargeControllerTemperature, "Charger Controller Temperature" },
+		{ Register::SmartBatterySenseTemperature, "Smart Battery Sense Temperature" },
+		{ Register::NetworkInfo, "Network Info" },
+		{ Register::NetworkMode, "Network Mode" },
+		{ Register::NetworkStatus, "Network Status" }
+	};
+
+	return getAsString(values, addr);
+}

lib/VeDirectFrameHandler/VeDirectData.h

@@ -0,0 +1,139 @@
+#pragma once
+
+#include <frozen/string.h>
+#include <frozen/map.h>
+
+#define VE_MAX_VALUE_LEN 33 // VE.Direct Protocol: max value size is 33 including /0
+#define VE_MAX_HEX_LEN 100 // Maximum size of hex frame - max payload 34 byte (=68 char) + safe buffer
+
+typedef struct {
+    uint16_t productID_PID = 0;             // product id
+    char serialNr_SER[VE_MAX_VALUE_LEN];    // serial number
+    char firmwareNr_FW[VE_MAX_VALUE_LEN];   // firmware release number
+    uint32_t batteryVoltage_V_mV = 0;       // battery voltage in mV
+    int32_t batteryCurrent_I_mA = 0;        // battery current in mA (can be negative)
+    float mpptEfficiency_Percent = 0;       // efficiency in percent (calculated, moving average)
+
+    frozen::string const& getPidAsString() const; // product ID as string
+} veStruct;
+
+struct veMpptStruct : veStruct {
+    uint8_t  stateOfTracker_MPPT;       // state of MPP tracker
+    uint16_t panelPower_PPV_W;          // panel power in W
+    uint32_t panelVoltage_VPV_mV;       // panel voltage in mV
+    uint32_t panelCurrent_mA;           // panel current in mA (calculated)
+    int16_t  batteryOutputPower_W;      // battery output power in W (calculated, can be negative if load output is used)
+    uint32_t loadCurrent_IL_mA;         // Load current in mA (Available only for models with a load output)
+    bool     loadOutputState_LOAD;      // virtual load output state (on if battery voltage reaches upper limit, off if battery reaches lower limit)
+    uint8_t  currentState_CS;           // current state of operation e.g. OFF or Bulk
+    uint8_t  errorCode_ERR;             // error code
+    uint32_t offReason_OR;              // off reason
+    uint16_t daySequenceNr_HSDS;        // day sequence number 1...365
+    uint32_t yieldTotal_H19_Wh;         // yield total resetable Wh
+    uint32_t yieldToday_H20_Wh;         // yield today Wh
+    uint16_t maxPowerToday_H21_W;       // maximum power today W
+    uint32_t yieldYesterday_H22_Wh;     // yield yesterday Wh
+    uint16_t maxPowerYesterday_H23_W;   // maximum power yesterday W
+
+    // these are values communicated through the HEX protocol. the pair's first
+    // value is the timestamp the respective info was last received. if it is
+    // zero, the value is deemed invalid. the timestamp is reset if no current
+    // value could be retrieved.
+    std::pair<uint32_t, int32_t> MpptTemperatureMilliCelsius;
+    std::pair<uint32_t, int32_t> SmartBatterySenseTemperatureMilliCelsius;
+    std::pair<uint32_t, uint32_t> NetworkTotalDcInputPowerMilliWatts;
+    std::pair<uint32_t, uint8_t> NetworkInfo;
+    std::pair<uint32_t, uint8_t> NetworkMode;
+    std::pair<uint32_t, uint8_t> NetworkStatus;
+
+    frozen::string const& getMpptAsString() const; // state of mppt as string
+    frozen::string const& getCsAsString() const;   // current state as string
+    frozen::string const& getErrAsString() const;  // error state as string
+    frozen::string const& getOrAsString() const;   // off reason as string
+};
+
+struct veShuntStruct : veStruct {
+    int32_t T;                      // Battery temperature
+    bool tempPresent;               // Battery temperature sensor is attached to the shunt
+    int32_t P;                      // Instantaneous power
+    int32_t CE;                     // Consumed Amp Hours
+    int32_t SOC;                    // State-of-charge
+    uint32_t TTG;                   // Time-to-go
+    bool ALARM;                     // Alarm condition active
+    uint16_t alarmReason_AR;        // Alarm Reason
+    int32_t H1;                     // Depth of the deepest discharge
+    int32_t H2;                     // Depth of the last discharge
+    int32_t H3;                     // Depth of the average discharge
+    int32_t H4;                     // Number of charge cycles
+    int32_t H5;                     // Number of full discharges
+    int32_t H6;                     // Cumulative Amp Hours drawn
+    int32_t H7;                     // Minimum main (battery) voltage
+    int32_t H8;                     // Maximum main (battery) voltage
+    int32_t H9;                     // Number of seconds since last full charge
+    int32_t H10;                    // Number of automatic synchronizations
+    int32_t H11;                    // Number of low main voltage alarms
+    int32_t H12;                    // Number of high main voltage alarms
+    int32_t H13;                    // Number of low auxiliary voltage alarms
+    int32_t H14;                    // Number of high auxiliary voltage alarms
+    int32_t H15;                    // Minimum auxiliary (battery) voltage
+    int32_t H16;                    // Maximum auxiliary (battery) voltage
+    int32_t H17;                    // Amount of discharged energy
+    int32_t H18;                    // Amount of charged energy
+    int8_t dcMonitorMode_MON;       // DC monitor mode
+};
+
+enum class VeDirectHexCommand : uint8_t {
+    ENTER_BOOT = 0x0,
+    PING = 0x1,
+    RSV1 = 0x2,
+    APP_VERSION = 0x3,
+    PRODUCT_ID = 0x4,
+    RSV2 = 0x5,
+    RESTART = 0x6,
+    GET = 0x7,
+    SET = 0x8,
+    RSV3 = 0x9,
+    ASYNC = 0xA,
+    RSV4 = 0xB,
+    RSV5 = 0xC,
+    RSV6 = 0xD,
+    RSV7 = 0xE,
+    RSV8 = 0xF
+};
+
+enum class VeDirectHexResponse : uint8_t {
+    DONE = 0x1,
+    UNKNOWN = 0x3,
+    ERROR = 0x4,
+    PING = 0x5,
+    GET = 0x7,
+    SET = 0x8,
+    ASYNC = 0xA
+};
+
+enum class VeDirectHexRegister : uint16_t {
+    DeviceMode = 0x0200,
+    DeviceState = 0x0201,
+    RemoteControlUsed = 0x0202,
+    PanelVoltage = 0xEDBB,
+    ChargerVoltage = 0xEDD5,
+    NetworkTotalDcInputPower = 0x2027,
+    ChargeControllerTemperature = 0xEDDB,
+    SmartBatterySenseTemperature = 0xEDEC,
+    NetworkInfo = 0x200D,
+    NetworkMode = 0x200E,
+    NetworkStatus = 0x200F,
+    HistoryTotal = 0x104F,
+    HistoryMPPTD30 = 0x10BE
+};
+
+struct VeDirectHexData {
+    VeDirectHexResponse rsp;        // hex response code
+    VeDirectHexRegister addr;       // register address
+    uint8_t flags;                  // flags
+    uint32_t value;                 // integer value of register
+    char text[VE_MAX_HEX_LEN];      // text/string response
+
+    frozen::string const& getResponseAsString() const;
+    frozen::string const& getRegisterAsString() const;
+};

lib/VeDirectFrameHandler/VeDirectFrameHandler.cpp

@@ -30,7 +30,7 @@
  * 2020.05.05 - 0.2 - initial release
  * 2020.06.21 - 0.2 - add MIT license, no code changes
  * 2020.08.20 - 0.3 - corrected #include reference
- *
+ * 2024.03.08 - 0.4 - adds the ability to send hex commands and disassemble hex messages
  */
 
 #include <Arduino.h>
@@ -39,18 +39,6 @@
 // The name of the record that contains the checksum.
 static constexpr char checksumTagName[] = "CHECKSUM";
 
-// state machine
-enum States {
-	IDLE = 1,
-	RECORD_BEGIN = 2,
-	RECORD_NAME = 3,
-	RECORD_VALUE = 4,
-	CHECKSUM = 5,
-	RECORD_HEX = 6
-};
-
-
-
 class Silent : public Print {
 	public:
 		size_t write(uint8_t c) final { return 0; }
@@ -58,10 +46,11 @@ class Silent : public Print {
 
 static Silent MessageOutputDummy;
 
-VeDirectFrameHandler::VeDirectFrameHandler() :
+template<typename T>
+VeDirectFrameHandler<T>::VeDirectFrameHandler() :
 	_msgOut(&MessageOutputDummy),
 	_lastUpdate(0),
-	_state(IDLE),
+	_state(State::IDLE),
 	_checksum(0),
 	_textPointer(0),
 	_hexSize(0),
@@ -72,21 +61,27 @@ VeDirectFrameHandler::VeDirectFrameHandler() :
 {
 }
 
-void VeDirectFrameHandler::init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort)
+template<typename T>
+void VeDirectFrameHandler<T>::init(char const* who, int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort)
 {
 	_vedirectSerial = std::make_unique<HardwareSerial>(hwSerialPort);
+	_vedirectSerial->end(); // make sure the UART will be re-initialized
 	_vedirectSerial->begin(19200, SERIAL_8N1, rx, tx);
 	_vedirectSerial->flush();
+	_canSend = (tx != -1);
 	_msgOut = msgOut;
 	_verboseLogging = verboseLogging;
 	_debugIn = 0;
+	snprintf(_logId, sizeof(_logId), "[VE.Direct %s %d/%d]", who, rx, tx);
+	if (_verboseLogging) { _msgOut->printf("%s init complete\r\n", _logId); }
 }
 
-void VeDirectFrameHandler::dumpDebugBuffer() {
+template<typename T>
-	_msgOut->printf("[VE.Direct] serial input (%d Bytes):", _debugIn);
+void VeDirectFrameHandler<T>::dumpDebugBuffer() {
+	_msgOut->printf("%s serial input (%d Bytes):", _logId, _debugIn);
 	for (int i = 0; i < _debugIn; ++i) {
 		if (i % 16 == 0) {
-			_msgOut->printf("\r\n[VE.Direct]");
+			_msgOut->printf("\r\n%s", _logId);
 		}
 		_msgOut->printf(" %02x", _debugBuffer[i]);
 	}
@@ -94,21 +89,30 @@ void VeDirectFrameHandler::dumpDebugBuffer() {
 	_debugIn = 0;
 }
 
-void VeDirectFrameHandler::loop()
+template<typename T>
+void VeDirectFrameHandler<T>::reset()
+{
+	_checksum = 0;
+	_state = State::IDLE;
+	_textData.clear();
+}
+
+template<typename T>
+void VeDirectFrameHandler<T>::loop()
 {
 	while ( _vedirectSerial->available()) {
 		rxData(_vedirectSerial->read());
 		_lastByteMillis = millis();
 	}
 
-	// there will never be a large gap between two bytes of the same frame.
+	// there will never be a large gap between two bytes.
 	// if such a large gap is observed, reset the state machine so it tries
-	// to decode a new frame once more data arrives.
+	// to decode a new frame / hex messages once more data arrives.
-	if (IDLE != _state && _lastByteMillis + 500 < millis()) {
+	if ((State::IDLE != _state) && ((millis() - _lastByteMillis) > 500)) {
-		_msgOut->printf("[VE.Direct] Resetting state machine (was %d) after timeout\r\n", _state);
+		_msgOut->printf("%s Resetting state machine (was %d) after timeout\r\n",
+				_logId, static_cast<unsigned>(_state));
 		if (_verboseLogging) { dumpDebugBuffer(); }
-		_checksum = 0;
+		reset();
-		_state = IDLE;
 	}
 }
 
@@ -117,44 +121,45 @@ void VeDirectFrameHandler::loop()
  *  This function is called by loop() which passes a byte of serial data
  *  Based on Victron's example code. But using String and Map instead of pointer and arrays
  */
-void VeDirectFrameHandler::rxData(uint8_t inbyte)
+template<typename T>
+void VeDirectFrameHandler<T>::rxData(uint8_t inbyte)
 {
 	if (_verboseLogging) {
 		_debugBuffer[_debugIn] = inbyte;
 		_debugIn = (_debugIn + 1) % _debugBuffer.size();
 		if (0 == _debugIn) {
-			_msgOut->println("[VE.Direct] ERROR: debug buffer overrun!");
+			_msgOut->printf("%s ERROR: debug buffer overrun!\r\n", _logId);
 		}
 	}
 
-	if ( (inbyte == ':') && (_state != CHECKSUM) ) {
+	if ( (inbyte == ':') && (_state != State::CHECKSUM) ) {
 		_prevState = _state; //hex frame can interrupt TEXT
-		_state = RECORD_HEX;
+		_state = State::RECORD_HEX;
 		_hexSize = 0;
 	}
-	if (_state != RECORD_HEX) {
+	if (_state != State::RECORD_HEX) {
 		_checksum += inbyte;
 	}
 	inbyte = toupper(inbyte);
 
 	switch(_state) {
-	case IDLE:
+	case State::IDLE:
 		/* wait for \n of the start of an record */
 		switch(inbyte) {
 		case '\n':
-			_state = RECORD_BEGIN;
+			_state = State::RECORD_BEGIN;
 			break;
 		case '\r': /* Skip */
 		default:
 			break;
 		}
 		break;
-	case RECORD_BEGIN:
+	case State::RECORD_BEGIN:
 		_textPointer = _name;
 		*_textPointer++ = inbyte;
-		_state = RECORD_NAME;
+		_state = State::RECORD_NAME;
 		break;
-	case RECORD_NAME:
+	case State::RECORD_NAME:
 		// The record name is being received, terminated by a \t
 		switch(inbyte) {
 		case '\t':
@@ -162,12 +167,12 @@ void VeDirectFrameHandler::rxData(uint8_t inbyte)
 			if ( _textPointer < (_name + sizeof(_name)) ) {
 				*_textPointer = 0; /* Zero terminate */
 				if (strcmp(_name, checksumTagName) == 0) {
-					_state = CHECKSUM;
+					_state = State::CHECKSUM;
 					break;
 				}
 			}
 			_textPointer = _value; /* Reset value pointer */
-			_state = RECORD_VALUE;
+			_state = State::RECORD_VALUE;
 			break;
 		case '#': /* Ignore # from serial number*/
 			break;
@@ -178,15 +183,15 @@ void VeDirectFrameHandler::rxData(uint8_t inbyte)
 			break;
 		}
 		break;
-	case RECORD_VALUE:
+	case State::RECORD_VALUE:
 		// The record value is being received.  The \r indicates a new record.
 		switch(inbyte) {
 		case '\n':
 			if ( _textPointer < (_value + sizeof(_value)) ) {
 				*_textPointer = 0; // make zero ended
-				textRxEvent(_name, _value);
+				_textData.push_back({_name, _value});
 			}
-			_state = RECORD_BEGIN;
+			_state = State::RECORD_BEGIN;
 			break;
 		case '\r': /* Skip */
 			break;
@@ -197,221 +202,120 @@ void VeDirectFrameHandler::rxData(uint8_t inbyte)
 			break;
 		}
 		break;
-	case CHECKSUM:
+	case State::CHECKSUM:
 	{
-		bool valid = _checksum == 0;
-		if (!valid) {
-			_msgOut->printf("[VE.Direct] checksum 0x%02x != 0, invalid frame\r\n", _checksum);
-		}
 		if (_verboseLogging) { dumpDebugBuffer(); }
-		_checksum = 0;
+		if (_checksum == 0) {
-		_state = IDLE;
+			for (auto const& event : _textData) {
-		if (valid) { frameValidEvent(); }
+				processTextData(event.first, event.second);
+			}
+			_lastUpdate = millis();
+			frameValidEvent();
+		}
+		else {
+			_msgOut->printf("%s checksum 0x%02x != 0x00, invalid frame\r\n", _logId, _checksum);
+		}
+		reset();
 		break;
 	}
-	case RECORD_HEX:
+	case State::RECORD_HEX:
 		_state = hexRxEvent(inbyte);
 		break;
 	}
 }
 
 /*
- * textRxEvent
  * This function is called every time a new name/value is successfully parsed.  It writes the values to the temporary buffer.
  */
-bool VeDirectFrameHandler::textRxEvent(std::string const& who, char* name, char* value, veStruct& frame) {
+template<typename T>
+void VeDirectFrameHandler<T>::processTextData(std::string const& name, std::string const& value) {
 	if (_verboseLogging) {
-		_msgOut->printf("[Victron %s] Text Event %s: Value: %s\r\n",
+		_msgOut->printf("%s Text Data '%s' = '%s'\r\n",
-				who.c_str(), name, value );
+				_logId, name.c_str(), value.c_str());
 	}
 
-	if (strcmp(name, "PID") == 0) {
+	if (processTextDataDerived(name, value)) { return; }
-		frame.PID = strtol(value, nullptr, 0);
+
-		return true;
+	if (name == "PID") {
+		_tmpFrame.productID_PID = strtol(value.c_str(), nullptr, 0);
+		return;
 	}
 
-	if (strcmp(name, "SER") == 0) {
+	if (name == "SER") {
-		strcpy(frame.SER, value);
+		strcpy(_tmpFrame.serialNr_SER, value.c_str());
-		return true;
+		return;
 	}
 
-	if (strcmp(name, "FW") == 0) {
+	if (name == "FW") {
-		strcpy(frame.FW, value);
+		strcpy(_tmpFrame.firmwareNr_FW, value.c_str());
-		return true;
+		return;
 	}
 
-	if (strcmp(name, "V") == 0) {
+	if (name == "V") {
-		frame.V = round(atof(value) / 10.0) / 100.0;
+		_tmpFrame.batteryVoltage_V_mV = atol(value.c_str());
-		return true;
+		return;
 	}
 
-	if (strcmp(name, "I") == 0) {
+	if (name == "I") {
-		frame.I = round(atof(value) / 10.0) / 100.0;
+		_tmpFrame.batteryCurrent_I_mA = atol(value.c_str());
-		return true;
+		return;
 	}
 
-	return false;
+	_msgOut->printf("%s Unknown text data '%s' (value '%s')\r\n",
+			_logId, name.c_str(), value.c_str());
 }
 
-
-
 /*
  *  hexRxEvent
  *  This function records hex answers or async messages
  */
-int VeDirectFrameHandler::hexRxEvent(uint8_t inbyte) {
+template<typename T>
-	int ret=RECORD_HEX; // default - continue recording until end of frame
+typename VeDirectFrameHandler<T>::State VeDirectFrameHandler<T>::hexRxEvent(uint8_t inbyte)
+{
+	State ret = State::RECORD_HEX; // default - continue recording until end of frame
 
 	switch (inbyte) {
 	case '\n':
+		// now we can analyse the hex message
+		_hexBuffer[_hexSize] = '\0';
+		VeDirectHexData data;
+		if (disassembleHexData(data) && !hexDataHandler(data) && _verboseLogging) {
+			_msgOut->printf("%s Unhandled Hex %s Response, addr: 0x%04X (%s), "
+					"value: 0x%08X, flags: 0x%02X\r\n", _logId,
+					data.getResponseAsString().data(),
+					static_cast<unsigned>(data.addr),
+					data.getRegisterAsString().data(),
+					data.value, data.flags);
+		}
+
 		// restore previous state
 		ret=_prevState;
 		break;
 
 	default:
-		_hexSize++;
+		_hexBuffer[_hexSize++]=inbyte;
+
 		if (_hexSize>=VE_MAX_HEX_LEN) { // oops -buffer overflow - something went wrong, we abort
-			_msgOut->println("[VE.Direct] hexRx buffer overflow - aborting read");
+			_msgOut->printf("%s hexRx buffer overflow - aborting read\r\n", _logId);
 			_hexSize=0;
-			ret=IDLE;
+			ret = State::IDLE;
 		}
 	}
 
 	return ret;
 }
 
-bool VeDirectFrameHandler::isDataValid(veStruct const& frame) const {
+template<typename T>
-	return strlen(frame.SER) > 0 && _lastUpdate > 0 && (millis() - _lastUpdate) < (10 * 1000);
+bool VeDirectFrameHandler<T>::isDataValid() const
+{
+	// VE.Direct text frame data is valid if we receive a device serialnumber and
+	// the data is not older as 10 seconds
+	// we accept a glitch where the data is valid for ten seconds when serialNr_SER != "" and (millis() - _lastUpdate) overflows
+	return strlen(_tmpFrame.serialNr_SER) > 0 && (millis() - _lastUpdate) < (10 * 1000);
 }
 
-uint32_t VeDirectFrameHandler::getLastUpdate() const
+template<typename T>
+uint32_t VeDirectFrameHandler<T>::getLastUpdate() const
 {
 	return _lastUpdate;
 }
-
-/*
- * getPidAsString
- * This function returns the product id (PID) as readable text.
- */
-frozen::string const& VeDirectFrameHandler::veStruct::getPidAsString() const
-{
-	/**
-	 * this map is rendered from [1], which is more recent than [2]. Phoenix
-	 * inverters are not included in the map. unfortunately, the documents do
-	 * not fully align. PID 0xA07F is only present in [1]. PIDs 0xA048, 0xA110,
-	 * and 0xA111 are only present in [2]. PIDs 0xA06D and 0xA078 are rev3 in
-	 * [1] but rev2 in [2].
-	 *
-	 * [1] https://www.victronenergy.com/upload/documents/VE.Direct-Protocol-3.33.pdf
-	 * [2] https://www.victronenergy.com/upload/documents/BlueSolar-HEX-protocol.pdf
-	 */
-	static constexpr frozen::map<uint16_t, frozen::string, 105> values = {
-		{ 0x0203, "BMV-700" },
-		{ 0x0204, "BMV-702" },
-		{ 0x0205, "BMV-700H" },
-		{ 0x0300, "BlueSolar MPPT 70|15" },
-		{ 0xA040, "BlueSolar MPPT 75|50" },
-		{ 0xA041, "BlueSolar MPPT 150|35" },
-		{ 0xA042, "BlueSolar MPPT 75|15" },
-		{ 0xA043, "BlueSolar MPPT 100|15" },
-		{ 0xA044, "BlueSolar MPPT 100|30" },
-		{ 0xA045, "BlueSolar MPPT 100|50" },
-		{ 0xA046, "BlueSolar MPPT 150|70" },
-		{ 0xA047, "BlueSolar MPPT 150|100" },
-		{ 0xA048, "BlueSolar MPPT 75|50 rev2" },
-		{ 0xA049, "BlueSolar MPPT 100|50 rev2" },
-		{ 0xA04A, "BlueSolar MPPT 100|30 rev2" },
-		{ 0xA04B, "BlueSolar MPPT 150|35 rev2" },
-		{ 0xA04C, "BlueSolar MPPT 75|10" },
-		{ 0xA04D, "BlueSolar MPPT 150|45" },
-		{ 0xA04E, "BlueSolar MPPT 150|60" },
-		{ 0xA04F, "BlueSolar MPPT 150|85" },
-		{ 0xA050, "SmartSolar MPPT 250|100" },
-		{ 0xA051, "SmartSolar MPPT 150|100" },
-		{ 0xA052, "SmartSolar MPPT 150|85" },
-		{ 0xA053, "SmartSolar MPPT 75|15" },
-		{ 0xA054, "SmartSolar MPPT 75|10" },
-		{ 0xA055, "SmartSolar MPPT 100|15" },
-		{ 0xA056, "SmartSolar MPPT 100|30" },
-		{ 0xA057, "SmartSolar MPPT 100|50" },
-		{ 0xA058, "SmartSolar MPPT 150|35" },
-		{ 0xA059, "SmartSolar MPPT 150|100 rev2" },
-		{ 0xA05A, "SmartSolar MPPT 150|85 rev2" },
-		{ 0xA05B, "SmartSolar MPPT 250|70" },
-		{ 0xA05C, "SmartSolar MPPT 250|85" },
-		{ 0xA05D, "SmartSolar MPPT 250|60" },
-		{ 0xA05E, "SmartSolar MPPT 250|45" },
-		{ 0xA05F, "SmartSolar MPPT 100|20" },
-		{ 0xA060, "SmartSolar MPPT 100|20 48V" },
-		{ 0xA061, "SmartSolar MPPT 150|45" },
-		{ 0xA062, "SmartSolar MPPT 150|60" },
-		{ 0xA063, "SmartSolar MPPT 150|70" },
-		{ 0xA064, "SmartSolar MPPT 250|85 rev2" },
-		{ 0xA065, "SmartSolar MPPT 250|100 rev2" },
-		{ 0xA066, "BlueSolar MPPT 100|20" },
-		{ 0xA067, "BlueSolar MPPT 100|20 48V" },
-		{ 0xA068, "SmartSolar MPPT 250|60 rev2" },
-		{ 0xA069, "SmartSolar MPPT 250|70 rev2" },
-		{ 0xA06A, "SmartSolar MPPT 150|45 rev2" },
-		{ 0xA06B, "SmartSolar MPPT 150|60 rev2" },
-		{ 0xA06C, "SmartSolar MPPT 150|70 rev2" },
-		{ 0xA06D, "SmartSolar MPPT 150|85 rev3" },
-		{ 0xA06E, "SmartSolar MPPT 150|100 rev3" },
-		{ 0xA06F, "BlueSolar MPPT 150|45 rev2" },
-		{ 0xA070, "BlueSolar MPPT 150|60 rev2" },
-		{ 0xA071, "BlueSolar MPPT 150|70 rev2" },
-		{ 0xA072, "BlueSolar MPPT 150|45 rev3" },
-		{ 0xA073, "SmartSolar MPPT 150|45 rev3" },
-		{ 0xA074, "SmartSolar MPPT 75|10 rev2" },
-		{ 0xA075, "SmartSolar MPPT 75|15 rev2" },
-		{ 0xA076, "BlueSolar MPPT 100|30 rev3" },
-		{ 0xA077, "BlueSolar MPPT 100|50 rev3" },
-		{ 0xA078, "BlueSolar MPPT 150|35 rev3" },
-		{ 0xA079, "BlueSolar MPPT 75|10 rev2" },
-		{ 0xA07A, "BlueSolar MPPT 75|15 rev2" },
-		{ 0xA07B, "BlueSolar MPPT 100|15 rev2" },
-		{ 0xA07C, "BlueSolar MPPT 75|10 rev3" },
-		{ 0xA07D, "BlueSolar MPPT 75|15 rev3" },
-		{ 0xA07E, "SmartSolar MPPT 100|30 12V" },
-		{ 0xA07F, "All-In-1 SmartSolar MPPT 75|15 12V" },
-		{ 0xA102, "SmartSolar MPPT VE.Can 150|70" },
-		{ 0xA103, "SmartSolar MPPT VE.Can 150|45" },
-		{ 0xA104, "SmartSolar MPPT VE.Can 150|60" },
-		{ 0xA105, "SmartSolar MPPT VE.Can 150|85" },
-		{ 0xA106, "SmartSolar MPPT VE.Can 150|100" },
-		{ 0xA107, "SmartSolar MPPT VE.Can 250|45" },
-		{ 0xA108, "SmartSolar MPPT VE.Can 250|60" },
-		{ 0xA109, "SmartSolar MPPT VE.Can 250|70" },
-		{ 0xA10A, "SmartSolar MPPT VE.Can 250|85" },
-		{ 0xA10B, "SmartSolar MPPT VE.Can 250|100" },
-		{ 0xA10C, "SmartSolar MPPT VE.Can 150|70 rev2" },
-		{ 0xA10D, "SmartSolar MPPT VE.Can 150|85 rev2" },
-		{ 0xA10E, "SmartSolar MPPT VE.Can 150|100 rev2" },
-		{ 0xA10F, "BlueSolar MPPT VE.Can 150|100" },
-		{ 0xA110, "SmartSolar MPPT RS 450|100" },
-		{ 0xA111, "SmartSolar MPPT RS 450|200" },
-		{ 0xA112, "BlueSolar MPPT VE.Can 250|70" },
-		{ 0xA113, "BlueSolar MPPT VE.Can 250|100" },
-		{ 0xA114, "SmartSolar MPPT VE.Can 250|70 rev2" },
-		{ 0xA115, "SmartSolar MPPT VE.Can 250|100 rev2" },
-		{ 0xA116, "SmartSolar MPPT VE.Can 250|85 rev2" },
-		{ 0xA117, "BlueSolar MPPT VE.Can 150|100 rev2" },
-		{ 0xA340, "Phoenix Smart IP43 Charger 12|50 (1+1)" },
-		{ 0xA341, "Phoenix Smart IP43 Charger 12|50 (3)" },
-		{ 0xA342, "Phoenix Smart IP43 Charger 24|25 (1+1)" },
-		{ 0xA343, "Phoenix Smart IP43 Charger 24|25 (3)" },
-		{ 0xA344, "Phoenix Smart IP43 Charger 12|30 (1+1)" },
-		{ 0xA345, "Phoenix Smart IP43 Charger 12|30 (3)" },
-		{ 0xA346, "Phoenix Smart IP43 Charger 24|16 (1+1)" },
-		{ 0xA347, "Phoenix Smart IP43 Charger 24|16 (3)" },
-		{ 0xA381, "BMV-712 Smart" },
-		{ 0xA382, "BMV-710H Smart" },
-		{ 0xA383, "BMV-712 Smart Rev2" },
-		{ 0xA389, "SmartShunt 500A/50mV" },
-		{ 0xA38A, "SmartShunt 1000A/50mV" },
-		{ 0xA38B, "SmartShunt 2000A/50mV" },
-		{ 0xA3F0, "Smart BuckBoost 12V/12V-50A" },
-	};
-
-	return getAsString(values, PID);
-}

lib/VeDirectFrameHandler/VeDirectFrameHandler.h

@@ -6,6 +6,7 @@
  * 2020.05.05 - 0.2 - initial release
  * 2021.02.23 - 0.3 - change frameLen to 22 per VE.Direct Protocol version 3.30
  * 2022.08.20 - 0.4 - changes for OpenDTU
+ * 2024.03.08 - 0.4 - adds the ability to send hex commands and disassemble hex messages
  *
  */
 
@@ -13,67 +14,79 @@
 
 #include <Arduino.h>
 #include <array>
-#include <frozen/string.h>
-#include <frozen/map.h>
 #include <memory>
+#include <utility>
+#include <deque>
+#include "VeDirectData.h"
 
-#define VE_MAX_VALUE_LEN 33 // VE.Direct Protocol: max value size is 33 including /0
+template<typename T>
-#define VE_MAX_HEX_LEN 100 // Maximum size of hex frame - max payload 34 byte (=68 char) + safe buffer
-
 class VeDirectFrameHandler {
 public:
-    VeDirectFrameHandler();
+    virtual void loop();                         // main loop to read ve.direct data
-    virtual void init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort);
-    void loop();                                 // main loop to read ve.direct data
     uint32_t getLastUpdate() const;              // timestamp of last successful frame read
+    bool isDataValid() const;                    // return true if data valid and not outdated
+    T const& getData() const { return _tmpFrame; }
+    bool sendHexCommand(VeDirectHexCommand cmd, VeDirectHexRegister addr, uint32_t value = 0, uint8_t valsize = 0);
 
 protected:
+    VeDirectFrameHandler();
+    void init(char const* who, int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort);
+    virtual bool hexDataHandler(VeDirectHexData const &data) { return false; } // handles the disassembeled hex response
+
     bool _verboseLogging;
     Print* _msgOut;
     uint32_t _lastUpdate;
 
-    typedef struct {
+    T _tmpFrame;
-        uint16_t PID = 0;               // product id
-        char SER[VE_MAX_VALUE_LEN];     // serial number
-        char FW[VE_MAX_VALUE_LEN];      // firmware release number
-        double V = 0;                   // battery voltage in V
-        double I = 0;                   // battery current in A
-        double E = 0;                   // efficiency in percent (calculated, moving average)
 
-        frozen::string const& getPidAsString() const; // product ID as string
+    bool _canSend;
-    } veStruct;
+    char _logId[32];
-
-    bool textRxEvent(std::string const& who, char* name, char* value, veStruct& frame);
-    bool isDataValid(veStruct const& frame) const;      // return true if data valid and not outdated
-
-    template<typename T, size_t L>
-    static frozen::string const& getAsString(frozen::map<T, frozen::string, L> const& values, T val)
-    {
-        auto pos = values.find(val);
-        if (pos == values.end()) {
-            static constexpr frozen::string dummy("???");
-            return dummy;
-        }
-        return pos->second;
-    }
 
 private:
-    void setLastUpdate();                     // set timestampt after successful frame read
+    void reset();
     void dumpDebugBuffer();
     void rxData(uint8_t inbyte);              // byte of serial data
-    virtual void textRxEvent(char *, char *) = 0;
+    void processTextData(std::string const& name, std::string const& value);
-    virtual void frameValidEvent() = 0;
+    virtual bool processTextDataDerived(std::string const& name, std::string const& value) = 0;
-    int hexRxEvent(uint8_t);
+    virtual void frameValidEvent() { }
+    bool disassembleHexData(VeDirectHexData &data);     //return true if disassembling was possible
 
     std::unique_ptr<HardwareSerial> _vedirectSerial;
-    int _state;                                // current state
+
-    int _prevState;                            // previous state
+    enum class State {
+        IDLE = 1,
+        RECORD_BEGIN = 2,
+        RECORD_NAME = 3,
+        RECORD_VALUE = 4,
+        CHECKSUM = 5,
+        RECORD_HEX = 6
+    };
+    State _state;
+    State _prevState;
+
+    State hexRxEvent(uint8_t inbyte);
+
     uint8_t _checksum;                         // checksum value
     char * _textPointer;                       // pointer to the private buffer we're writing to, name or value
     int _hexSize;                              // length of hex buffer
+    char _hexBuffer[VE_MAX_HEX_LEN];           // buffer for received hex frames
     char _name[VE_MAX_VALUE_LEN];              // buffer for the field name
     char _value[VE_MAX_VALUE_LEN];             // buffer for the field value
     std::array<uint8_t, 512> _debugBuffer;
     unsigned _debugIn;
-    uint32_t _lastByteMillis;
+    uint32_t _lastByteMillis;                  // time of last parsed byte
+
+    /**
+     * not every frame contains every value the device is communicating, i.e.,
+     * a set of values can be fragmented across multiple frames. frames can be
+     * invalid. in order to only process data from valid frames, we add data
+     * to this queue and only process it once the frame was found to be valid.
+     * this also handles fragmentation nicely, since there is no need to reset
+     * our data buffer. we simply update the interpreted data from this event
+     * queue, which is fine as we know the source frame was valid.
+     */
+    std::deque<std::pair<std::string, std::string>> _textData;
 };
+
+template class VeDirectFrameHandler<veMpptStruct>;
+template class VeDirectFrameHandler<veShuntStruct>;

lib/VeDirectFrameHandler/VeDirectFrameHexHandler.cpp

@@ -0,0 +1,226 @@
+/* VeDirectFrame
+HexHandler.cpp
+ *
+ * Library to read/write from Victron devices using VE.Direct Hex protocol.
+ * Add on to Victron framehandler reference implementation.
+ *
+ * How to use:
+ * 1. Use sendHexCommand() to send hex messages. Use the Victron documentation to find the parameter.
+ * 2. The from class "VeDirectFrameHandler" derived class X must overwrite the function
+ *    void VeDirectFrameHandler::hexDataHandler(VeDirectHexData const &data)
+ *    to handle the received hex messages. All hex messages will be forwarted to function hexDataHandler()
+ * 3. Analyse the content of data (struct VeDirectHexData) to check if a message fits.
+ *
+ * 2024.03.08 - 0.4 - adds the ability to send hex commands and to parse hex messages
+ *
+ */
+#include <Arduino.h>
+#include "VeDirectFrameHandler.h"
+
+/*
+ * calcHexFrameCheckSum()
+ * help function to calculate the hex checksum
+ */
+#define ascii2hex(v) (v-48-(v>='A'?7:0))
+#define hex2byte(b) (ascii2hex(*(b)))*16+((ascii2hex(*(b+1))))
+static uint8_t calcHexFrameCheckSum(const char* buffer, int size) {
+    uint8_t checksum=0x55-ascii2hex(buffer[1]);
+    for (int i=2; i<size; i+=2)
+        checksum -= hex2byte(buffer+i);
+    return (checksum);
+}
+
+
+/*
+ * AsciiHexLE2Int()
+ * help function to convert AsciiHex Little Endian to uint32_t
+ * ascii:   pointer to Ascii Hex Little Endian data
+ * anz:     1,2,4 or 8 nibble
+ */
+static uint32_t AsciiHexLE2Int(const char *ascii, const uint8_t anz) {
+    char help[9] = {};
+
+    // sort from little endian format to normal format
+    switch (anz) {
+        case 1:
+            help[0] = ascii[0];
+            break;
+        case 2:
+        case 4:
+        case 8:
+            for (uint8_t i = 0; i < anz; i += 2) {
+                help[i] = ascii[anz-i-2];
+                help[i+1] = ascii[anz-i-1];
+            }
+        default:
+            break;
+    }
+    return (static_cast<uint32_t>(strtoul(help, nullptr, 16)));
+}
+
+
+/*
+ * disassembleHexData()
+ * analysis the hex message and extract: response, address, flags and value/text
+ * buffer:  pointer to message (ascii hex little endian format)
+ * data:    disassembeled message
+ * return:  true = successful disassembeld, false = hex sum fault or message
+ *          do not aligin with VE.Diekt syntax
+ */
+template<typename T>
+bool VeDirectFrameHandler<T>::disassembleHexData(VeDirectHexData &data) {
+    bool state = false;
+    char * buffer = _hexBuffer;
+    auto len = strlen(buffer);
+
+    // reset hex data first
+    data = {};
+
+    if ((len > 3) && (calcHexFrameCheckSum(buffer, len) == 0x00)) {
+        data.rsp = static_cast<VeDirectHexResponse>(AsciiHexLE2Int(buffer+1, 1));
+
+        using Response = VeDirectHexResponse;
+        switch (data.rsp) {
+            case Response::DONE:
+            case Response::ERROR:
+            case Response::PING:
+            case Response::UNKNOWN:
+                strncpy(data.text, buffer+2, len-4);
+                state = true;
+                break;
+            case Response::GET:
+            case Response::SET:
+            case Response::ASYNC:
+                data.addr = static_cast<VeDirectHexRegister>(AsciiHexLE2Int(buffer+2, 4));
+
+                // future option: Up to now we do not use historical data
+                if ((data.addr >= VeDirectHexRegister::HistoryTotal) && (data.addr <= VeDirectHexRegister::HistoryMPPTD30)) {
+                    state = true;
+                    break;
+                }
+
+                // future option: to analyse the flags here?
+                data.flags = AsciiHexLE2Int(buffer+6, 2);
+
+               if (len == 12) {    // 8bit value
+                    data.value = AsciiHexLE2Int(buffer+8, 2);
+                    state = true;
+                }
+
+                if (len == 14) {    // 16bit value
+                    data.value = AsciiHexLE2Int(buffer+8, 4);
+                    state = true;
+                }
+
+                if (len == 18) {    // 32bit value
+                    data.value = AsciiHexLE2Int(buffer+8, 8);
+                    state = true;
+                }
+                break;
+            default:
+                break;  // something went wrong
+        }
+    }
+
+    if (!state)
+        _msgOut->printf("%s failed to disassemble the hex message: %s\r\n", _logId, buffer);
+
+    return (state);
+}
+
+
+/*
+ * uint2toHexLEString()
+ * help function to convert up to 32 bits into little endian hex String
+ * ascii:   pointer to Ascii Hex Little Endian data
+ * anz:     1,2,4 or 8 nibble
+ */
+static String Int2HexLEString(uint32_t value, uint8_t anz) {
+    char hexchar[] = "0123456789ABCDEF";
+    char help[9] = {};
+
+    switch (anz) {
+        case 1:
+            help[0] = hexchar[(value & 0x0000000F)];
+            break;
+        case 2:
+        case 4:
+        case 8:
+            for (uint8_t i = 0; i < anz; i += 2) {
+                help[i] = hexchar[(value>>((1+1*i)*4)) & 0x0000000F];
+                help[i+1] = hexchar[(value>>((1*i)*4)) & 0x0000000F];
+            }
+        default:
+            ;
+    }
+    return String(help);
+}
+
+
+/*
+ * sendHexCommand()
+ * send the hex commend after assembling the command string
+ * cmd:     command
+ * addr:    register address, default 0
+ * value:   value to write into a register, default 0
+ * valsize: size of the value, 8, 16 or 32 bit, default 0
+ * return:  true = message assembeld and send, false = it was not possible to put the message together
+ * SAMPLE:  ping command: sendHexCommand(PING),
+ *          read total DC input power sendHexCommand(GET, 0xEDEC)
+ *          set Charge current limit 10A sendHexCommand(SET, 0x2015, 64, 16)
+ *
+ * WARNING: some values are stored in non-volatile memory. Continuous writing, for example from a control loop, will
+ *          lead to early failure.
+ *          On MPPT for example 0xEDE0 - 0xEDFF. Check the Vivtron doc "BlueSolar-HEX-protocol.pdf"
+ */
+template<typename T>
+bool VeDirectFrameHandler<T>::sendHexCommand(VeDirectHexCommand cmd, VeDirectHexRegister addr, uint32_t value, uint8_t valsize) {
+    bool ret = false;
+    uint8_t flags = 0x00;  // always 0x00
+
+    String txData = ":" + Int2HexLEString(static_cast<uint32_t>(cmd), 1);    // add the command nibble
+
+    using Command = VeDirectHexCommand;
+    switch (cmd) {
+    case Command::PING:
+    case Command::APP_VERSION:
+    case Command::PRODUCT_ID:
+        ret = true;
+        break;
+    case Command::GET:
+    case Command::ASYNC:
+        txData += Int2HexLEString(static_cast<uint16_t>(addr), 4);
+        txData += Int2HexLEString(flags, 2);         // add the flags (2 nibble)
+        ret = true;
+        break;
+    case Command::SET:
+        txData += Int2HexLEString(static_cast<uint16_t>(addr), 4);
+        txData += Int2HexLEString(flags, 2);         // add the flags (2 nibble)
+        if ((valsize == 8) || (valsize == 16) || (valsize == 32)) {
+            txData += Int2HexLEString(value, valsize/4);     // add value (2-8 nibble)
+            ret = true;
+        }
+        break;
+    default:
+        ret = false;
+        break;
+    }
+
+    if (ret) {
+        // add the checksum (2 nibble)
+        txData += Int2HexLEString(calcHexFrameCheckSum(txData.c_str(), txData.length()), 2);
+        String send = txData + "\n";                // hex command end byte
+        _vedirectSerial->write(send.c_str(), send.length());
+
+        if (_verboseLogging) {
+            auto blen = _vedirectSerial->availableForWrite();
+            _msgOut->printf("%s Sending Hex Command: %s, Free FIFO-Buffer: %u\r\n",
+                _logId, txData.c_str(), blen);
+        }
+    }
+
+    if (!ret)
+        _msgOut->printf("%s send hex command fault: %s\r\n", _logId, txData.c_str());
+
+    return (ret);
+}

lib/VeDirectFrameHandler/VeDirectMpptController.cpp

@@ -1,65 +1,82 @@
+/* VeDirectMpptController.cpp
+ *
+ *
+ * 2020.08.20 - 0.0 - ???
+ * 2024.03.18 - 0.1 - add of: - temperature from "Smart Battery Sense" connected over VE.Smart network
+ * 					  		  - temperature from internal MPPT sensor
+ * 					  		  - "total DC input power" from MPPT's connected over VE.Smart network
+ */
+
 #include <Arduino.h>
 #include "VeDirectMpptController.h"
 
+//#define PROCESS_NETWORK_STATE
+
 void VeDirectMpptController::init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort)
 {
-	VeDirectFrameHandler::init(rx, tx, msgOut, verboseLogging, hwSerialPort);
+	VeDirectFrameHandler::init("MPPT", rx, tx, msgOut, verboseLogging, hwSerialPort);
-	_spData = std::make_shared<veMpptStruct>();
-	if (_verboseLogging) { _msgOut->println("Finished init MPPTController"); }
 }
 
-bool VeDirectMpptController::isDataValid() const {
+bool VeDirectMpptController::processTextDataDerived(std::string const& name, std::string const& value)
-	return VeDirectFrameHandler::isDataValid(*_spData);
-}
-
-void VeDirectMpptController::textRxEvent(char* name, char* value)
 {
-	if (VeDirectFrameHandler::textRxEvent("MPPT", name, value, _tmpFrame)) {
+	if (name == "IL") {
-		return;
+		_tmpFrame.loadCurrent_IL_mA = atol(value.c_str());
+		return true;
+	}
+	if (name == "LOAD") {
+		_tmpFrame.loadOutputState_LOAD = (value == "ON");
+		return true;
+	}
+	if (name == "CS") {
+		_tmpFrame.currentState_CS = atoi(value.c_str());
+		return true;
+	}
+	if (name == "ERR") {
+		_tmpFrame.errorCode_ERR = atoi(value.c_str());
+		return true;
+	}
+	if (name == "OR") {
+		_tmpFrame.offReason_OR = strtol(value.c_str(), nullptr, 0);
+		return true;
+	}
+	if (name == "MPPT") {
+		_tmpFrame.stateOfTracker_MPPT = atoi(value.c_str());
+		return true;
+	}
+	if (name == "HSDS") {
+		_tmpFrame.daySequenceNr_HSDS = atoi(value.c_str());
+		return true;
+	}
+	if (name == "VPV") {
+		_tmpFrame.panelVoltage_VPV_mV = atol(value.c_str());
+		return true;
+	}
+	if (name == "PPV") {
+		_tmpFrame.panelPower_PPV_W = atoi(value.c_str());
+		return true;
+	}
+	if (name == "H19") {
+		_tmpFrame.yieldTotal_H19_Wh = atol(value.c_str()) * 10;
+		return true;
+	}
+	if (name == "H20") {
+		_tmpFrame.yieldToday_H20_Wh = atol(value.c_str()) * 10;
+		return true;
+	}
+	if (name == "H21") {
+		_tmpFrame.maxPowerToday_H21_W = atoi(value.c_str());
+		return true;
+	}
+	if (name == "H22") {
+		_tmpFrame.yieldYesterday_H22_Wh = atol(value.c_str()) * 10;
+		return true;
+	}
+	if (name == "H23") {
+		_tmpFrame.maxPowerYesterday_H23_W = atoi(value.c_str());
+		return true;
 	}
 
-	if (strcmp(name, "LOAD") == 0) {
+	return false;
-		if (strcmp(value, "ON") == 0)
-			_tmpFrame.LOAD = true;
-		else
-			_tmpFrame.LOAD = false;
-	}
-	else if (strcmp(name, "CS") == 0) {
-		_tmpFrame.CS = atoi(value);
-	}
-	else if (strcmp(name, "ERR") == 0) {
-		_tmpFrame.ERR = atoi(value);
-	}
-	else if (strcmp(name, "OR") == 0) {
-		_tmpFrame.OR = strtol(value, nullptr, 0);
-	}
-	else if (strcmp(name, "MPPT") == 0) {
-		_tmpFrame.MPPT = atoi(value);
-	}
-	else if (strcmp(name, "HSDS") == 0) {
-		_tmpFrame.HSDS = atoi(value);
-	}
-	else if (strcmp(name, "VPV") == 0) {
-		_tmpFrame.VPV = round(atof(value) / 10.0) / 100.0;
-	}
-	else if (strcmp(name, "PPV") == 0) {
-		_tmpFrame.PPV = atoi(value);
-	}
-	else if (strcmp(name, "H19") == 0) {
-		_tmpFrame.H19 = atof(value) / 100.0;
-	}
-	else if (strcmp(name, "H20") == 0) {
-		_tmpFrame.H20 = atof(value) / 100.0;
-	}
-	else if (strcmp(name, "H21") == 0) {
-		_tmpFrame.H21 = atoi(value);
-	}
-	else if (strcmp(name, "H22") == 0) {
-		_tmpFrame.H22 = atof(value) / 100.0;
-	}
-	else if (strcmp(name, "H23") == 0) {
-		_tmpFrame.H23 = atoi(value);
-	}
 }
 
 /*
@@ -67,110 +84,174 @@ void VeDirectMpptController::textRxEvent(char* name, char* value)
  *  This function is called at the end of the received frame.
  */
 void VeDirectMpptController::frameValidEvent() {
-	_tmpFrame.P = _tmpFrame.V * _tmpFrame.I;
+	// power into the battery, (+) means charging, (-) means discharging
+	_tmpFrame.batteryOutputPower_W = static_cast<int16_t>((_tmpFrame.batteryVoltage_V_mV / 1000.0f) * (_tmpFrame.batteryCurrent_I_mA / 1000.0f));
 
-	_tmpFrame.IPV = 0;
+	// calculation of the panel current
-	if (_tmpFrame.VPV > 0) {
+	if ((_tmpFrame.panelVoltage_VPV_mV > 0) && (_tmpFrame.panelPower_PPV_W >= 1)) {
-		_tmpFrame.IPV = _tmpFrame.PPV / _tmpFrame.VPV;
+		_tmpFrame.panelCurrent_mA = static_cast<uint32_t>(_tmpFrame.panelPower_PPV_W * 1000000.0f / _tmpFrame.panelVoltage_VPV_mV);
+	} else {
+		_tmpFrame.panelCurrent_mA = 0;
 	}
 
-	_tmpFrame.E = 0;
+	// calculation of the MPPT efficiency
-	if ( _tmpFrame.PPV > 0) {
+	float totalPower_W = (_tmpFrame.loadCurrent_IL_mA / 1000.0f + _tmpFrame.batteryCurrent_I_mA / 1000.0f) * _tmpFrame.batteryVoltage_V_mV /1000.0f;
-		_efficiency.addNumber(static_cast<double>(_tmpFrame.P * 100) / _tmpFrame.PPV);
+	if (_tmpFrame.panelPower_PPV_W > 0) {
-		_tmpFrame.E = _efficiency.getAverage();
+		_efficiency.addNumber(totalPower_W * 100.0f / _tmpFrame.panelPower_PPV_W);
+		_tmpFrame.mpptEfficiency_Percent = _efficiency.getAverage();
+	} else {
+		_tmpFrame.mpptEfficiency_Percent = 0.0f;
 	}
 
-	_spData = std::make_shared<veMpptStruct>(_tmpFrame);
+	if (!_canSend) { return; }
-	_tmpFrame = {};
+
-	_lastUpdate = millis();
+	// Copy from the "VE.Direct Protocol" documentation
+	// For firmware version v1.52 and below, when no VE.Direct queries are sent to the device, the
+	// charger periodically sends human readable (TEXT) data to the serial port. For firmware
+	// versions v1.53 and above, the charger always periodically sends TEXT data to the serial port.
+	// --> We just use hex commandes for firmware >= 1.53 to keep text messages alive
+	if (atoi(_tmpFrame.firmwareNr_FW) < 153) { return; }
+
+	using Command = VeDirectHexCommand;
+	using Register = VeDirectHexRegister;
+
+	sendHexCommand(Command::GET, Register::ChargeControllerTemperature);
+	sendHexCommand(Command::GET, Register::SmartBatterySenseTemperature);
+	sendHexCommand(Command::GET, Register::NetworkTotalDcInputPower);
+
+#ifdef PROCESS_NETWORK_STATE
+	sendHexCommand(Command::GET, Register::NetworkInfo);
+	sendHexCommand(Command::GET, Register::NetworkMode);
+	sendHexCommand(Command::GET, Register::NetworkStatus);
+#endif // PROCESS_NETWORK_STATE
 }
 
-/*
+
- * getCsAsString
+void VeDirectMpptController::loop()
- * This function returns the state of operations (CS) as readable text.
- */
-frozen::string const& VeDirectMpptController::veMpptStruct::getCsAsString() const
 {
-	static constexpr frozen::map<uint8_t, frozen::string, 9> values = {
+	VeDirectFrameHandler::loop();
-		{ 0,   "OFF" },
+
-		{ 2,   "Fault" },
+	auto resetTimestamp = [this](auto& pair) {
-		{ 3,   "Bulk" },
+		if (pair.first > 0 && (millis() - pair.first) > (10 * 1000)) {
-		{ 4,   "Absorbtion" },
+			pair.first = 0;
-		{ 5,   "Float" },
+		}
-		{ 7,   "Equalize (manual)" },
-		{ 245, "Starting-up" },
-		{ 247, "Auto equalize / Recondition" },
-		{ 252, "External Control" }
 	};
 
-	return getAsString(values, CS);
+	resetTimestamp(_tmpFrame.MpptTemperatureMilliCelsius);
+	resetTimestamp(_tmpFrame.SmartBatterySenseTemperatureMilliCelsius);
+	resetTimestamp(_tmpFrame.NetworkTotalDcInputPowerMilliWatts);
+
+#ifdef PROCESS_NETWORK_STATE
+	resetTimestamp(_tmpFrame.NetworkInfo);
+	resetTimestamp(_tmpFrame.NetworkMode);
+	resetTimestamp(_tmpFrame.NetworkStatus);
+#endif // PROCESS_NETWORK_STATE
 }
 
+
 /*
- * getMpptAsString
+ * hexDataHandler()
- * This function returns the state of MPPT (MPPT) as readable text.
+ * analyse the content of VE.Direct hex messages
+ * Handels the received hex data from the MPPT
  */
-frozen::string const& VeDirectMpptController::veMpptStruct::getMpptAsString() const
+bool VeDirectMpptController::hexDataHandler(VeDirectHexData const &data) {
-{
+	if (data.rsp != VeDirectHexResponse::GET &&
-	static constexpr frozen::map<uint8_t, frozen::string, 3> values = {
+			data.rsp != VeDirectHexResponse::ASYNC) { return false; }
-		{ 0, "OFF" },
-		{ 1, "Voltage or current limited" },
-		{ 2, "MPP Tracker active" }
-	};
 
-	return getAsString(values, MPPT);
+	auto regLog = static_cast<uint16_t>(data.addr);
-}
+
-
+	switch (data.addr) {
-/*
+		case VeDirectHexRegister::ChargeControllerTemperature:
- * getErrAsString
+			_tmpFrame.MpptTemperatureMilliCelsius =
- * This function returns error state (ERR) as readable text.
+				{ millis(), static_cast<int32_t>(data.value) * 10 };
- */
+
-frozen::string const& VeDirectMpptController::veMpptStruct::getErrAsString() const
+			if (_verboseLogging) {
-{
+				_msgOut->printf("%s Hex Data: MPPT Temperature (0x%04X): %.2f°C\r\n",
-	static constexpr frozen::map<uint8_t, frozen::string, 20> values = {
+						_logId, regLog,
-		{ 0,   "No error" },
+						_tmpFrame.MpptTemperatureMilliCelsius.second / 1000.0);
-		{ 2,   "Battery voltage too high" },
+			}
-		{ 17,  "Charger temperature too high" },
+			return true;
-		{ 18,  "Charger over current" },
+			break;
-		{ 19,  "Charger current reversed" },
+
-		{ 20,  "Bulk time limit exceeded" },
+		case VeDirectHexRegister::SmartBatterySenseTemperature:
-		{ 21,  "Current sensor issue(sensor bias/sensor broken)" },
+			if (data.value == 0xFFFF) {
-		{ 26,  "Terminals overheated" },
+				if (_verboseLogging) {
-		{ 28,  "Converter issue (dual converter models only)" },
+					_msgOut->printf("%s Hex Data: Smart Battery Sense Temperature is not available\r\n", _logId);
-		{ 33,  "Input voltage too high (solar panel)" },
+				}
-		{ 34,  "Input current too high (solar panel)" },
+				return true; // we know what to do with it, and we decided to ignore the value
-		{ 38,  "Input shutdown (due to excessive battery voltage)" },
+			}
-		{ 39,  "Input shutdown (due to current flow during off mode)" },
+
-		{ 40,  "Input" },
+			_tmpFrame.SmartBatterySenseTemperatureMilliCelsius =
-		{ 65,  "Lost communication with one of devices" },
+				{ millis(), static_cast<int32_t>(data.value) * 10 - 272150 };
-		{ 67,  "Synchronisedcharging device configuration issue" },
+
-		{ 68,  "BMS connection lost" },
+			if (_verboseLogging) {
-		{ 116, "Factory calibration data lost" },
+				_msgOut->printf("%s Hex Data: Smart Battery Sense Temperature (0x%04X): %.2f°C\r\n",
-		{ 117, "Invalid/incompatible firmware" },
+						_logId, regLog,
-		{ 118, "User settings invalid" }
+						_tmpFrame.SmartBatterySenseTemperatureMilliCelsius.second / 1000.0);
-	};
+			}
-
+			return true;
-	return getAsString(values, ERR);
+			break;
-}
+
-
+		case VeDirectHexRegister::NetworkTotalDcInputPower:
-/*
+			if (data.value == 0xFFFFFFFF) {
- * getOrAsString
+				if (_verboseLogging) {
- * This function returns the off reason (OR) as readable text.
+					_msgOut->printf("%s Hex Data: Network total DC power value "
- */
+							"indicates non-networked controller\r\n", _logId);
-frozen::string const& VeDirectMpptController::veMpptStruct::getOrAsString() const
+				}
-{
+				_tmpFrame.NetworkTotalDcInputPowerMilliWatts = { 0, 0 };
-	static constexpr frozen::map<uint32_t, frozen::string, 10> values = {
+				return true; // we know what to do with it, and we decided to ignore the value
-		{ 0x00000000, "Not off" },
+			}
-		{ 0x00000001, "No input power" },
+
-		{ 0x00000002, "Switched off (power switch)" },
+			_tmpFrame.NetworkTotalDcInputPowerMilliWatts =
-		{ 0x00000004, "Switched off (device moderegister)" },
+				{ millis(), data.value * 10 };
-		{ 0x00000008, "Remote input" },
+
-		{ 0x00000010, "Protection active" },
+			if (_verboseLogging) {
-		{ 0x00000020, "Paygo" },
+				_msgOut->printf("%s Hex Data: Network Total DC Power (0x%04X): %.2fW\r\n",
-		{ 0x00000040, "BMS" },
+						_logId, regLog,
-		{ 0x00000080, "Engine shutdown detection" },
+						_tmpFrame.NetworkTotalDcInputPowerMilliWatts.second / 1000.0);
-		{ 0x00000100, "Analysing input voltage" }
+			}
-	};
+			return true;
-
+			break;
-	return getAsString(values, OR);
+
+#ifdef PROCESS_NETWORK_STATE
+		case VeDirectHexRegister::NetworkInfo:
+			_tmpFrame.NetworkInfo =
+				{ millis(), static_cast<uint8_t>(data.value) };
+
+			if (_verboseLogging) {
+				_msgOut->printf("%s Hex Data: Network Info (0x%04X): 0x%X\r\n",
+						_logId, regLog, data.value);
+			}
+			return true;
+			break;
+
+		case VeDirectHexRegister::NetworkMode:
+			_tmpFrame.NetworkMode =
+				{ millis(), static_cast<uint8_t>(data.value) };
+
+			if (_verboseLogging) {
+				_msgOut->printf("%s Hex Data: Network Mode (0x%04X): 0x%X\r\n",
+						_logId, regLog, data.value);
+			}
+			return true;
+			break;
+
+		case VeDirectHexRegister::NetworkStatus:
+			_tmpFrame.NetworkStatus =
+				{ millis(), static_cast<uint8_t>(data.value) };
+
+			if (_verboseLogging) {
+				_msgOut->printf("%s Hex Data: Network Status (0x%04X): 0x%X\r\n",
+						_logId, regLog, data.value);
+			}
+			return true;
+			break;
+#endif // PROCESS_NETWORK_STATE
+
+		default:
+			return false;
+			break;
+	}
+
+	return false;
 }

lib/VeDirectFrameHandler/VeDirectMpptController.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Arduino.h>
+#include "VeDirectData.h"
 #include "VeDirectFrameHandler.h"
 
 template<typename T, size_t WINDOW_SIZE>
@@ -23,9 +24,9 @@ public:
         _index = (_index + 1) % WINDOW_SIZE;
     }
 
-    double getAverage() const {
+    float getAverage() const {
         if (_count == 0) { return 0.0; }
-        return static_cast<double>(_sum) / _count;
+        return static_cast<float>(_sum) / _count;
     }
 
 private:
@@ -35,43 +36,19 @@ private:
     size_t _count;
 };
 
-class VeDirectMpptController : public VeDirectFrameHandler {
+class VeDirectMpptController : public VeDirectFrameHandler<veMpptStruct> {
 public:
     VeDirectMpptController() = default;
 
     void init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging, uint16_t hwSerialPort);
-    bool isDataValid() const;                        // return true if data valid and not outdated
 
-    struct veMpptStruct : veStruct {
+    using data_t = veMpptStruct;
-        uint8_t  MPPT;                  // state of MPP tracker
-        int32_t PPV;                    // panel power in W
-        int32_t P;                      // battery output power in W (calculated)
-        double VPV;                     // panel voltage in V
-        double IPV;                     // panel current in A (calculated)
-        bool LOAD;                      // virtual load output state (on if battery voltage reaches upper limit, off if battery reaches lower limit)
-        uint8_t  CS;                    // current state of operation e.g. OFF or Bulk
-        uint8_t ERR;                    // error code
-        uint32_t OR;                    // off reason
-        uint32_t HSDS;                  // day sequence number 1...365
-        double H19;                     // yield total kWh
-        double H20;                     // yield today kWh
-        int32_t H21;                    // maximum power today W
-        double H22;                     // yield yesterday kWh
-        int32_t H23;                    // maximum power yesterday W
 
-        frozen::string const& getMpptAsString() const; // state of mppt as string
+    void loop() final;
-        frozen::string const& getCsAsString() const;   // current state as string
-        frozen::string const& getErrAsString() const;  // error state as string
-        frozen::string const& getOrAsString() const;   // off reason as string
-    };
-
-    using spData_t = std::shared_ptr<veMpptStruct const>;
-    spData_t getData() const { return _spData; }
 
 private:
-    void textRxEvent(char* name, char* value) final;
+    bool hexDataHandler(VeDirectHexData const &data) final;
+    bool processTextDataDerived(std::string const& name, std::string const& value) final;
     void frameValidEvent() final;
-    spData_t _spData = nullptr;
+    MovingAverage<float, 5> _efficiency;
-    veMpptStruct _tmpFrame{};                        // private struct for received name and value pairs
-    MovingAverage<double, 5> _efficiency;
 };

lib/VeDirectFrameHandler/VeDirectShuntController.cpp

@@ -3,110 +3,123 @@
 
 VeDirectShuntController VeDirectShunt;
 
-VeDirectShuntController::VeDirectShuntController()
-{
-}
-
 void VeDirectShuntController::init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging)
 {
-	VeDirectFrameHandler::init(rx, tx, msgOut, verboseLogging, 2);
+	VeDirectFrameHandler::init("SmartShunt", rx, tx, msgOut, verboseLogging,
-	if (_verboseLogging) {
+			((ARDUINO_USB_CDC_ON_BOOT != 1)?2:0));
-		_msgOut->println("Finished init ShuntController");
-	}
 }
 
-void VeDirectShuntController::textRxEvent(char* name, char* value)
+bool VeDirectShuntController::processTextDataDerived(std::string const& name, std::string const& value)
 {
-	if (VeDirectFrameHandler::textRxEvent("SmartShunt", name, value, _tmpFrame)) {
+	if (name == "T") {
-		return;
+		_tmpFrame.T = atoi(value.c_str());
-	}
-
-	if (strcmp(name, "T") == 0) {
-		_tmpFrame.T = atoi(value);
 		_tmpFrame.tempPresent = true;
+		return true;
 	}
-	else if (strcmp(name, "P") == 0) {
+	if (name == "P") {
-		_tmpFrame.P = atoi(value);
+		_tmpFrame.P = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "CE") == 0) {
+	if (name == "CE") {
-		_tmpFrame.CE = atoi(value);
+		_tmpFrame.CE = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "SOC") == 0) {
+	if (name == "SOC") {
-		_tmpFrame.SOC = atoi(value);
+		_tmpFrame.SOC = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "TTG") == 0) {
+	if (name == "TTG") {
-		_tmpFrame.TTG = atoi(value);
+		_tmpFrame.TTG = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "ALARM") == 0) {
+	if (name == "ALARM") {
-		_tmpFrame.ALARM = (strcmp(value, "ON") == 0);
+		_tmpFrame.ALARM = (value == "ON");
+		return true;
 	}
-	else if (strcmp(name, "H1") == 0) {
+	if (name == "AR") {
-		_tmpFrame.H1 = atoi(value);
+		_tmpFrame.alarmReason_AR = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H2") == 0) {
+	if (name == "H1") {
-		_tmpFrame.H2 = atoi(value);
+		_tmpFrame.H1 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H3") == 0) {
+	if (name == "H2") {
-		_tmpFrame.H3 = atoi(value);
+		_tmpFrame.H2 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H4") == 0) {
+	if (name == "H3") {
-		_tmpFrame.H4 = atoi(value);
+		_tmpFrame.H3 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H5") == 0) {
+	if (name == "H4") {
-		_tmpFrame.H5 = atoi(value);
+		_tmpFrame.H4 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H6") == 0) {
+	if (name == "H5") {
-		_tmpFrame.H6 = atoi(value);
+		_tmpFrame.H5 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H7") == 0) {
+	if (name == "H6") {
-		_tmpFrame.H7 = atoi(value);
+		_tmpFrame.H6 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H8") == 0) {
+	if (name == "H7") {
-		_tmpFrame.H8 = atoi(value);
+		_tmpFrame.H7 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H9") == 0) {
+	if (name == "H8") {
-		_tmpFrame.H9 = atoi(value);
+		_tmpFrame.H8 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H10") == 0) {
+	if (name == "H9") {
-		_tmpFrame.H10 = atoi(value);
+		_tmpFrame.H9 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H11") == 0) {
+	if (name == "H10") {
-		_tmpFrame.H11 = atoi(value);
+		_tmpFrame.H10 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H12") == 0) {
+	if (name == "H11") {
-		_tmpFrame.H12 = atoi(value);
+		_tmpFrame.H11 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H13") == 0) {
+	if (name == "H12") {
-		_tmpFrame.H13 = atoi(value);
+		_tmpFrame.H12 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H14") == 0) {
+	if (name == "H13") {
-		_tmpFrame.H14 = atoi(value);
+		_tmpFrame.H13 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H15") == 0) {
+	if (name == "H14") {
-		_tmpFrame.H15 = atoi(value);
+		_tmpFrame.H14 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H16") == 0) {
+	if (name == "H15") {
-		_tmpFrame.H16 = atoi(value);
+		_tmpFrame.H15 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H17") == 0) {
+	if (name == "H16") {
-		_tmpFrame.H17 = atoi(value);
+		_tmpFrame.H16 = atoi(value.c_str());
+		return true;
 	}
-	else if (strcmp(name, "H18") == 0) {
+	if (name == "H17") {
-		_tmpFrame.H18 = atoi(value);
+		_tmpFrame.H17 = atoi(value.c_str());
+		return true;
 	}
-}
+	if (name == "H18") {
-
+		_tmpFrame.H18 = atoi(value.c_str());
-/*
+		return true;
- *  frameValidEvent
+	}
- *  This function is called at the end of the received frame.
+	if (name == "BMV") {
- */
+		// This field contains a textual description of the BMV model,
-void VeDirectShuntController::frameValidEvent() {
+		// for example 602S or 702. It is deprecated, refer to the field PID instead.
-	// other than in the MPPT controller, the SmartShunt seems to split all data
+		return true;
-	// into two seperate messagesas. Thus we update veFrame only every second message
+	}
-	// after a value for PID has been received
+	if (name == "MON") {
-	if (_tmpFrame.PID == 0) { return; }
+		_tmpFrame.dcMonitorMode_MON = static_cast<int8_t>(atoi(value.c_str()));
-
+		return true;
-	veFrame = _tmpFrame;
+	}
-	_tmpFrame = {};
+	return false;
-	_lastUpdate = millis();
 }

lib/VeDirectFrameHandler/VeDirectShuntController.h

@@ -1,49 +1,19 @@
 #pragma once
 
 #include <Arduino.h>
+#include "VeDirectData.h"
 #include "VeDirectFrameHandler.h"
 
-class VeDirectShuntController : public VeDirectFrameHandler {
+class VeDirectShuntController : public VeDirectFrameHandler<veShuntStruct> {
 public:
-    VeDirectShuntController();
+    VeDirectShuntController() = default;
 
     void init(int8_t rx, int8_t tx, Print* msgOut, bool verboseLogging);
 
-    struct veShuntStruct : veStruct {
+    using data_t = veShuntStruct;
-        int32_t T;                      // Battery temperature
-        bool tempPresent = false;       // Battery temperature sensor is attached to the shunt
-        int32_t P;                      // Instantaneous power
-        int32_t CE;                     // Consumed Amp Hours
-        int32_t SOC;                    // State-of-charge
-        uint32_t TTG;                   // Time-to-go
-        bool ALARM;                     // Alarm condition active
-        uint32_t AR;                    // Alarm Reason
-        int32_t H1;                     // Depth of the deepest discharge
-        int32_t H2;                     // Depth of the last discharge
-        int32_t H3;                     // Depth of the average discharge
-        int32_t H4;                     // Number of charge cycles
-        int32_t H5;                     // Number of full discharges
-        int32_t H6;                     // Cumulative Amp Hours drawn
-        int32_t H7;                     // Minimum main (battery) voltage
-        int32_t H8;                     // Maximum main (battery) voltage
-        int32_t H9;                     // Number of seconds since last full charge
-        int32_t H10;                    // Number of automatic synchronizations
-        int32_t H11;                    // Number of low main voltage alarms
-        int32_t H12;                    // Number of high main voltage alarms
-        int32_t H13;                    // Number of low auxiliary voltage alarms
-        int32_t H14;                    // Number of high auxiliary voltage alarms
-        int32_t H15;                    // Minimum auxiliary (battery) voltage
-        int32_t H16;                    // Maximum auxiliary (battery) voltage
-        int32_t H17;                    // Amount of discharged energy
-        int32_t H18;                    // Amount of charged energy
-    };
-
-    veShuntStruct veFrame{};
 
 private:
-    void textRxEvent(char * name, char * value) final;
+    bool processTextDataDerived(std::string const& name, std::string const& value) final;
-    void frameValidEvent() final;
-    veShuntStruct _tmpFrame{};                        // private struct for received name and value pairs
 };
 
 extern VeDirectShuntController VeDirectShunt;

patches/async_tcp/event_queue_size.patch

@@ -0,0 +1,26 @@
+diff --color -ruN a/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.cpp b/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.cpp
+--- a/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.cpp
++++ b/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.cpp
+@@ -97,7 +97,7 @@
+
+ static inline bool _init_async_event_queue(){
+     if(!_async_queue){
+-        _async_queue = xQueueCreate(32, sizeof(lwip_event_packet_t *));
++        _async_queue = xQueueCreate(CONFIG_ASYNC_TCP_EVENT_QUEUE_SIZE, sizeof(lwip_event_packet_t *));
+         if(!_async_queue){
+             return false;
+         }
+diff --color -ruN a/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.h b/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.h
+--- a/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.h
++++ b/.pio/libdeps/$$$env$$$/AsyncTCP-esphome/src/AsyncTCP.h
+@@ -53,6 +53,10 @@
+ #define CONFIG_ASYNC_TCP_STACK_SIZE 8192 * 2
+ #endif
+
++#ifndef CONFIG_ASYNC_TCP_EVENT_QUEUE_SIZE
++#define CONFIG_ASYNC_TCP_EVENT_QUEUE_SIZE 32
++#endif
++
+ class AsyncClient;
+
+ #define ASYNC_MAX_ACK_TIME 5000

patches/esp32c3/EspAsyncWebserver.patch

@@ -1,13 +0,0 @@
-diff --git a/.pio/libdeps/$$$env$$$/ESP Async WebServer/src/AsyncWebSocket.cpp b/.pio/libdeps/$$$env$$$/ESP Async WebServer/src/AsyncWebSocket.cpp
-index 12be5f8..8505f73 100644
---- a/.pio/libdeps/$$$env$$$/ESP Async WebServer/src/AsyncWebSocket.cpp
-+++ b/.pio/libdeps/$$$env$$$/ESP Async WebServer/src/AsyncWebSocket.cpp
-@@ -737,7 +737,7 @@ void AsyncWebSocketClient::binary(const __FlashStringHelper *data, size_t len)
- IPAddress AsyncWebSocketClient::remoteIP() const
- {
-     if (!_client)
--        return IPAddress(0U);
-+        return IPAddress((uint32_t)0);
- 
-     return _client->remoteIP();
- }

platformio.ini

@@ -19,12 +19,13 @@ extra_configs =
 custom_ci_action = generic,generic_esp32,generic_esp32s3,generic_esp32s3_usb
 
 framework = arduino
-platform = espressif32@6.5.0
+platform = espressif32@6.6.0
 
 build_flags =
     -DPIOENV=\"$PIOENV\"
     -D_TASK_STD_FUNCTION=1
     -D_TASK_THREAD_SAFE=1
+    -DCONFIG_ASYNC_TCP_EVENT_QUEUE_SIZE=128
     -Wall -Wextra -Wunused -Wmisleading-indentation -Wduplicated-cond -Wlogical-op -Wnull-dereference
 ;   Have to remove -Werror because of
 ;   https://github.com/espressif/arduino-esp32/issues/9044 and
@@ -36,11 +37,11 @@ build_unflags =
     -std=gnu++11
 
 lib_deps =
-    mathieucarbou/ESP Async WebServer @ 2.8.1
+    mathieucarbou/ESP Async WebServer @ 2.9.3
-    bblanchon/ArduinoJson @ ^6.21.5
+    bblanchon/ArduinoJson @ ^7.0.4
     https://github.com/bertmelis/espMqttClient.git#v1.6.0
     nrf24/RF24 @ ^1.4.8
-    olikraus/U8g2 @ ^2.35.15
+    olikraus/U8g2 @ ^2.35.17
     buelowp/sunset @ ^1.1.7
     https://github.com/arkhipenko/TaskScheduler#testing
     https://github.com/coryjfowler/MCP_CAN_lib
@@ -64,7 +65,7 @@ board_build.embed_files =
     webapp_dist/js/app.js.gz
     webapp_dist/site.webmanifest
 
-custom_patches =
+custom_patches = async_tcp
 
 monitor_filters = esp32_exception_decoder, time, log2file, colorize
 monitor_speed = 115200
@@ -92,13 +93,13 @@ build_flags = ${env.build_flags}
 
 [env:generic_esp32c3]
 board = esp32-c3-devkitc-02
-custom_patches = ${env.custom_patches},esp32c3
+custom_patches = ${env.custom_patches}
 build_flags = ${env.build_flags}
 
 
 [env:generic_esp32c3_usb]
 board = esp32-c3-devkitc-02
-custom_patches = ${env.custom_patches},esp32c3
+custom_patches = ${env.custom_patches}
 build_flags = ${env.build_flags}
     -DARDUINO_USB_MODE=1
     -DARDUINO_USB_CDC_ON_BOOT=1

src/BatteryStats.cpp

@@ -61,8 +61,8 @@ bool BatteryStats::updateAvailable(uint32_t since) const
 
 void BatteryStats::getLiveViewData(JsonVariant& root) const
 {
-    root[F("manufacturer")] = _manufacturer;
+    root["manufacturer"] = _manufacturer;
-    root[F("data_age")] = getAgeSeconds();
+    root["data_age"] = getAgeSeconds();
 
     addLiveViewValue(root, "SoC", _soc, "%", _socPrecision);
     addLiveViewValue(root, "voltage", _voltage, "V", 2);
@@ -218,39 +218,39 @@ uint32_t BatteryStats::getMqttFullPublishIntervalMs() const
 
 void BatteryStats::mqttPublish() const
 {
-    MqttSettings.publish(F("battery/manufacturer"), _manufacturer);
+    MqttSettings.publish("battery/manufacturer", _manufacturer);
-    MqttSettings.publish(F("battery/dataAge"), String(getAgeSeconds()));
+    MqttSettings.publish("battery/dataAge", String(getAgeSeconds()));
-    MqttSettings.publish(F("battery/stateOfCharge"), String(_soc));
+    MqttSettings.publish("battery/stateOfCharge", String(_soc));
-    MqttSettings.publish(F("battery/voltage"), String(_voltage));
+    MqttSettings.publish("battery/voltage", String(_voltage));
 }
 
 void PylontechBatteryStats::mqttPublish() const
 {
     BatteryStats::mqttPublish();
 
-    MqttSettings.publish(F("battery/settings/chargeVoltage"), String(_chargeVoltage));
+    MqttSettings.publish("battery/settings/chargeVoltage", String(_chargeVoltage));
-    MqttSettings.publish(F("battery/settings/chargeCurrentLimitation"), String(_chargeCurrentLimitation));
+    MqttSettings.publish("battery/settings/chargeCurrentLimitation", String(_chargeCurrentLimitation));
-    MqttSettings.publish(F("battery/settings/dischargeCurrentLimitation"), String(_dischargeCurrentLimitation));
+    MqttSettings.publish("battery/settings/dischargeCurrentLimitation", String(_dischargeCurrentLimitation));
-    MqttSettings.publish(F("battery/stateOfHealth"), String(_stateOfHealth));
+    MqttSettings.publish("battery/stateOfHealth", String(_stateOfHealth));
-    MqttSettings.publish(F("battery/current"), String(_current));
+    MqttSettings.publish("battery/current", String(_current));
-    MqttSettings.publish(F("battery/temperature"), String(_temperature));
+    MqttSettings.publish("battery/temperature", String(_temperature));
-    MqttSettings.publish(F("battery/alarm/overCurrentDischarge"), String(_alarmOverCurrentDischarge));
+    MqttSettings.publish("battery/alarm/overCurrentDischarge", String(_alarmOverCurrentDischarge));
-    MqttSettings.publish(F("battery/alarm/overCurrentCharge"), String(_alarmOverCurrentCharge));
+    MqttSettings.publish("battery/alarm/overCurrentCharge", String(_alarmOverCurrentCharge));
-    MqttSettings.publish(F("battery/alarm/underTemperature"), String(_alarmUnderTemperature));
+    MqttSettings.publish("battery/alarm/underTemperature", String(_alarmUnderTemperature));
-    MqttSettings.publish(F("battery/alarm/overTemperature"), String(_alarmOverTemperature));
+    MqttSettings.publish("battery/alarm/overTemperature", String(_alarmOverTemperature));
-    MqttSettings.publish(F("battery/alarm/underVoltage"), String(_alarmUnderVoltage));
+    MqttSettings.publish("battery/alarm/underVoltage", String(_alarmUnderVoltage));
-    MqttSettings.publish(F("battery/alarm/overVoltage"), String(_alarmOverVoltage));
+    MqttSettings.publish("battery/alarm/overVoltage", String(_alarmOverVoltage));
-    MqttSettings.publish(F("battery/alarm/bmsInternal"), String(_alarmBmsInternal));
+    MqttSettings.publish("battery/alarm/bmsInternal", String(_alarmBmsInternal));
-    MqttSettings.publish(F("battery/warning/highCurrentDischarge"), String(_warningHighCurrentDischarge));
+    MqttSettings.publish("battery/warning/highCurrentDischarge", String(_warningHighCurrentDischarge));
-    MqttSettings.publish(F("battery/warning/highCurrentCharge"), String(_warningHighCurrentCharge));
+    MqttSettings.publish("battery/warning/highCurrentCharge", String(_warningHighCurrentCharge));
-    MqttSettings.publish(F("battery/warning/lowTemperature"), String(_warningLowTemperature));
+    MqttSettings.publish("battery/warning/lowTemperature", String(_warningLowTemperature));
-    MqttSettings.publish(F("battery/warning/highTemperature"), String(_warningHighTemperature));
+    MqttSettings.publish("battery/warning/highTemperature", String(_warningHighTemperature));
-    MqttSettings.publish(F("battery/warning/lowVoltage"), String(_warningLowVoltage));
+    MqttSettings.publish("battery/warning/lowVoltage", String(_warningLowVoltage));
-    MqttSettings.publish(F("battery/warning/highVoltage"), String(_warningHighVoltage));
+    MqttSettings.publish("battery/warning/highVoltage", String(_warningHighVoltage));
-    MqttSettings.publish(F("battery/warning/bmsInternal"), String(_warningBmsInternal));
+    MqttSettings.publish("battery/warning/bmsInternal", String(_warningBmsInternal));
-    MqttSettings.publish(F("battery/charging/chargeEnabled"), String(_chargeEnabled));
+    MqttSettings.publish("battery/charging/chargeEnabled", String(_chargeEnabled));
-    MqttSettings.publish(F("battery/charging/dischargeEnabled"), String(_dischargeEnabled));
+    MqttSettings.publish("battery/charging/dischargeEnabled", String(_dischargeEnabled));
-    MqttSettings.publish(F("battery/charging/chargeImmediately"), String(_chargeImmediately));
+    MqttSettings.publish("battery/charging/chargeImmediately", String(_chargeImmediately));
 }
 
 void JkBmsBatteryStats::mqttPublish() const
@@ -333,7 +333,12 @@ void JkBmsBatteryStats::updateFrom(JkBms::DataPointContainer const& dp)
     _manufacturer = "JKBMS";
     auto oProductId = dp.get<Label::ProductId>();
     if (oProductId.has_value()) {
-        _manufacturer = oProductId->c_str();
+        // the first twelve chars are expected to be the "User Private Data"
+        // setting (see smartphone app). the remainder is expected be the BMS
+        // name, which can be changed at will using the smartphone app. so
+        // there is not always a "JK" in this string. if there is, we still cut
+        // the string there to avoid possible regressions.
+        _manufacturer = oProductId->substr(12).c_str();
         auto pos = oProductId->rfind("JK");
         if (pos != std::string::npos) {
             _manufacturer = oProductId->substr(pos).c_str();
@@ -373,11 +378,11 @@ void JkBmsBatteryStats::updateFrom(JkBms::DataPointContainer const& dp)
     _lastUpdate = millis();
 }
 
-void VictronSmartShuntStats::updateFrom(VeDirectShuntController::veShuntStruct const& shuntData) {
+void VictronSmartShuntStats::updateFrom(VeDirectShuntController::data_t const& shuntData) {
-    BatteryStats::setVoltage(shuntData.V, millis());
+    BatteryStats::setVoltage(shuntData.batteryVoltage_V_mV / 1000.0, millis());
     BatteryStats::setSoC(static_cast<float>(shuntData.SOC) / 10, 1/*precision*/, millis());
 
-    _current = shuntData.I;
+    _current = static_cast<float>(shuntData.batteryCurrent_I_mA) / 1000;
     _modelName = shuntData.getPidAsString().data();
     _chargeCycles = shuntData.H4;
     _timeToGo = shuntData.TTG / 60;
@@ -390,11 +395,11 @@ void VictronSmartShuntStats::updateFrom(VeDirectShuntController::veShuntStruct c
     _consumedAmpHours = static_cast<float>(shuntData.CE) / 1000;
     _lastFullCharge = shuntData.H9 / 60;
     // shuntData.AR is a bitfield, so we need to check each bit individually
-    _alarmLowVoltage = shuntData.AR & 1;
+    _alarmLowVoltage = shuntData.alarmReason_AR & 1;
-    _alarmHighVoltage = shuntData.AR & 2;
+    _alarmHighVoltage = shuntData.alarmReason_AR & 2;
-    _alarmLowSOC = shuntData.AR & 4;
+    _alarmLowSOC = shuntData.alarmReason_AR & 4;
-    _alarmLowTemperature = shuntData.AR & 32;
+    _alarmLowTemperature = shuntData.alarmReason_AR & 32;
-    _alarmHighTemperature = shuntData.AR & 64;
+    _alarmHighTemperature = shuntData.alarmReason_AR & 64;
 
     _lastUpdate = VeDirectShunt.getLastUpdate();
 }
@@ -424,11 +429,11 @@ void VictronSmartShuntStats::getLiveViewData(JsonVariant& root) const {
 void VictronSmartShuntStats::mqttPublish() const {
     BatteryStats::mqttPublish();
 
-    MqttSettings.publish(F("battery/current"), String(_current));
+    MqttSettings.publish("battery/current", String(_current));
-    MqttSettings.publish(F("battery/chargeCycles"), String(_chargeCycles));
+    MqttSettings.publish("battery/chargeCycles", String(_chargeCycles));
-    MqttSettings.publish(F("battery/chargedEnergy"), String(_chargedEnergy));
+    MqttSettings.publish("battery/chargedEnergy", String(_chargedEnergy));
-    MqttSettings.publish(F("battery/dischargedEnergy"), String(_dischargedEnergy));
+    MqttSettings.publish("battery/dischargedEnergy", String(_dischargedEnergy));
-    MqttSettings.publish(F("battery/instantaneousPower"), String(_instantaneousPower));
+    MqttSettings.publish("battery/instantaneousPower", String(_instantaneousPower));
-    MqttSettings.publish(F("battery/consumedAmpHours"), String(_consumedAmpHours));
+    MqttSettings.publish("battery/consumedAmpHours", String(_consumedAmpHours));
-    MqttSettings.publish(F("battery/lastFullCharge"), String(_lastFullCharge));
+    MqttSettings.publish("battery/lastFullCharge", String(_lastFullCharge));
 }

src/Configuration.cpp

@@ -25,17 +25,13 @@ bool ConfigurationClass::write()
     }
     config.Cfg.SaveCount++;
 
-    DynamicJsonDocument doc(JSON_BUFFER_SIZE);
+    JsonDocument doc;
 
-    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
+    JsonObject cfg = doc["cfg"].to<JsonObject>();
-        return false;
-    }
-
-    JsonObject cfg = doc.createNestedObject("cfg");
     cfg["version"] = config.Cfg.Version;
     cfg["save_count"] = config.Cfg.SaveCount;
 
-    JsonObject wifi = doc.createNestedObject("wifi");
+    JsonObject wifi = doc["wifi"].to<JsonObject>();
     wifi["ssid"] = config.WiFi.Ssid;
     wifi["password"] = config.WiFi.Password;
     wifi["ip"] = IPAddress(config.WiFi.Ip).toString();
@@ -47,10 +43,10 @@ bool ConfigurationClass::write()
     wifi["hostname"] = config.WiFi.Hostname;
     wifi["aptimeout"] = config.WiFi.ApTimeout;
 
-    JsonObject mdns = doc.createNestedObject("mdns");
+    JsonObject mdns = doc["mdns"].to<JsonObject>();
     mdns["enabled"] = config.Mdns.Enabled;
 
-    JsonObject ntp = doc.createNestedObject("ntp");
+    JsonObject ntp = doc["ntp"].to<JsonObject>();
     ntp["server"] = config.Ntp.Server;
     ntp["timezone"] = config.Ntp.Timezone;
     ntp["timezone_descr"] = config.Ntp.TimezoneDescr;
@@ -58,7 +54,7 @@ bool ConfigurationClass::write()
     ntp["longitude"] = config.Ntp.Longitude;
     ntp["sunsettype"] = config.Ntp.SunsetType;
 
-    JsonObject mqtt = doc.createNestedObject("mqtt");
+    JsonObject mqtt = doc["mqtt"].to<JsonObject>();
     mqtt["enabled"] = config.Mqtt.Enabled;
     mqtt["verbose_logging"] = config.Mqtt.VerboseLogging;
     mqtt["hostname"] = config.Mqtt.Hostname;
@@ -70,27 +66,27 @@ bool ConfigurationClass::write()
     mqtt["publish_interval"] = config.Mqtt.PublishInterval;
     mqtt["clean_session"] = config.Mqtt.CleanSession;
 
-    JsonObject mqtt_lwt = mqtt.createNestedObject("lwt");
+    JsonObject mqtt_lwt = mqtt["lwt"].to<JsonObject>();
     mqtt_lwt["topic"] = config.Mqtt.Lwt.Topic;
     mqtt_lwt["value_online"] = config.Mqtt.Lwt.Value_Online;
     mqtt_lwt["value_offline"] = config.Mqtt.Lwt.Value_Offline;
     mqtt_lwt["qos"] = config.Mqtt.Lwt.Qos;
 
-    JsonObject mqtt_tls = mqtt.createNestedObject("tls");
+    JsonObject mqtt_tls = mqtt["tls"].to<JsonObject>();
     mqtt_tls["enabled"] = config.Mqtt.Tls.Enabled;
     mqtt_tls["root_ca_cert"] = config.Mqtt.Tls.RootCaCert;
     mqtt_tls["certlogin"] = config.Mqtt.Tls.CertLogin;
     mqtt_tls["client_cert"] = config.Mqtt.Tls.ClientCert;
     mqtt_tls["client_key"] = config.Mqtt.Tls.ClientKey;
 
-    JsonObject mqtt_hass = mqtt.createNestedObject("hass");
+    JsonObject mqtt_hass = mqtt["hass"].to<JsonObject>();
     mqtt_hass["enabled"] = config.Mqtt.Hass.Enabled;
     mqtt_hass["retain"] = config.Mqtt.Hass.Retain;
     mqtt_hass["topic"] = config.Mqtt.Hass.Topic;
     mqtt_hass["individual_panels"] = config.Mqtt.Hass.IndividualPanels;
     mqtt_hass["expire"] = config.Mqtt.Hass.Expire;
 
-    JsonObject dtu = doc.createNestedObject("dtu");
+    JsonObject dtu = doc["dtu"].to<JsonObject>();
     dtu["serial"] = config.Dtu.Serial;
     dtu["poll_interval"] = config.Dtu.PollInterval;
     dtu["verbose_logging"] = config.Dtu.VerboseLogging;
@@ -99,14 +95,14 @@ bool ConfigurationClass::write()
     dtu["cmt_frequency"] = config.Dtu.Cmt.Frequency;
     dtu["cmt_country_mode"] = config.Dtu.Cmt.CountryMode;
 
-    JsonObject security = doc.createNestedObject("security");
+    JsonObject security = doc["security"].to<JsonObject>();
     security["password"] = config.Security.Password;
     security["allow_readonly"] = config.Security.AllowReadonly;
 
-    JsonObject device = doc.createNestedObject("device");
+    JsonObject device = doc["device"].to<JsonObject>();
     device["pinmapping"] = config.Dev_PinMapping;
 
-    JsonObject display = device.createNestedObject("display");
+    JsonObject display = device["display"].to<JsonObject>();
     display["powersafe"] = config.Display.PowerSafe;
     display["screensaver"] = config.Display.ScreenSaver;
     display["rotation"] = config.Display.Rotation;
@@ -115,15 +111,15 @@ bool ConfigurationClass::write()
     display["diagram_duration"] = config.Display.Diagram.Duration;
     display["diagram_mode"] = config.Display.Diagram.Mode;
 
-    JsonArray leds = device.createNestedArray("led");
+    JsonArray leds = device["led"].to<JsonArray>();
     for (uint8_t i = 0; i < PINMAPPING_LED_COUNT; i++) {
-        JsonObject led = leds.createNestedObject();
+        JsonObject led = leds.add<JsonObject>();
         led["brightness"] = config.Led_Single[i].Brightness;
     }
 
-    JsonArray inverters = doc.createNestedArray("inverters");
+    JsonArray inverters = doc["inverters"].to<JsonArray>();
     for (uint8_t i = 0; i < INV_MAX_COUNT; i++) {
-        JsonObject inv = inverters.createNestedObject();
+        JsonObject inv = inverters.add<JsonObject>();
         inv["serial"] = config.Inverter[i].Serial;
         inv["name"] = config.Inverter[i].Name;
         inv["order"] = config.Inverter[i].Order;
@@ -136,21 +132,21 @@ bool ConfigurationClass::write()
         inv["zero_day"] = config.Inverter[i].ZeroYieldDayOnMidnight;
         inv["yieldday_correction"] = config.Inverter[i].YieldDayCorrection;
 
-        JsonArray channel = inv.createNestedArray("channel");
+        JsonArray channel = inv["channel"].to<JsonArray>();
         for (uint8_t c = 0; c < INV_MAX_CHAN_COUNT; c++) {
-            JsonObject chanData = channel.createNestedObject();
+            JsonObject chanData = channel.add<JsonObject>();
             chanData["name"] = config.Inverter[i].channel[c].Name;
             chanData["max_power"] = config.Inverter[i].channel[c].MaxChannelPower;
             chanData["yield_total_offset"] = config.Inverter[i].channel[c].YieldTotalOffset;
         }
     }
 
-    JsonObject vedirect = doc.createNestedObject("vedirect");
+    JsonObject vedirect = doc["vedirect"].to<JsonObject>();
     vedirect["enabled"] = config.Vedirect.Enabled;
     vedirect["verbose_logging"] = config.Vedirect.VerboseLogging;
     vedirect["updates_only"] = config.Vedirect.UpdatesOnly;
 
-    JsonObject powermeter = doc.createNestedObject("powermeter");
+    JsonObject powermeter = doc["powermeter"].to<JsonObject>();
     powermeter["enabled"] = config.PowerMeter.Enabled;
     powermeter["verbose_logging"] = config.PowerMeter.VerboseLogging;
     powermeter["interval"] = config.PowerMeter.Interval;
@@ -162,9 +158,9 @@ bool ConfigurationClass::write()
     powermeter["sdmaddress"] = config.PowerMeter.SdmAddress;
     powermeter["http_individual_requests"] = config.PowerMeter.HttpIndividualRequests;
 
-    JsonArray powermeter_http_phases = powermeter.createNestedArray("http_phases");
+    JsonArray powermeter_http_phases = powermeter["http_phases"].to<JsonArray>();
     for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) {
-        JsonObject powermeter_phase = powermeter_http_phases.createNestedObject();
+        JsonObject powermeter_phase = powermeter_http_phases.add<JsonObject>();
 
         powermeter_phase["enabled"] = config.PowerMeter.Http_Phase[i].Enabled;
         powermeter_phase["url"] = config.PowerMeter.Http_Phase[i].Url;
@@ -175,9 +171,11 @@ bool ConfigurationClass::write()
         powermeter_phase["header_value"] = config.PowerMeter.Http_Phase[i].HeaderValue;
         powermeter_phase["timeout"] = config.PowerMeter.Http_Phase[i].Timeout;
         powermeter_phase["json_path"] = config.PowerMeter.Http_Phase[i].JsonPath;
+        powermeter_phase["unit"] = config.PowerMeter.Http_Phase[i].PowerUnit;
+        powermeter_phase["sign_inverted"] = config.PowerMeter.Http_Phase[i].SignInverted;
     }
 
-    JsonObject powerlimiter = doc.createNestedObject("powerlimiter");
+    JsonObject powerlimiter = doc["powerlimiter"].to<JsonObject>();
     powerlimiter["enabled"] = config.PowerLimiter.Enabled;
     powerlimiter["verbose_logging"] = config.PowerLimiter.VerboseLogging;
     powerlimiter["solar_passtrough_enabled"] = config.PowerLimiter.SolarPassThroughEnabled;
@@ -191,6 +189,7 @@ bool ConfigurationClass::write()
     powerlimiter["target_power_consumption"] = config.PowerLimiter.TargetPowerConsumption;
     powerlimiter["target_power_consumption_hysteresis"] = config.PowerLimiter.TargetPowerConsumptionHysteresis;
     powerlimiter["lower_power_limit"] = config.PowerLimiter.LowerPowerLimit;
+    powerlimiter["base_load_limit"] = config.PowerLimiter.BaseLoadLimit;
     powerlimiter["upper_power_limit"] = config.PowerLimiter.UpperPowerLimit;
     powerlimiter["ignore_soc"] = config.PowerLimiter.IgnoreSoc;
     powerlimiter["battery_soc_start_threshold"] = config.PowerLimiter.BatterySocStartThreshold;
@@ -203,7 +202,7 @@ bool ConfigurationClass::write()
     powerlimiter["full_solar_passthrough_start_voltage"] = config.PowerLimiter.FullSolarPassThroughStartVoltage;
     powerlimiter["full_solar_passthrough_stop_voltage"] = config.PowerLimiter.FullSolarPassThroughStopVoltage;
 
-    JsonObject battery = doc.createNestedObject("battery");
+    JsonObject battery = doc["battery"].to<JsonObject>();
     battery["enabled"] = config.Battery.Enabled;
     battery["verbose_logging"] = config.Battery.VerboseLogging;
     battery["provider"] = config.Battery.Provider;
@@ -212,14 +211,23 @@ bool ConfigurationClass::write()
     battery["mqtt_topic"] = config.Battery.MqttSocTopic;
     battery["mqtt_voltage_topic"] = config.Battery.MqttVoltageTopic;
 
-    JsonObject huawei = doc.createNestedObject("huawei");
+    JsonObject huawei = doc["huawei"].to<JsonObject>();
     huawei["enabled"] = config.Huawei.Enabled;
+    huawei["verbose_logging"] = config.Huawei.VerboseLogging;
     huawei["can_controller_frequency"] = config.Huawei.CAN_Controller_Frequency;
     huawei["auto_power_enabled"] = config.Huawei.Auto_Power_Enabled;
+    huawei["auto_power_batterysoc_limits_enabled"] = config.Huawei.Auto_Power_BatterySoC_Limits_Enabled;
+    huawei["emergency_charge_enabled"] = config.Huawei.Emergency_Charge_Enabled;
     huawei["voltage_limit"] = config.Huawei.Auto_Power_Voltage_Limit;
     huawei["enable_voltage_limit"] = config.Huawei.Auto_Power_Enable_Voltage_Limit;
     huawei["lower_power_limit"] = config.Huawei.Auto_Power_Lower_Power_Limit;
     huawei["upper_power_limit"] = config.Huawei.Auto_Power_Upper_Power_Limit;
+    huawei["stop_batterysoc_threshold"] = config.Huawei.Auto_Power_Stop_BatterySoC_Threshold;
+    huawei["target_power_consumption"] = config.Huawei.Auto_Power_Target_Power_Consumption;
+
+    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
+        return false;
+    }
 
     // Serialize JSON to file
     if (serializeJson(doc, f) == 0) {
@@ -235,11 +243,7 @@ bool ConfigurationClass::read()
 {
     File f = LittleFS.open(CONFIG_FILENAME, "r", false);
 
-    DynamicJsonDocument doc(JSON_BUFFER_SIZE);
+    JsonDocument doc;
-
-    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
-        return false;
-    }
 
     // Deserialize the JSON document
     const DeserializationError error = deserializeJson(doc, f);
@@ -247,6 +251,10 @@ bool ConfigurationClass::read()
         MessageOutput.println("Failed to read file, using default configuration");
     }
 
+    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
+        return false;
+    }
+
     JsonObject cfg = doc["cfg"];
     config.Cfg.Version = cfg["version"] | CONFIG_VERSION;
     config.Cfg.SaveCount = cfg["save_count"] | 0;
@@ -415,13 +423,15 @@ bool ConfigurationClass::read()
 
         config.PowerMeter.Http_Phase[i].Enabled = powermeter_phase["enabled"] | (i == 0);
         strlcpy(config.PowerMeter.Http_Phase[i].Url, powermeter_phase["url"] | "", sizeof(config.PowerMeter.Http_Phase[i].Url));
-        config.PowerMeter.Http_Phase[i].AuthType = powermeter_phase["auth_type"] | Auth::none;
+        config.PowerMeter.Http_Phase[i].AuthType = powermeter_phase["auth_type"] | PowerMeterHttpConfig::Auth::None;
         strlcpy(config.PowerMeter.Http_Phase[i].Username, powermeter_phase["username"] | "", sizeof(config.PowerMeter.Http_Phase[i].Username));
         strlcpy(config.PowerMeter.Http_Phase[i].Password, powermeter_phase["password"] | "", sizeof(config.PowerMeter.Http_Phase[i].Password));
         strlcpy(config.PowerMeter.Http_Phase[i].HeaderKey, powermeter_phase["header_key"] | "", sizeof(config.PowerMeter.Http_Phase[i].HeaderKey));
         strlcpy(config.PowerMeter.Http_Phase[i].HeaderValue, powermeter_phase["header_value"] | "", sizeof(config.PowerMeter.Http_Phase[i].HeaderValue));
         config.PowerMeter.Http_Phase[i].Timeout = powermeter_phase["timeout"] | POWERMETER_HTTP_TIMEOUT;
         strlcpy(config.PowerMeter.Http_Phase[i].JsonPath, powermeter_phase["json_path"] | "", sizeof(config.PowerMeter.Http_Phase[i].JsonPath));
+        config.PowerMeter.Http_Phase[i].PowerUnit = powermeter_phase["unit"] | PowerMeterHttpConfig::Unit::Watts;
+        config.PowerMeter.Http_Phase[i].SignInverted = powermeter_phase["sign_inverted"] | false;
     }
 
     JsonObject powerlimiter = doc["powerlimiter"];
@@ -439,6 +449,7 @@ bool ConfigurationClass::read()
     config.PowerLimiter.TargetPowerConsumption = powerlimiter["target_power_consumption"] | POWERLIMITER_TARGET_POWER_CONSUMPTION;
     config.PowerLimiter.TargetPowerConsumptionHysteresis = powerlimiter["target_power_consumption_hysteresis"] | POWERLIMITER_TARGET_POWER_CONSUMPTION_HYSTERESIS;
     config.PowerLimiter.LowerPowerLimit = powerlimiter["lower_power_limit"] | POWERLIMITER_LOWER_POWER_LIMIT;
+    config.PowerLimiter.BaseLoadLimit = powerlimiter["base_load_limit"] | POWERLIMITER_BASE_LOAD_LIMIT;
     config.PowerLimiter.UpperPowerLimit = powerlimiter["upper_power_limit"] | POWERLIMITER_UPPER_POWER_LIMIT;
     config.PowerLimiter.IgnoreSoc = powerlimiter["ignore_soc"] | POWERLIMITER_IGNORE_SOC;
     config.PowerLimiter.BatterySocStartThreshold = powerlimiter["battery_soc_start_threshold"] | POWERLIMITER_BATTERY_SOC_START_THRESHOLD;
@@ -462,12 +473,17 @@ bool ConfigurationClass::read()
 
     JsonObject huawei = doc["huawei"];
     config.Huawei.Enabled = huawei["enabled"] | HUAWEI_ENABLED;
+    config.Huawei.VerboseLogging = huawei["verbose_logging"] | VERBOSE_LOGGING;
     config.Huawei.CAN_Controller_Frequency = huawei["can_controller_frequency"] | HUAWEI_CAN_CONTROLLER_FREQUENCY;
     config.Huawei.Auto_Power_Enabled = huawei["auto_power_enabled"] | false;
+    config.Huawei.Auto_Power_BatterySoC_Limits_Enabled = huawei["auto_power_batterysoc_limits_enabled"] | false;
+    config.Huawei.Emergency_Charge_Enabled = huawei["emergency_charge_enabled"] | false;
     config.Huawei.Auto_Power_Voltage_Limit = huawei["voltage_limit"] | HUAWEI_AUTO_POWER_VOLTAGE_LIMIT;
     config.Huawei.Auto_Power_Enable_Voltage_Limit =  huawei["enable_voltage_limit"] | HUAWEI_AUTO_POWER_ENABLE_VOLTAGE_LIMIT;
     config.Huawei.Auto_Power_Lower_Power_Limit = huawei["lower_power_limit"] | HUAWEI_AUTO_POWER_LOWER_POWER_LIMIT;
     config.Huawei.Auto_Power_Upper_Power_Limit = huawei["upper_power_limit"] | HUAWEI_AUTO_POWER_UPPER_POWER_LIMIT;
+    config.Huawei.Auto_Power_Stop_BatterySoC_Threshold = huawei["stop_batterysoc_threshold"] | HUAWEI_AUTO_POWER_STOP_BATTERYSOC_THRESHOLD;
+    config.Huawei.Auto_Power_Target_Power_Consumption = huawei["target_power_consumption"] | HUAWEI_AUTO_POWER_TARGET_POWER_CONSUMPTION;
 
     f.close();
     return true;
@@ -481,11 +497,7 @@ void ConfigurationClass::migrate()
         return;
     }
 
-    DynamicJsonDocument doc(JSON_BUFFER_SIZE);
+    JsonDocument doc;
-
-    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     // Deserialize the JSON document
     const DeserializationError error = deserializeJson(doc, f);
@@ -494,6 +506,10 @@ void ConfigurationClass::migrate()
         return;
     }
 
+    if (!Utils::checkJsonAlloc(doc, __FUNCTION__, __LINE__)) {
+        return;
+    }
+
     if (config.Cfg.Version < 0x00011700) {
         JsonArray inverters = doc["inverters"];
         for (uint8_t i = 0; i < INV_MAX_COUNT; i++) {
@@ -529,6 +545,12 @@ void ConfigurationClass::migrate()
         config.Dtu.Cmt.Frequency *= 1000;
     }
 
+    if (config.Cfg.Version < 0x00011c00) {
+        if (!strcmp(config.Ntp.Server, NTP_SERVER_OLD)) {
+            strlcpy(config.Ntp.Server, NTP_SERVER, sizeof(config.Ntp.Server));
+        }
+    }
+
     f.close();
 
     config.Cfg.Version = CONFIG_VERSION;

src/HttpPowerMeter.cpp

@@ -16,15 +16,17 @@ void HttpPowerMeterClass::init()
 
 float HttpPowerMeterClass::getPower(int8_t phase)
 {
+    if (phase < 1 || phase > POWERMETER_MAX_PHASES) { return 0.0; }
+
     return power[phase - 1];
 }
 
 bool HttpPowerMeterClass::updateValues()
 {
-    const CONFIG_T& config = Configuration.get();
+    auto const& config = Configuration.get();
 
     for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) {
-        POWERMETER_HTTP_PHASE_CONFIG_T phaseConfig = config.PowerMeter.Http_Phase[i];
+        auto const& phaseConfig = config.PowerMeter.Http_Phase[i];
 
         if (!phaseConfig.Enabled) {
             power[i] = 0.0;
@@ -32,8 +34,7 @@ bool HttpPowerMeterClass::updateValues()
         }
 
         if (i == 0 || config.PowerMeter.HttpIndividualRequests) {
-            if (!queryPhase(i, phaseConfig.Url, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout,
+            if (!queryPhase(i, phaseConfig)) {
-                    phaseConfig.JsonPath)) {
                 MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed.\r\n", i + 1);
                 MessageOutput.printf("%s\r\n", httpPowerMeterError);
                 return false;
@@ -41,7 +42,7 @@ bool HttpPowerMeterClass::updateValues()
             continue;
         }
 
-        if(!tryGetFloatValueForPhase(i, phaseConfig.JsonPath)) {
+        if(!tryGetFloatValueForPhase(i, phaseConfig.JsonPath, phaseConfig.PowerUnit, phaseConfig.SignInverted)) {
             MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d (from JSON fetched with Phase 1 config) failed.\r\n", i + 1);
             MessageOutput.printf("%s\r\n", httpPowerMeterError);
             return false;
@@ -50,8 +51,7 @@ bool HttpPowerMeterClass::updateValues()
     return true;
 }
 
-bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType, const char* username, const char* password,
+bool HttpPowerMeterClass::queryPhase(int phase, PowerMeterHttpConfig const& config)
-    const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath)
 {
     //hostByName in WiFiGeneric fails to resolve local names. issue described in
     //https://github.com/espressif/arduino-esp32/issues/3822
@@ -63,7 +63,7 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType
     String uri;
     String base64Authorization;
     uint16_t port;
-    extractUrlComponents(url, protocol, host, uri, port, base64Authorization);
+    extractUrlComponents(config.Url, protocol, host, uri, port, base64Authorization);
 
     IPAddress ipaddr((uint32_t)0);
     //first check if "host" is already an IP adress
@@ -105,43 +105,42 @@ bool HttpPowerMeterClass::queryPhase(int phase, const String& url, Auth authType
       wifiClient = std::make_unique<WiFiClient>();
     }
 
-    return httpRequest(phase, *wifiClient, ipaddr.toString(), port, uri, https, authType,  username, password, httpHeader, httpValue, timeout, jsonPath);
+    return httpRequest(phase, *wifiClient, ipaddr.toString(), port, uri, https, config);
 }
 
-bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& host, uint16_t port, const String& uri, bool https, Auth authType, const char* username,
+bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& host, uint16_t port, const String& uri, bool https, PowerMeterHttpConfig const& config)
-    const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath)
 {
     if(!httpClient.begin(wifiClient, host, port, uri, https)){
         snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), (https ? "https" : "http"), host.c_str());
         return false;
     }
 
-    prepareRequest(timeout, httpHeader, httpValue);
+    prepareRequest(config.Timeout, config.HeaderKey, config.HeaderValue);
-    if (authType == Auth::digest) {
+    if (config.AuthType == Auth_t::Digest) {
         const char *headers[1] = {"WWW-Authenticate"};
         httpClient.collectHeaders(headers, 1);
-    } else if (authType == Auth::basic) {
+    } else if (config.AuthType == Auth_t::Basic) {
-        String authString = username;
+        String authString = config.Username;
         authString += ":";
-        authString += password;
+        authString += config.Password;
         String auth = "Basic ";
         auth.concat(base64::encode(authString));
         httpClient.addHeader("Authorization", auth);
     }
     int httpCode = httpClient.GET();
 
-    if (httpCode == HTTP_CODE_UNAUTHORIZED && authType == Auth::digest) {
+    if (httpCode == HTTP_CODE_UNAUTHORIZED && config.AuthType == Auth_t::Digest) {
         // Handle authentication challenge
         if (httpClient.hasHeader("WWW-Authenticate")) {
             String authReq  = httpClient.header("WWW-Authenticate");
-            String authorization = getDigestAuth(authReq, String(username), String(password), "GET", String(uri), 1);
+            String authorization = getDigestAuth(authReq, String(config.Username), String(config.Password), "GET", String(uri), 1);
             httpClient.end();
             if(!httpClient.begin(wifiClient, host, port, uri, https)){
                 snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for  %s://%s using digest auth"), (https ? "https" : "http"), host.c_str());
                 return false;
             }
 
-            prepareRequest(timeout, httpHeader, httpValue);
+            prepareRequest(config.Timeout, config.HeaderKey, config.HeaderValue);
             httpClient.addHeader("Authorization", authorization);
             httpCode = httpClient.GET();
         }
@@ -160,7 +159,9 @@ bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const S
     httpResponse = httpClient.getString(); // very unfortunate that we cannot parse WifiClient stream directly
     httpClient.end();
 
-    return tryGetFloatValueForPhase(phase, jsonPath);
+    // TODO(schlimmchen): postpone calling tryGetFloatValueForPhase, as it
+    // will be called twice for each phase when doing separate requests.
+    return tryGetFloatValueForPhase(phase, config.JsonPath, config.PowerUnit, config.SignInverted);
 }
 
 String HttpPowerMeterClass::extractParam(String& authReq, const String& param, const char delimit) {
@@ -218,7 +219,7 @@ String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& usernam
     return authorization;
 }
 
-bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, const char* jsonPath)
+bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, const char* jsonPath, Unit_t unit, bool signInverted)
 {
     FirebaseJson json;
     json.setJsonData(httpResponse);
@@ -228,7 +229,22 @@ bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, const char* jsonPa
         return false;
     }
 
+    // this value is supposed to be in Watts and positive if energy is consumed.
     power[phase] = value.to<float>();
+
+    switch (unit) {
+        case Unit_t::MilliWatts:
+            power[phase] /= 1000;
+            break;
+        case Unit_t::KiloWatts:
+            power[phase] *= 1000;
+            break;
+        default:
+            break;
+    }
+
+    if (signInverted) { power[phase] *= -1; }
+
     return true;
 }
 

src/Huawei_can.cpp

@@ -2,17 +2,20 @@
 /*
  * Copyright (C) 2023 Malte Schmidt and others
  */
+#include "Battery.h"
 #include "Huawei_can.h"
 #include "MessageOutput.h"
 #include "PowerMeter.h"
 #include "PowerLimiter.h"
 #include "Configuration.h"
+#include "Battery.h"
 #include <SPI.h>
 #include <mcp_can.h>
 
 #include <freertos/FreeRTOS.h>
 #include <freertos/semphr.h>
 #include <freertos/task.h>
+#include <algorithm>
 #include <math.h>
 
 HuaweiCanClass HuaweiCan;
@@ -239,10 +242,6 @@ RectifierParameters_t * HuaweiCanClass::get()
     return &_rp;
 }
 
-uint32_t HuaweiCanClass::getLastUpdate()
-{
-    return _lastUpdateReceivedMillis;
-}
 
 void HuaweiCanClass::processReceivedParameters()
 {
@@ -272,6 +271,8 @@ void HuaweiCanClass::loop()
       return;
   }
 
+  bool verboseLogging = config.Huawei.VerboseLogging;
+
   processReceivedParameters();
 
   uint8_t com_error = HuaweiCanComm.getErrorCode(true);
@@ -283,7 +284,7 @@ void HuaweiCanClass::loop()
   }
 
   // Print updated data
-  if (HuaweiCanComm.gotNewRxDataFrame(false)) {
+  if (HuaweiCanComm.gotNewRxDataFrame(false) && verboseLogging) {
     MessageOutput.printf("[HuaweiCanClass::loop] In:  %.02fV, %.02fA, %.02fW\n", _rp.input_voltage, _rp.input_current, _rp.input_power);
     MessageOutput.printf("[HuaweiCanClass::loop] Out: %.02fV, %.02fA of %.02fA, %.02fW\n", _rp.output_voltage, _rp.output_current, _rp.max_output_current, _rp.output_power);
     MessageOutput.printf("[HuaweiCanClass::loop] Eff : %.01f%%, Temp in: %.01fC, Temp out: %.01fC\n", _rp.efficiency * 100, _rp.input_temp, _rp.output_temp);
@@ -298,18 +299,45 @@ void HuaweiCanClass::loop()
     digitalWrite(_huaweiPower, 1);
   }
 
-  // ***********************
-  // Automatic power control
-  // ***********************
 
-  if (_mode == HUAWEI_MODE_AUTO_INT ) {
+  if (_mode == HUAWEI_MODE_AUTO_INT || _batteryEmergencyCharging) {
 
-    // Set voltage limit in periodic intervals
+    // Set voltage limit in periodic intervals if we're in auto mode or if emergency battery charge is requested.
     if ( _nextAutoModePeriodicIntMillis < millis()) {
       MessageOutput.printf("[HuaweiCanClass::loop] Periodically setting voltage limit: %f \r\n", config.Huawei.Auto_Power_Voltage_Limit);
       _setValue(config.Huawei.Auto_Power_Voltage_Limit, HUAWEI_ONLINE_VOLTAGE);
       _nextAutoModePeriodicIntMillis = millis() + 60000;
     }
+  }
+  // ***********************
+  // Emergency charge
+  // ***********************
+  auto stats = Battery.getStats();
+  if (config.Huawei.Emergency_Charge_Enabled && stats->getImmediateChargingRequest()) {
+    _batteryEmergencyCharging = true;
+
+    // Set output current
+    float efficiency =  (_rp.efficiency > 0.5 ? _rp.efficiency : 1.0);
+    float outputCurrent = efficiency * (config.Huawei.Auto_Power_Upper_Power_Limit / _rp.output_voltage);
+    MessageOutput.printf("[HuaweiCanClass::loop] Emergency Charge Output current %f \r\n", outputCurrent);
+    _setValue(outputCurrent, HUAWEI_ONLINE_CURRENT);
+    return;
+  }
+
+  if (_batteryEmergencyCharging && !stats->getImmediateChargingRequest()) {
+    // Battery request has changed. Set current to 0, wait for PSU to respond and then clear state
+    _setValue(0, HUAWEI_ONLINE_CURRENT);
+    if (_rp.output_current < 1) {
+      _batteryEmergencyCharging = false;
+    }
+    return;
+  }
+
+  // ***********************
+  // Automatic power control
+  // ***********************
+
+  if (_mode == HUAWEI_MODE_AUTO_INT ) {
 
     // Check if we should run automatic power calculation at all.
     // We may have set a value recently and still wait for output stabilization
@@ -352,8 +380,26 @@ void HuaweiCanClass::loop()
 
       // Calculate new power limit
       float newPowerLimit = -1 * round(PowerMeter.getPowerTotal());
-      newPowerLimit += _rp.output_power;
+      float efficiency =  (_rp.efficiency > 0.5 ? _rp.efficiency : 1.0);
-      MessageOutput.printf("[HuaweiCanClass::loop] PL: %f, OP: %f \r\n", newPowerLimit, _rp.output_power);
+
+      // Powerlimit is the requested output power + permissable Grid consumption factoring in the efficiency factor
+      newPowerLimit += _rp.output_power + config.Huawei.Auto_Power_Target_Power_Consumption / efficiency;
+
+      if (verboseLogging){
+        MessageOutput.printf("[HuaweiCanClass::loop] newPowerLimit: %f, output_power: %f \r\n", newPowerLimit, _rp.output_power);
+      }
+
+      if (config.Battery.Enabled && config.Huawei.Auto_Power_BatterySoC_Limits_Enabled) {
+        uint8_t _batterySoC = Battery.getStats()->getSoC();
+        if (_batterySoC >= config.Huawei.Auto_Power_Stop_BatterySoC_Threshold) {
+          newPowerLimit = 0;
+          if (verboseLogging) {
+            MessageOutput.printf("[HuaweiCanClass::loop] Current battery SoC %i reached "
+                    "stop threshold %i, set newPowerLimit to %f \r\n", _batterySoC,
+                    config.Huawei.Auto_Power_Stop_BatterySoC_Threshold, newPowerLimit);
+          }
+        }
+      }
 
       if (newPowerLimit > config.Huawei.Auto_Power_Lower_Power_Limit) {
 
@@ -377,10 +423,17 @@ void HuaweiCanClass::loop()
           newPowerLimit = config.Huawei.Auto_Power_Upper_Power_Limit;
         }
 
-        // Set the actual output limit
+        // Calculate output current
-        float efficiency =  (_rp.efficiency > 0.5 ? _rp.efficiency : 1.0); 
+        float calculatedCurrent = efficiency * (newPowerLimit / _rp.output_voltage);
-        float outputCurrent = efficiency * (newPowerLimit / _rp.output_voltage);
+
-        MessageOutput.printf("[HuaweiCanClass::loop] Output current %f \r\n", outputCurrent);
+        // Limit output current to value requested by BMS
+        float permissableCurrent = stats->getChargeCurrentLimitation() - (stats->getChargeCurrent() - _rp.output_current); // BMS current limit - current from other sources
+        float outputCurrent = std::min(calculatedCurrent, permissableCurrent);
+        outputCurrent= outputCurrent > 0 ? outputCurrent : 0;
+
+        if (verboseLogging) {
+            MessageOutput.printf("[HuaweiCanClass::loop] Setting output current to %.2fA. This is the lower value of calculated %.2fA and BMS permissable %.2fA currents\r\n", outputCurrent, calculatedCurrent, permissableCurrent);
+        }
         _autoPowerEnabled = true;
         _setValue(outputCurrent, HUAWEI_ONLINE_CURRENT);
 
@@ -415,6 +468,7 @@ void HuaweiCanClass::_setValue(float in, uint8_t parameterType)
 
     if (in < 0) {
       MessageOutput.printf("[HuaweiCanClass::_setValue]  Error: Tried to set voltage/current to negative value %f \r\n", in);
+      return;
     }
 
     // Start PSU if needed
@@ -466,7 +520,5 @@ void HuaweiCanClass::setMode(uint8_t mode) {
   }
 }
 
-bool HuaweiCanClass::getAutoPowerStatus() {
+
-  return _autoPowerEnabled;
-}
 

src/InverterSettings.cpp

@@ -51,9 +51,9 @@ void InverterSettingsClass::init(Scheduler& scheduler)
 
         if (PinMapping.isValidCmt2300Config()) {
             Hoymiles.initCMT(pin.cmt_sdio, pin.cmt_clk, pin.cmt_cs, pin.cmt_fcs, pin.cmt_gpio2, pin.cmt_gpio3);
-            MessageOutput.println(F("  Setting country mode... "));
+            MessageOutput.println("  Setting country mode... ");
             Hoymiles.getRadioCmt()->setCountryMode(static_cast<CountryModeId_t>(config.Dtu.Cmt.CountryMode));
-            MessageOutput.println(F("  Setting CMT target frequency... "));
+            MessageOutput.println("  Setting CMT target frequency... ");
             Hoymiles.getRadioCmt()->setInverterTargetFrequency(config.Dtu.Cmt.Frequency);
         }
 

src/JkBmsController.cpp

@@ -198,7 +198,7 @@ class DummySerial {
 };
 DummySerial HwSerial;
 #else
-HardwareSerial HwSerial(2);
+HardwareSerial HwSerial((ARDUINO_USB_CDC_ON_BOOT != 1)?2:0);
 #endif
 
 namespace JkBms {
@@ -220,6 +220,7 @@ bool Controller::init(bool verboseLogging)
         return false;
     }
 
+    HwSerial.end(); // make sure the UART will be re-initialized
     HwSerial.begin(115200, SERIAL_8N1, pin.battery_rx, pin.battery_tx);
     HwSerial.flush();
 

src/MqttHandlVedirectHass.cpp

@@ -58,42 +58,44 @@ void MqttHandleVedirectHassClass::publishConfig()
 
     // device info
     for (int idx = 0; idx < VictronMppt.controllerAmount(); ++idx) {
-        // ensure data is received from victron
+        auto optMpptData = VictronMppt.getData(idx);
-        if (!VictronMppt.isDataValid(idx)) {
+        if (!optMpptData.has_value()) { continue; }
-            continue;
-        }
 
-        std::optional<VeDirectMpptController::spData_t> spOptMpptData = VictronMppt.getData(idx);
+        publishBinarySensor("MPPT load output state", "mdi:export", "LOAD", "ON", "OFF", *optMpptData);
-        if (!spOptMpptData.has_value()) {
+        publishSensor("MPPT serial number", "mdi:counter", "SER", nullptr, nullptr, nullptr, *optMpptData);
-            continue;
+        publishSensor("MPPT firmware number", "mdi:counter", "FW", nullptr, nullptr, nullptr, *optMpptData);
-        }
+        publishSensor("MPPT state of operation", "mdi:wrench", "CS", nullptr, nullptr, nullptr, *optMpptData);
-
+        publishSensor("MPPT error code", "mdi:bell", "ERR", nullptr, nullptr, nullptr, *optMpptData);
-        VeDirectMpptController::spData_t &spMpptData = spOptMpptData.value();
+        publishSensor("MPPT off reason", "mdi:wrench", "OR", nullptr, nullptr, nullptr, *optMpptData);
-
+        publishSensor("MPPT tracker operation mode", "mdi:wrench", "MPPT", nullptr, nullptr, nullptr, *optMpptData);
-        publishBinarySensor("MPPT load output state", "mdi:export", "LOAD", "ON", "OFF", spMpptData);
+        publishSensor("MPPT Day sequence number (0...364)", "mdi:calendar-month-outline", "HSDS", NULL, "total", "d", *optMpptData);
-        publishSensor("MPPT serial number", "mdi:counter", "SER", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT firmware number", "mdi:counter", "FW", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT state of operation", "mdi:wrench", "CS", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT error code", "mdi:bell", "ERR", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT off reason", "mdi:wrench", "OR", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT tracker operation mode", "mdi:wrench", "MPPT", nullptr, nullptr, nullptr, spMpptData);
-        publishSensor("MPPT Day sequence number (0...364)", "mdi:calendar-month-outline", "HSDS", NULL, "total", "d", spMpptData);
 
         // battery info
-        publishSensor("Battery voltage", NULL, "V", "voltage", "measurement", "V", spMpptData);
+        publishSensor("Battery voltage", NULL, "V", "voltage", "measurement", "V", *optMpptData);
-        publishSensor("Battery current", NULL, "I", "current", "measurement", "A", spMpptData);
+        publishSensor("Battery current", NULL, "I", "current", "measurement", "A", *optMpptData);
-        publishSensor("Battery power (calculated)", NULL, "P", "power", "measurement", "W", spMpptData);
+        publishSensor("Battery power (calculated)", NULL, "P", "power", "measurement", "W", *optMpptData);
-        publishSensor("Battery efficiency (calculated)", NULL, "E", NULL, "measurement", "%", spMpptData);
+        publishSensor("Battery efficiency (calculated)", NULL, "E", NULL, "measurement", "%", *optMpptData);
 
         // panel info
-        publishSensor("Panel voltage", NULL, "VPV", "voltage", "measurement", "V", spMpptData);
+        publishSensor("Panel voltage", NULL, "VPV", "voltage", "measurement", "V", *optMpptData);
-        publishSensor("Panel current (calculated)", NULL, "IPV", "current", "measurement", "A", spMpptData);
+        publishSensor("Panel current (calculated)", NULL, "IPV", "current", "measurement", "A", *optMpptData);
-        publishSensor("Panel power", NULL, "PPV", "power", "measurement", "W", spMpptData);
+        publishSensor("Panel power", NULL, "PPV", "power", "measurement", "W", *optMpptData);
-        publishSensor("Panel yield total", NULL, "H19", "energy", "total_increasing", "kWh", spMpptData);
+        publishSensor("Panel yield total", NULL, "H19", "energy", "total_increasing", "kWh", *optMpptData);
-        publishSensor("Panel yield today", NULL, "H20", "energy", "total", "kWh", spMpptData);
+        publishSensor("Panel yield today", NULL, "H20", "energy", "total", "kWh", *optMpptData);
-        publishSensor("Panel maximum power today", NULL, "H21", "power", "measurement", "W", spMpptData);
+        publishSensor("Panel maximum power today", NULL, "H21", "power", "measurement", "W", *optMpptData);
-        publishSensor("Panel yield yesterday", NULL, "H22", "energy", "total", "kWh", spMpptData);
+        publishSensor("Panel yield yesterday", NULL, "H22", "energy", "total", "kWh", *optMpptData);
-        publishSensor("Panel maximum power yesterday", NULL, "H23", "power", "measurement", "W", spMpptData);
+        publishSensor("Panel maximum power yesterday", NULL, "H23", "power", "measurement", "W", *optMpptData);
+
+        // optional info, provided only if TX is connected to charge controller
+        if (optMpptData->NetworkTotalDcInputPowerMilliWatts.first != 0) {
+            publishSensor("VE.Smart network total DC input power", "mdi:solar-power", "NetworkTotalDcInputPower", "power", "measurement", "W", *optMpptData);
+        }
+        if (optMpptData->MpptTemperatureMilliCelsius.first != 0) {
+            publishSensor("MPPT temperature", "mdi:temperature-celsius", "MpptTemperature", "temperature", "measurement", "W", *optMpptData);
+        }
+        if (optMpptData->SmartBatterySenseTemperatureMilliCelsius.first != 0) {
+            publishSensor("Smart Battery Sense temperature", "mdi:temperature-celsius", "SmartBatterySenseTemperature", "temperature", "measurement", "W", *optMpptData);
+        }
     }
 
     yield();
@@ -102,9 +104,9 @@ void MqttHandleVedirectHassClass::publishConfig()
 void MqttHandleVedirectHassClass::publishSensor(const char *caption, const char *icon, const char *subTopic,
                                                 const char *deviceClass, const char *stateClass,
                                                 const char *unitOfMeasurement,
-                                                const VeDirectMpptController::spData_t &spMpptData)
+                                                const VeDirectMpptController::data_t &mpptData)
 {
-    String serial = spMpptData->SER;
+    String serial = mpptData.serialNr_SER;
 
     String sensorId = caption;
     sensorId.replace(" ", "_");
@@ -122,10 +124,8 @@ void MqttHandleVedirectHassClass::publishSensor(const char *caption, const char
     statTopic.concat("/");
     statTopic.concat(subTopic);
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+
-        return;
-    }
     root["name"] = caption;
     root["stat_t"] = statTopic;
     root["uniq_id"] = serial + "_" + sensorId;
@@ -138,8 +138,8 @@ void MqttHandleVedirectHassClass::publishSensor(const char *caption, const char
         root["unit_of_meas"] = unitOfMeasurement;
     }
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    JsonObject deviceObj = root["dev"].to<JsonObject>();
-    createDeviceInfo(deviceObj, spMpptData);
+    createDeviceInfo(deviceObj, mpptData);
 
     if (Configuration.get().Mqtt.Hass.Expire) {
         root["exp_aft"] = Configuration.get().Mqtt.PublishInterval * 3;
@@ -151,7 +151,9 @@ void MqttHandleVedirectHassClass::publishSensor(const char *caption, const char
         root["stat_cla"] = stateClass;
     }
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     char buffer[512];
     serializeJson(root, buffer);
@@ -160,9 +162,9 @@ void MqttHandleVedirectHassClass::publishSensor(const char *caption, const char
 }
 void MqttHandleVedirectHassClass::publishBinarySensor(const char *caption, const char *icon, const char *subTopic,
                                                       const char *payload_on, const char *payload_off,
-                                                      const VeDirectMpptController::spData_t &spMpptData)
+                                                      const VeDirectMpptController::data_t &mpptData)
 {
-    String serial = spMpptData->SER;
+    String serial = mpptData.serialNr_SER;
 
     String sensorId = caption;
     sensorId.replace(" ", "_");
@@ -180,10 +182,7 @@ void MqttHandleVedirectHassClass::publishBinarySensor(const char *caption, const
     statTopic.concat("/");
     statTopic.concat(subTopic);
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
     root["name"] = caption;
     root["uniq_id"] = serial + "_" + sensorId;
     root["stat_t"] = statTopic;
@@ -194,10 +193,12 @@ void MqttHandleVedirectHassClass::publishBinarySensor(const char *caption, const
         root["icon"] = icon;
     }
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    JsonObject deviceObj = root["dev"].to<JsonObject>();
-    createDeviceInfo(deviceObj, spMpptData);
+    createDeviceInfo(deviceObj, mpptData);
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     char buffer[512];
     serializeJson(root, buffer);
@@ -205,14 +206,14 @@ void MqttHandleVedirectHassClass::publishBinarySensor(const char *caption, const
 }
 
 void MqttHandleVedirectHassClass::createDeviceInfo(JsonObject &object,
-                                                   const VeDirectMpptController::spData_t &spMpptData)
+                                                   const VeDirectMpptController::data_t &mpptData)
 {
-    String serial = spMpptData->SER;
+    String serial = mpptData.serialNr_SER;
     object["name"] = "Victron(" + serial + ")";
     object["ids"] = serial;
     object["cu"] = String("http://") + NetworkSettings.localIP().toString();
     object["mf"] = "OpenDTU";
-    object["mdl"] = spMpptData->getPidAsString();
+    object["mdl"] = mpptData.getPidAsString();
     object["sw"] = AUTO_GIT_HASH;
 }
 

src/MqttHandleBatteryHass.cpp

@@ -144,10 +144,7 @@ void MqttHandleBatteryHassClass::publishSensor(const char* caption, const char*
     // statTopic.concat("/");
     statTopic.concat(subTopic);
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
     root["name"] = caption;
     root["stat_t"] = statTopic;
     root["uniq_id"] = serial + "_" + sensorId;
@@ -160,7 +157,7 @@ void MqttHandleBatteryHassClass::publishSensor(const char* caption, const char*
         root["unit_of_meas"] = unitOfMeasurement;
     }
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    JsonObject deviceObj = root["dev"].to<JsonObject>();
     createDeviceInfo(deviceObj);
 
     if (Configuration.get().Mqtt.Hass.Expire) {
@@ -173,7 +170,9 @@ void MqttHandleBatteryHassClass::publishSensor(const char* caption, const char*
         root["stat_cla"] = stateClass;
     }
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     char buffer[512];
     serializeJson(root, buffer);
@@ -201,10 +200,8 @@ void MqttHandleBatteryHassClass::publishBinarySensor(const char* caption, const
     // statTopic.concat("/");
     statTopic.concat(subTopic);
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+  
-        return;
-    }
     root["name"] = caption;
     root["uniq_id"] = serial + "_" + sensorId;
     root["stat_t"] = statTopic;
@@ -215,10 +212,12 @@ void MqttHandleBatteryHassClass::publishBinarySensor(const char* caption, const
         root["icon"] = icon;
     }
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    auto deviceObj = root["dev"].to<JsonObject>();
     createDeviceInfo(deviceObj);
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     char buffer[512];
     serializeJson(root, buffer);

src/MqttHandleHass.cpp

@@ -137,10 +137,7 @@ void MqttHandleHassClass::publishInverterField(std::shared_ptr<InverterAbstract>
             name = "CH" + chanNum + " " + fieldName;
         }
 
-        DynamicJsonDocument root(1024);
+        JsonDocument root;
-        if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-            return;
-        }
 
         root["name"] = name;
         root["stat_t"] = stateTopic;
@@ -163,6 +160,10 @@ void MqttHandleHassClass::publishInverterField(std::shared_ptr<InverterAbstract>
             root["stat_cla"] = stateCls;
         }
 
+        if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+            return;
+        }
+
         String buffer;
         serializeJson(root, buffer);
         publish(configTopic, buffer);
@@ -185,10 +186,7 @@ void MqttHandleHassClass::publishInverterButton(std::shared_ptr<InverterAbstract
 
     const String cmdTopic = MqttSettings.getPrefix() + serial + "/" + subTopic;
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = caption;
     root["uniq_id"] = serial + "_" + buttonId;
@@ -204,6 +202,10 @@ void MqttHandleHassClass::publishInverterButton(std::shared_ptr<InverterAbstract
 
     createInverterInfo(root, inv);
 
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
+
     String buffer;
     serializeJson(root, buffer);
     publish(configTopic, buffer);
@@ -227,10 +229,7 @@ void MqttHandleHassClass::publishInverterNumber(
     const String cmdTopic = MqttSettings.getPrefix() + serial + "/" + commandTopic;
     const String statTopic = MqttSettings.getPrefix() + serial + "/" + stateTopic;
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = caption;
     root["uniq_id"] = serial + "_" + buttonId;
@@ -246,6 +245,10 @@ void MqttHandleHassClass::publishInverterNumber(
 
     createInverterInfo(root, inv);
 
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
+
     String buffer;
     serializeJson(root, buffer);
     publish(configTopic, buffer);
@@ -265,10 +268,7 @@ void MqttHandleHassClass::publishInverterBinarySensor(std::shared_ptr<InverterAb
 
     const String statTopic = MqttSettings.getPrefix() + serial + "/" + subTopic;
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = caption;
     root["uniq_id"] = serial + "_" + sensorId;
@@ -278,6 +278,10 @@ void MqttHandleHassClass::publishInverterBinarySensor(std::shared_ptr<InverterAb
 
     createInverterInfo(root, inv);
 
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
+
     String buffer;
     serializeJson(root, buffer);
     publish(configTopic, buffer);
@@ -293,10 +297,7 @@ void MqttHandleHassClass::publishDtuSensor(const char* name, const char* device_
         topic = id;
     }
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = name;
     root["uniq_id"] = getDtuUniqueId() + "_" + id;
@@ -322,6 +323,8 @@ void MqttHandleHassClass::publishDtuSensor(const char* name, const char* device_
 
     createDtuInfo(root);
 
+ 
+
     String buffer;
     const String configTopic = "sensor/" + getDtuUniqueId() + "/" + id + "/config";
     serializeJson(root, buffer);
@@ -339,10 +342,7 @@ void MqttHandleHassClass::publishDtuBinarySensor(const char* name, const char* d
         topic = String("dtu/") + "/" + id;
     }
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = name;
     root["uniq_id"] = getDtuUniqueId() + "_" + id;
@@ -359,13 +359,17 @@ void MqttHandleHassClass::publishDtuBinarySensor(const char* name, const char* d
 
     createDtuInfo(root);
 
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
+
     String buffer;
     const String configTopic = "binary_sensor/" + getDtuUniqueId() + "/" + id + "/config";
     serializeJson(root, buffer);
     publish(configTopic, buffer);
 }
 
-void MqttHandleHassClass::createInverterInfo(DynamicJsonDocument& root, std::shared_ptr<InverterAbstract> inv)
+void MqttHandleHassClass::createInverterInfo(JsonDocument& root, std::shared_ptr<InverterAbstract> inv)
 {
     createDeviceInfo(
         root,
@@ -378,7 +382,7 @@ void MqttHandleHassClass::createInverterInfo(DynamicJsonDocument& root, std::sha
         getDtuUniqueId());
 }
 
-void MqttHandleHassClass::createDtuInfo(DynamicJsonDocument& root)
+void MqttHandleHassClass::createDtuInfo(JsonDocument& root)
 {
     createDeviceInfo(
         root,
@@ -391,12 +395,12 @@ void MqttHandleHassClass::createDtuInfo(DynamicJsonDocument& root)
 }
 
 void MqttHandleHassClass::createDeviceInfo(
-    DynamicJsonDocument& root,
+    JsonDocument& root,
     const String& name, const String& identifiers, const String& configuration_url,
     const String& manufacturer, const String& model, const String& sw_version,
     const String& via_device)
 {
-    auto object = root.createNestedObject("dev");
+    auto object = root["dev"].to<JsonObject>();
 
     object["name"] = name;
     object["ids"] = identifiers;

src/MqttHandleHuawei.cpp

@@ -75,6 +75,7 @@ void MqttHandleHuaweiClass::loop()
       MqttSettings.publish("huawei/input_temp", String(rp->input_temp));
       MqttSettings.publish("huawei/output_temp", String(rp->output_temp));
       MqttSettings.publish("huawei/efficiency", String(rp->efficiency));
+      MqttSettings.publish("huawei/mode", String(HuaweiCan.getMode()));
 
 
       yield();

src/MqttHandlePowerLimiter.cpp

@@ -76,6 +76,8 @@ void MqttHandlePowerLimiterClass::loop()
     auto val = static_cast<unsigned>(PowerLimiter.getMode());
     MqttSettings.publish("powerlimiter/status/mode", String(val));
 
+    MqttSettings.publish("powerlimiter/status/inverter_update_timeouts", String(PowerLimiter.getInverterUpdateTimeouts()));
+
     // no thresholds are relevant for setups without a battery
     if (config.PowerLimiter.IsInverterSolarPowered) { return; }
 

src/MqttHandlePowerLimiterHass.cpp

@@ -112,10 +112,7 @@ void MqttHandlePowerLimiterHassClass::publishSelect(
     const String cmdTopic = MqttSettings.getPrefix() + "powerlimiter/cmd/" + commandTopic;
     const String statTopic = MqttSettings.getPrefix() + "powerlimiter/status/" + stateTopic;
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = caption;
     root["uniq_id"] = selectId;
@@ -125,15 +122,17 @@ void MqttHandlePowerLimiterHassClass::publishSelect(
     root["ent_cat"] = category;
     root["cmd_t"] = cmdTopic;
     root["stat_t"] = statTopic;
-    JsonArray options = root.createNestedArray("options");
+    JsonArray options = root["options"].to<JsonArray>();
     options.add("0");
     options.add("1");
     options.add("2");
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    JsonObject deviceObj = root["dev"].to<JsonObject>();
     createDeviceInfo(deviceObj);
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     String buffer;
     serializeJson(root, buffer);
@@ -155,10 +154,7 @@ void MqttHandlePowerLimiterHassClass::publishNumber(
     const String cmdTopic = MqttSettings.getPrefix() + "powerlimiter/cmd/" + commandTopic;
     const String statTopic = MqttSettings.getPrefix() + "powerlimiter/status/" + stateTopic;
 
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-        return;
-    }
 
     root["name"] = caption;
     root["uniq_id"] = numberId;
@@ -178,10 +174,12 @@ void MqttHandlePowerLimiterHassClass::publishNumber(
         root["exp_aft"] = config.Mqtt.PublishInterval * 3;
     }
 
-    JsonObject deviceObj = root.createNestedObject("dev");
+    JsonObject deviceObj = root["dev"].to<JsonObject>();
     createDeviceInfo(deviceObj);
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+        return;
+    }
 
     String buffer;
     serializeJson(root, buffer);

src/MqttHandleVedirect.cpp

@@ -59,21 +59,13 @@ void MqttHandleVedirectClass::loop()
         #endif
 
         for (int idx = 0; idx < VictronMppt.controllerAmount(); ++idx) {
-            if (!VictronMppt.isDataValid(idx)) {
+            std::optional<VeDirectMpptController::data_t> optMpptData = VictronMppt.getData(idx);
-                continue;
+            if (!optMpptData.has_value()) { continue; }
-            }
 
-            std::optional<VeDirectMpptController::spData_t> spOptMpptData = VictronMppt.getData(idx);
+            auto const& kvFrame = _kvFrames[optMpptData->serialNr_SER];
-            if (!spOptMpptData.has_value()) {
+            publish_mppt_data(*optMpptData, kvFrame);
-                continue;
-            }
-
-            VeDirectMpptController::spData_t &spMpptData = spOptMpptData.value();
-
-            VeDirectMpptController::veMpptStruct _kvFrame = _kvFrames[spMpptData->SER];
-            publish_mppt_data(spMpptData, _kvFrame);
             if (!_PublishFull) {
-                _kvFrames[spMpptData->SER] = *spMpptData;
+                _kvFrames[optMpptData->serialNr_SER] = *optMpptData;
             }
         }
 
@@ -104,79 +96,48 @@ void MqttHandleVedirectClass::loop()
     }
 }
 
-void MqttHandleVedirectClass::publish_mppt_data(const VeDirectMpptController::spData_t &spMpptData,
+void MqttHandleVedirectClass::publish_mppt_data(const VeDirectMpptController::data_t &currentData,
-                                                VeDirectMpptController::veMpptStruct &frame) const {
+                                                const VeDirectMpptController::data_t &previousData) const {
     String value;
     String topic = "victron/";
-    topic.concat(spMpptData->SER);
+    topic.concat(currentData.serialNr_SER);
     topic.concat("/");
 
-    if (_PublishFull || spMpptData->PID != frame.PID)
+#define PUBLISH(sm, t, val) \
-        MqttSettings.publish(topic + "PID", spMpptData->getPidAsString().data());
+    if (_PublishFull || currentData.sm != previousData.sm) { \
-    if (_PublishFull || strcmp(spMpptData->SER, frame.SER) != 0)
+        MqttSettings.publish(topic + t, String(val)); \
-        MqttSettings.publish(topic + "SER", spMpptData->SER );
-    if (_PublishFull || strcmp(spMpptData->FW, frame.FW) != 0)
-        MqttSettings.publish(topic + "FW", spMpptData->FW);
-    if (_PublishFull || spMpptData->LOAD != frame.LOAD)
-        MqttSettings.publish(topic + "LOAD", spMpptData->LOAD ? "ON" : "OFF");
-    if (_PublishFull || spMpptData->CS != frame.CS)
-        MqttSettings.publish(topic + "CS", spMpptData->getCsAsString().data());
-    if (_PublishFull || spMpptData->ERR != frame.ERR)
-        MqttSettings.publish(topic + "ERR", spMpptData->getErrAsString().data());
-    if (_PublishFull || spMpptData->OR != frame.OR)
-        MqttSettings.publish(topic + "OR", spMpptData->getOrAsString().data());
-    if (_PublishFull || spMpptData->MPPT != frame.MPPT)
-        MqttSettings.publish(topic + "MPPT", spMpptData->getMpptAsString().data());
-    if (_PublishFull || spMpptData->HSDS != frame.HSDS) {
-        value = spMpptData->HSDS;
-        MqttSettings.publish(topic + "HSDS", value);
     }
-    if (_PublishFull || spMpptData->V != frame.V) {
+
-        value = spMpptData->V;
+    PUBLISH(productID_PID,           "PID",  currentData.getPidAsString().data());
-        MqttSettings.publish(topic + "V", value);
+    PUBLISH(serialNr_SER,            "SER",  currentData.serialNr_SER);
-    }
+    PUBLISH(firmwareNr_FW,            "FW",  currentData.firmwareNr_FW);
-    if (_PublishFull || spMpptData->I != frame.I) {
+    PUBLISH(loadOutputState_LOAD,   "LOAD",  (currentData.loadOutputState_LOAD ? "ON" : "OFF"));
-        value = spMpptData->I;
+    PUBLISH(currentState_CS,          "CS",  currentData.getCsAsString().data());
-        MqttSettings.publish(topic + "I", value);
+    PUBLISH(errorCode_ERR,           "ERR",  currentData.getErrAsString().data());
-    }
+    PUBLISH(offReason_OR,             "OR",  currentData.getOrAsString().data());
-    if (_PublishFull || spMpptData->P != frame.P) {
+    PUBLISH(stateOfTracker_MPPT,    "MPPT",  currentData.getMpptAsString().data());
-        value = spMpptData->P;
+    PUBLISH(daySequenceNr_HSDS,     "HSDS",  currentData.daySequenceNr_HSDS);
-        MqttSettings.publish(topic + "P", value);
+    PUBLISH(batteryVoltage_V_mV,       "V",  currentData.batteryVoltage_V_mV / 1000.0);
-    }
+    PUBLISH(batteryCurrent_I_mA,       "I",  currentData.batteryCurrent_I_mA / 1000.0);
-    if (_PublishFull || spMpptData->VPV != frame.VPV) {
+    PUBLISH(batteryOutputPower_W,      "P",  currentData.batteryOutputPower_W);
-        value = spMpptData->VPV;
+    PUBLISH(panelVoltage_VPV_mV,     "VPV",  currentData.panelVoltage_VPV_mV / 1000.0);
-        MqttSettings.publish(topic + "VPV", value);
+    PUBLISH(panelCurrent_mA,         "IPV",  currentData.panelCurrent_mA / 1000.0);
-    }
+    PUBLISH(panelPower_PPV_W,        "PPV",  currentData.panelPower_PPV_W);
-    if (_PublishFull || spMpptData->IPV != frame.IPV) {
+    PUBLISH(mpptEfficiency_Percent,    "E",  currentData.mpptEfficiency_Percent);
-        value = spMpptData->IPV;
+    PUBLISH(yieldTotal_H19_Wh,       "H19",  currentData.yieldTotal_H19_Wh / 1000.0);
-        MqttSettings.publish(topic + "IPV", value);
+    PUBLISH(yieldToday_H20_Wh,       "H20",  currentData.yieldToday_H20_Wh / 1000.0);
-    }
+    PUBLISH(maxPowerToday_H21_W,     "H21",  currentData.maxPowerToday_H21_W);
-    if (_PublishFull || spMpptData->PPV != frame.PPV) {
+    PUBLISH(yieldYesterday_H22_Wh,   "H22",  currentData.yieldYesterday_H22_Wh / 1000.0);
-        value = spMpptData->PPV;
+    PUBLISH(maxPowerYesterday_H23_W, "H23",  currentData.maxPowerYesterday_H23_W);
-        MqttSettings.publish(topic + "PPV", value);
+#undef PUBLILSH
-    }
+
-    if (_PublishFull || spMpptData->E != frame.E) {
+#define PUBLISH_OPT(sm, t, val) \
-        value = spMpptData->E;
+    if (currentData.sm.first != 0 && (_PublishFull || currentData.sm.second != previousData.sm.second)) { \
-        MqttSettings.publish(topic + "E", value);
+        MqttSettings.publish(topic + t, String(val)); \
-    }
-    if (_PublishFull || spMpptData->H19 != frame.H19) {
-        value = spMpptData->H19;
-        MqttSettings.publish(topic + "H19", value);
-    }
-    if (_PublishFull || spMpptData->H20 != frame.H20) {
-        value = spMpptData->H20;
-        MqttSettings.publish(topic + "H20", value);
-    }
-    if (_PublishFull || spMpptData->H21 != frame.H21) {
-        value = spMpptData->H21;
-        MqttSettings.publish(topic + "H21", value);
-    }
-    if (_PublishFull || spMpptData->H22 != frame.H22) {
-        value = spMpptData->H22;
-        MqttSettings.publish(topic + "H22", value);
-    }
-    if (_PublishFull || spMpptData->H23 != frame.H23) {
-        value = spMpptData->H23;
-        MqttSettings.publish(topic + "H23", value);
     }
+
+    PUBLISH_OPT(NetworkTotalDcInputPowerMilliWatts,       "NetworkTotalDcInputPower",     currentData.NetworkTotalDcInputPowerMilliWatts.second / 1000.0);
+    PUBLISH_OPT(MpptTemperatureMilliCelsius,              "MpptTemperature",              currentData.MpptTemperatureMilliCelsius.second / 1000.0);
+    PUBLISH_OPT(SmartBatterySenseTemperatureMilliCelsius, "SmartBatterySenseTemperature", currentData.SmartBatterySenseTemperatureMilliCelsius.second / 1000.0);
+#undef PUBLILSH_OPT
 }

src/PinMapping.cpp

@@ -8,8 +8,6 @@
 #include <LittleFS.h>
 #include <string.h>
 
-#define JSON_BUFFER_SIZE 6144
-
 #ifndef DISPLAY_TYPE
 #define DISPLAY_TYPE 0U
 #endif
@@ -94,6 +92,14 @@
 #define VICTRON_PIN_RX -1
 #endif
 
+#ifndef VICTRON_PIN_TX2
+#define VICTRON_PIN_TX2 -1
+#endif
+
+#ifndef VICTRON_PIN_RX2
+#define VICTRON_PIN_RX2 -1
+#endif
+
 #ifndef BATTERY_PIN_RX
 #define BATTERY_PIN_RX -1
 #endif
@@ -234,7 +240,7 @@ bool PinMappingClass::init(const String& deviceMapping)
         return false;
     }
 
-    DynamicJsonDocument doc(JSON_BUFFER_SIZE);
+    JsonDocument doc;
     // Deserialize the JSON document
     DeserializationError error = deserializeJson(doc, f);
     if (error) {
@@ -284,8 +290,8 @@ bool PinMappingClass::init(const String& deviceMapping)
             // OpenDTU-OnBattery-specific pins below
             _pinMapping.victron_rx = doc[i]["victron"]["rx"] | VICTRON_PIN_RX;
             _pinMapping.victron_tx = doc[i]["victron"]["tx"] | VICTRON_PIN_TX;
-            _pinMapping.victron_rx2 = doc[i]["victron"]["rx2"] | VICTRON_PIN_RX;
+            _pinMapping.victron_rx2 = doc[i]["victron"]["rx2"] | VICTRON_PIN_RX2;
-            _pinMapping.victron_tx2 = doc[i]["victron"]["tx2"] | VICTRON_PIN_TX;
+            _pinMapping.victron_tx2 = doc[i]["victron"]["tx2"] | VICTRON_PIN_TX2;
 
             _pinMapping.battery_rx = doc[i]["battery"]["rx"] | BATTERY_PIN_RX;
             _pinMapping.battery_rxen = doc[i]["battery"]["rxen"] | BATTERY_PIN_RXEN;

src/PowerLimiter.cpp

@@ -3,6 +3,7 @@
  * Copyright (C) 2022 Thomas Basler and others
  */
 
+#include "Utils.h"
 #include "Battery.h"
 #include "PowerMeter.h"
 #include "PowerLimiter.h"
@@ -31,13 +32,11 @@ frozen::string const& PowerLimiterClass::getStatusText(PowerLimiterClass::Status
 {
     static const frozen::string missing = "programmer error: missing status text";
 
-    static const frozen::map<Status, frozen::string, 21> texts = {
+    static const frozen::map<Status, frozen::string, 19> texts = {
         { Status::Initializing, "initializing (should not see me)" },
         { Status::DisabledByConfig, "disabled by configuration" },
         { Status::DisabledByMqtt, "disabled by MQTT" },
         { Status::WaitingForValidTimestamp, "waiting for valid date and time to be available" },
-        { Status::PowerMeterDisabled, "no power meter is configured/enabled" },
-        { Status::PowerMeterTimeout, "power meter readings are outdated" },
         { Status::PowerMeterPending, "waiting for sufficiently recent power meter reading" },
         { Status::InverterInvalid, "invalid inverter selection/configuration" },
         { Status::InverterChanged, "target inverter changed" },
@@ -47,7 +46,7 @@ frozen::string const& PowerLimiterClass::getStatusText(PowerLimiterClass::Status
         { Status::InverterPowerCmdPending, "waiting for a start/stop/restart command to complete" },
         { Status::InverterDevInfoPending, "waiting for inverter device information to be available" },
         { Status::InverterStatsPending, "waiting for sufficiently recent inverter data" },
-        { Status::CalculatedLimitBelowMinLimit, "calculated limit is less than lower power limit" },
+        { Status::CalculatedLimitBelowMinLimit, "calculated limit is less than minimum power limit" },
         { Status::UnconditionalSolarPassthrough, "unconditionally passing through all solar power (MQTT override)" },
         { Status::NoVeDirect, "VE.Direct disabled, connection broken, or data outdated" },
         { Status::NoEnergy, "no energy source available to power the inverter from" },
@@ -82,8 +81,7 @@ void PowerLimiterClass::announceStatus(PowerLimiterClass::Status status)
 /**
  * returns true if the inverter state was changed or is about to change, i.e.,
  * if it is actually in need of a shutdown. returns false otherwise, i.e., the
- * inverter is already shut down and the inverter limit is set to the configured
+ * inverter is already shut down.
- * lower power limit.
  */
 bool PowerLimiterClass::shutdown(PowerLimiterClass::Status status)
 {
@@ -93,14 +91,6 @@ bool PowerLimiterClass::shutdown(PowerLimiterClass::Status status)
 
     _oTargetPowerState = false;
 
-    auto const& config = Configuration.get();
-    if ( (Status::PowerMeterTimeout == status ||
-          Status::CalculatedLimitBelowMinLimit == status)
-        && config.PowerLimiter.IsInverterSolarPowered) {
-      _oTargetPowerState = true;
-    }
-
-    _oTargetPowerLimitWatts = config.PowerLimiter.LowerPowerLimit;
     return updateInverter();
 }
 
@@ -184,29 +174,34 @@ void PowerLimiterClass::loop()
         return unconditionalSolarPassthrough(_inverter);
     }
 
-    // the normal mode of operation requires a valid
-    // power meter reading to calculate a power limit
-    if (!config.PowerMeter.Enabled) {
-        shutdown(Status::PowerMeterDisabled);
-        return;
-    }
-
-    if (millis() - PowerMeter.getLastPowerMeterUpdate() > (30 * 1000)) {
-        shutdown(Status::PowerMeterTimeout);
-        return;
-    }
-
     // concerns both power limits and start/stop/restart commands and is
     // only updated if a respective response was received from the inverter
     auto lastUpdateCmd = std::max(
             _inverter->SystemConfigPara()->getLastUpdateCommand(),
             _inverter->PowerCommand()->getLastUpdateCommand());
 
-    if (_inverter->Statistics()->getLastUpdate() <= lastUpdateCmd) {
+    // we need inverter stats younger than the last update command
+    if (_oInverterStatsMillis.has_value() && lastUpdateCmd > *_oInverterStatsMillis) {
+        _oInverterStatsMillis = std::nullopt;
+    }
+
+    if (!_oInverterStatsMillis.has_value()) {
+        auto lastStats = _inverter->Statistics()->getLastUpdate();
+        if (lastStats <= lastUpdateCmd) {
             return announceStatus(Status::InverterStatsPending);
         }
 
-    if (PowerMeter.getLastPowerMeterUpdate() <= lastUpdateCmd) {
+        _oInverterStatsMillis = lastStats;
+    }
+
+    // if the power meter is being used, i.e., if its data is valid, we want to
+    // wait for a new reading after adjusting the inverter limit. otherwise, we
+    // proceed as we will use a fallback limit independent of the power meter.
+    // the power meter reading is expected to be at most 2 seconds old when it
+    // arrives. this can be the case for readings provided by networked meter
+    // readers, where a packet needs to travel through the network for some
+    // time after the actual measurement was done by the reader.
+    if (PowerMeter.isDataValid() && PowerMeter.getLastPowerMeterUpdate() <= (*_oInverterStatsMillis + 2000)) {
         return announceStatus(Status::PowerMeterPending);
     }
 
@@ -352,7 +347,7 @@ int32_t PowerLimiterClass::inverterPowerDcToAc(std::shared_ptr<InverterAbstract>
     CONFIG_T& config = Configuration.get();
 
     float inverterEfficiencyPercent = inverter->Statistics()->getChannelFieldValue(
-        TYPE_AC, CH0, FLD_EFF);
+        TYPE_INV, CH0, FLD_EFF);
 
     // fall back to hoymiles peak efficiency as per datasheet if inverter
     // is currently not producing (efficiency is zero in that case)
@@ -369,15 +364,30 @@ int32_t PowerLimiterClass::inverterPowerDcToAc(std::shared_ptr<InverterAbstract>
  * can currently only be set using MQTT. in this mode of operation, the
  * inverter shall behave as if it was connected to the solar panels directly,
  * i.e., all solar power (and only solar power) is fed to the AC side,
- * independent from the power meter reading.
+ * independent from the power meter reading. if the inverter is actually
+ * already connected to solar modules rather than a battery, the upper power
+ * limit is set as the inverter limit.
  */
 void PowerLimiterClass::unconditionalSolarPassthrough(std::shared_ptr<InverterAbstract> inverter)
 {
+    if ((millis() - _lastCalculation) < _calculationBackoffMs) { return; }
+    _lastCalculation = millis();
+
+    auto const& config = Configuration.get();
+
+    if (config.PowerLimiter.IsInverterSolarPowered) {
+        _calculationBackoffMs = 10 * 1000;
+        setNewPowerLimit(inverter, config.PowerLimiter.UpperPowerLimit);
+        announceStatus(Status::UnconditionalSolarPassthrough);
+        return;
+    }
+
     if (!VictronMppt.isDataValid()) {
         shutdown(Status::NoVeDirect);
         return;
     }
 
+    _calculationBackoffMs = 1 * 1000;
     int32_t solarPower = VictronMppt.getPowerOutputWatts();
     setNewPowerLimit(inverter, inverterPowerDcToAc(inverter, solarPower));
     announceStatus(Status::UnconditionalSolarPassthrough);
@@ -403,13 +413,12 @@ uint8_t PowerLimiterClass::getPowerLimiterState() {
     return PL_UI_STATE_INACTIVE;
 }
 
-// Logic table
+// Logic table ("PowerMeter value" can be "base load setting" as a fallback)
-// | Case # | batteryPower | solarPower > 0 | useFullSolarPassthrough | Result                                                      |
+// | Case # | batteryPower | solarPower     | useFullSolarPassthrough | Resulting inverter limit                               |
-// | 1      | false        | false          | doesn't matter          | PL = 0                                                      |
+// | 1      | false        |  < 20 W        | doesn't matter          | 0 (inverter off)                                       |
-// | 2      | false        | true           | doesn't matter          | PL = Victron Power                                          |
+// | 2      | false        | >= 20 W        | doesn't matter          | min(PowerMeter value, solarPower)                      |
-// | 3      | true         | doesn't matter | false                   | PL = PowerMeter value (Battery can supply unlimited energy) |
+// | 3      | true         | doesn't matter | false                   | PowerMeter value (Battery can supply unlimited energy) |
-// | 4      | true         | false          | true                    | PL = PowerMeter value                                       |
+// | 4      | true         | fully passed   | true                    | max(PowerMeter value, solarPower)                      |
-// | 5      | true         | true           | true                    | PL = max(PowerMeter value, Victron Power)                   |
 
 bool PowerLimiterClass::calcPowerLimit(std::shared_ptr<InverterAbstract> inverter, int32_t solarPowerDC, bool batteryPower)
 {
@@ -418,6 +427,7 @@ bool PowerLimiterClass::calcPowerLimit(std::shared_ptr<InverterAbstract> inverte
                 (batteryPower?"allowed":"prevented"), solarPowerDC);
     }
 
+    // Case 1:
     if (solarPowerDC <= 0 && !batteryPower) {
         return shutdown(Status::NoEnergy);
     }
@@ -431,38 +441,52 @@ bool PowerLimiterClass::calcPowerLimit(std::shared_ptr<InverterAbstract> inverte
         return shutdown(Status::HuaweiPsu);
     }
 
-    auto powerMeter = static_cast<int32_t>(PowerMeter.getPowerTotal());
+    auto meterValid = PowerMeter.isDataValid();
 
+    auto meterValue = static_cast<int32_t>(PowerMeter.getPowerTotal());
+
+    // We don't use FLD_PAC from the statistics, because that data might be too
+    // old and unreliable. TODO(schlimmchen): is this comment outdated?
     auto inverterOutput = static_cast<int32_t>(inverter->Statistics()->getChannelFieldValue(TYPE_AC, CH0, FLD_PAC));
 
     auto solarPowerAC = inverterPowerDcToAc(inverter, solarPowerDC);
 
     auto const& config = Configuration.get();
+    auto targetConsumption = config.PowerLimiter.TargetPowerConsumption;
+    auto baseLoad = config.PowerLimiter.BaseLoadLimit;
+    bool meterIncludesInv = config.PowerLimiter.IsInverterBehindPowerMeter;
 
     if (_verboseLogging) {
-        MessageOutput.printf("[DPL::calcPowerLimit] power meter: %d W, "
+        MessageOutput.printf("[DPL::calcPowerLimit] target consumption: %d W, "
-                "target consumption: %d W, inverter output: %d W, solar power (AC): %d\r\n",
+                "base load: %d W, power meter does %sinclude inverter output\r\n",
-                powerMeter,
+                targetConsumption,
-                config.PowerLimiter.TargetPowerConsumption,
+                baseLoad,
+                (meterIncludesInv?"":"NOT "));
+
+        MessageOutput.printf("[DPL::calcPowerLimit] power meter value: %d W, "
+                "power meter valid: %s, inverter output: %d W, solar power (AC): %d W\r\n",
+                meterValue,
+                (meterValid?"yes":"no"),
                 inverterOutput,
                 solarPowerAC);
     }
 
-    auto newPowerLimit = powerMeter;
+    auto newPowerLimit = baseLoad;
 
-    if (config.PowerLimiter.IsInverterBehindPowerMeter) {
+    if (meterValid) {
-        // If the inverter the behind the power meter (part of measurement),
+        newPowerLimit = meterValue;
-        // the produced power of this inverter has also to be taken into account.
+
-        // We don't use FLD_PAC from the statistics, because that
+        if (meterIncludesInv) {
-        // data might be too old and unreliable.
+            // If the inverter is wired behind the power meter, i.e., if its
+            // output is part of the power meter measurement, the produced
+            // power of this inverter has to be taken into account.
             newPowerLimit += inverterOutput;
         }
 
-    // We're not trying to hit 0 exactly but take an offset into account
+        newPowerLimit -= targetConsumption;
-    // This means we never fully compensate the used power with the inverter 
+    }
-    // Case 3
-    newPowerLimit -= config.PowerLimiter.TargetPowerConsumption;
 
+    // Case 2:
     if (!batteryPower) {
         newPowerLimit = std::min(newPowerLimit, solarPowerAC);
 
@@ -476,6 +500,7 @@ bool PowerLimiterClass::calcPowerLimit(std::shared_ptr<InverterAbstract> inverte
         return setNewPowerLimit(inverter, newPowerLimit);
     }
 
+    // Case 4:
     // convert all solar power if full solar-passthrough is active
     if (useFullSolarPassthrough()) {
         newPowerLimit = std::max(newPowerLimit, solarPowerAC);
@@ -489,10 +514,11 @@ bool PowerLimiterClass::calcPowerLimit(std::shared_ptr<InverterAbstract> inverte
     }
 
     if (_verboseLogging) {
-        MessageOutput.printf("[DPL::calcPowerLimit] match power meter with limit of %d W\r\n",
+        MessageOutput.printf("[DPL::calcPowerLimit] match household consumption with limit of %d W\r\n",
             newPowerLimit);
     }
 
+    // Case 3:
     return setNewPowerLimit(inverter, newPowerLimit);
 }
 
@@ -512,15 +538,39 @@ bool PowerLimiterClass::updateInverter()
 
     if (nullptr == _inverter) { return reset(); }
 
+    // do not reset _inverterUpdateTimeouts below if no state change requested
+    if (!_oTargetPowerState.has_value() && !_oTargetPowerLimitWatts.has_value()) {
+        return reset();
+    }
+
     if (!_oUpdateStartMillis.has_value()) {
         _oUpdateStartMillis = millis();
     }
 
     if ((millis() - *_oUpdateStartMillis) > 30 * 1000) {
-        MessageOutput.printf("[DPL::updateInverter] timeout, "
+        ++_inverterUpdateTimeouts;
+        MessageOutput.printf("[DPL::updateInverter] timeout (%d in succession), "
                 "state transition pending: %s, limit pending: %s\r\n",
+                _inverterUpdateTimeouts,
                 (_oTargetPowerState.has_value()?"yes":"no"),
                 (_oTargetPowerLimitWatts.has_value()?"yes":"no"));
+
+        // NOTE that this is not always 5 minutes, since this counts timeouts,
+        // not absolute time. after any timeout, an update cycle ends. a new
+        // timeout can only happen after starting a new update cycle, which in
+        // turn is only started if the DPL did calculate a new limit, which in
+        // turn does not happen while the inverter is unreachable, no matter
+        // how long (a whole night) that might be.
+        if (_inverterUpdateTimeouts >= 10) {
+            MessageOutput.println("[DPL::loop] issuing inverter restart command after update timed out repeatedly");
+            _inverter->sendRestartControlRequest();
+        }
+
+        if (_inverterUpdateTimeouts >= 20) {
+            MessageOutput.println("[DPL::loop] restarting system since inverter is unresponsive");
+            Utils::restartDtu();
+        }
+
         return reset();
     }
 
@@ -623,6 +673,8 @@ bool PowerLimiterClass::updateInverter()
     // enable power production only after setting the desired limit
     if (switchPowerState(true)) { return true; }
 
+    _inverterUpdateTimeouts = 0;
+
     return reset();
 }
 
@@ -691,14 +743,20 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr<InverterAbstract> inver
 
     if (_verboseLogging) {
         MessageOutput.printf("[DPL::setNewPowerLimit] input limit: %d W, "
-                "lower limit: %d W, upper limit: %d W, hysteresis: %d W\r\n",
+                "min limit: %d W, max limit: %d W, hysteresis: %d W\r\n",
                 newPowerLimit, lowerLimit, upperLimit, hysteresis);
     }
 
     if (newPowerLimit < lowerLimit) {
+        if (!config.PowerLimiter.IsInverterSolarPowered) {
             return shutdown(Status::CalculatedLimitBelowMinLimit);
         }
 
+        MessageOutput.println("[DPL::setNewPowerLimit] keep solar-powered "
+                "inverter running at min limit");
+        newPowerLimit = lowerLimit;
+    }
+
     // enforce configured upper power limit
     int32_t effPowerLimit = std::min(newPowerLimit, upperLimit);
 

src/PowerMeter.cpp

@@ -136,6 +136,23 @@ uint32_t PowerMeterClass::getLastPowerMeterUpdate()
     return _lastPowerMeterUpdate;
 }
 
+bool PowerMeterClass::isDataValid()
+{
+    auto const& config = Configuration.get();
+
+    std::lock_guard<std::mutex> l(_mutex);
+
+    bool valid = config.PowerMeter.Enabled &&
+        _lastPowerMeterUpdate > 0 &&
+        ((millis() - _lastPowerMeterUpdate) < (30 * 1000));
+
+    // reset if timed out to avoid glitch once
+    // (millis() - _lastPowerMeterUpdate) overflows
+    if (!valid) { _lastPowerMeterUpdate = 0; }
+
+    return valid;
+}
+
 void PowerMeterClass::mqtt()
 {
     if (!MqttSettings.getConnected()) { return; }

src/Utils.cpp

@@ -69,9 +69,9 @@ void Utils::restartDtu()
     ESP.restart();
 }
 
-bool Utils::checkJsonAlloc(const DynamicJsonDocument& doc, const char* function, const uint16_t line)
+bool Utils::checkJsonAlloc(const JsonDocument& doc, const char* function, const uint16_t line)
 {
-    if (doc.capacity() == 0) {
+    if (doc.overflowed()) {
         MessageOutput.printf("Alloc failed: %s, %d\r\n", function, line);
         return false;
     }
@@ -79,16 +79,6 @@ bool Utils::checkJsonAlloc(const DynamicJsonDocument& doc, const char* function,
     return true;
 }
 
-bool Utils::checkJsonOverflow(const DynamicJsonDocument& doc, const char* function, const uint16_t line)
-{
-    if (doc.overflowed()) {
-        MessageOutput.printf("DynamicJsonDocument overflowed: %s, %d\r\n", function, line);
-        return true;
-    }
-
-    return false;
-}
-
 /// @brief Remove all files but the PINMAPPING_FILENAME
 void Utils::removeAllFiles()
 {

src/VictronMppt.cpp

@@ -119,7 +119,7 @@ uint32_t VictronMpptClass::getDataAgeMillis(size_t idx) const
     return millis() - _controllers[idx]->getLastUpdate();
 }
 
-std::optional<VeDirectMpptController::spData_t> VictronMpptClass::getData(size_t idx) const
+std::optional<VeDirectMpptController::data_t> VictronMpptClass::getData(size_t idx) const
 {
     std::lock_guard<std::mutex> lock(_mutex);
 
@@ -129,7 +129,9 @@ std::optional<VeDirectMpptController::spData_t> VictronMpptClass::getData(size_t
         return std::nullopt;
     }
 
-    return std::optional<VeDirectMpptController::spData_t>{_controllers[idx]->getData()};
+    if (!_controllers[idx]->isDataValid()) { return std::nullopt; }
+
+    return _controllers[idx]->getData();
 }
 
 int32_t VictronMpptClass::getPowerOutputWatts() const
@@ -138,7 +140,18 @@ int32_t VictronMpptClass::getPowerOutputWatts() const
 
     for (const auto& upController : _controllers) {
         if (!upController->isDataValid()) { continue; }
-        sum += upController->getData()->P;
+
+        // if any charge controller is part of a VE.Smart network, and if the
+        // charge controller is connected in a way that allows to send
+        // requests, we should have the "network total DC input power"
+        // available. if so, to estimate the output power, we multiply by
+        // the calculated efficiency of the connected charge controller.
+        auto networkPower = upController->getData().NetworkTotalDcInputPowerMilliWatts;
+        if (networkPower.first > 0) {
+            return static_cast<int32_t>(networkPower.second / 1000.0 * upController->getData().mpptEfficiency_Percent / 100);
+        }
+
+        sum += upController->getData().batteryOutputPower_W;
     }
 
     return sum;
@@ -150,43 +163,52 @@ int32_t VictronMpptClass::getPanelPowerWatts() const
 
     for (const auto& upController : _controllers) {
         if (!upController->isDataValid()) { continue; }
-        sum += upController->getData()->PPV;
+
+        // if any charge controller is part of a VE.Smart network, and if the
+        // charge controller is connected in a way that allows to send
+        // requests, we should have the "network total DC input power" available.
+        auto networkPower = upController->getData().NetworkTotalDcInputPowerMilliWatts;
+        if (networkPower.first > 0) {
+            return static_cast<int32_t>(networkPower.second / 1000.0);
+        }
+
+        sum += upController->getData().panelPower_PPV_W;
     }
 
     return sum;
 }
 
-double VictronMpptClass::getYieldTotal() const
+float VictronMpptClass::getYieldTotal() const
 {
-    double sum = 0;
+    float sum = 0;
 
     for (const auto& upController : _controllers) {
         if (!upController->isDataValid()) { continue; }
-        sum += upController->getData()->H19;
+        sum += upController->getData().yieldTotal_H19_Wh / 1000.0;
     }
 
     return sum;
 }
 
-double VictronMpptClass::getYieldDay() const
+float VictronMpptClass::getYieldDay() const
 {
-    double sum = 0;
+    float sum = 0;
 
     for (const auto& upController : _controllers) {
         if (!upController->isDataValid()) { continue; }
-        sum += upController->getData()->H20;
+        sum += upController->getData().yieldToday_H20_Wh / 1000.0;
     }
 
     return sum;
 }
 
-double VictronMpptClass::getOutputVoltage() const
+float VictronMpptClass::getOutputVoltage() const
 {
-    double min = -1;
+    float min = -1;
 
     for (const auto& upController : _controllers) {
         if (!upController->isDataValid()) { continue; }
-        double volts = upController->getData()->V;
+        float volts = upController->getData().batteryVoltage_V_mV / 1000.0;
         if (min == -1) { min = volts; }
         min = std::min(min, volts);
     }

src/VictronSmartShunt.cpp

@@ -31,6 +31,6 @@ void VictronSmartShunt::loop()
 
     if (VeDirectShunt.getLastUpdate() <= _lastUpdate) { return; }
 
-    _stats->updateFrom(VeDirectShunt.veFrame);
+    _stats->updateFrom(VeDirectShunt.getData());
     _lastUpdate = VeDirectShunt.getLastUpdate();
 }

src/WebApi.cpp

@@ -4,6 +4,7 @@
  */
 #include "WebApi.h"
 #include "Configuration.h"
+#include "MessageOutput.h"
 #include "defaults.h"
 #include <AsyncJson.h>
 
@@ -93,4 +94,58 @@ void WebApiClass::writeConfig(JsonVariant& retMsg, const WebApiError code, const
     }
 }
 
+bool WebApiClass::parseRequestData(AsyncWebServerRequest* request, AsyncJsonResponse* response, JsonDocument& json_document)
+{
+    auto& retMsg = response->getRoot();
+    retMsg["type"] = "warning";
+
+    if (!request->hasParam("data", true)) {
+        retMsg["message"] = "No values found!";
+        retMsg["code"] = WebApiError::GenericNoValueFound;
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
+        return false;
+    }
+
+    const String json = request->getParam("data", true)->value();
+    const DeserializationError error = deserializeJson(json_document, json);
+    if (error) {
+        retMsg["message"] = "Failed to parse data!";
+        retMsg["code"] = WebApiError::GenericParseError;
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
+        return false;
+    }
+
+    return true;
+}
+
+uint64_t WebApiClass::parseSerialFromRequest(AsyncWebServerRequest* request, String param_name)
+{
+    if (request->hasParam(param_name)) {
+        String s = request->getParam(param_name)->value();
+        return strtoll(s.c_str(), NULL, 16);
+    }
+
+    return 0;
+}
+
+bool WebApiClass::sendJsonResponse(AsyncWebServerRequest* request, AsyncJsonResponse* response, const char* function, const uint16_t line)
+{
+    bool ret_val = true;
+    if (response->overflowed()) {
+        auto& root = response->getRoot();
+
+        root.clear();
+        root["message"] = String("500 Internal Server Error: ") + function + ", " + line;
+        root["code"] = WebApiError::GenericInternalServerError;
+        root["type"] = "danger";
+        response->setCode(500);
+        MessageOutput.printf("WebResponse failed: %s, %d\r\n", function, line);
+        ret_val = false;
+    }
+
+    response->setLength();
+    request->send(response);
+    return ret_val;
+}
+
 WebApiClass WebApi;

src/WebApi_Huawei.cpp

@@ -72,40 +72,16 @@ void WebApiHuaweiClass::onPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
-    auto& retMsg = response->getRoot();
+    JsonDocument root;
-    retMsg["type"] = "warning";
+    if (!WebApi.parseRequestData(request, response, root)) {
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
         return;
     }
 
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    DeserializationError error = deserializeJson(root, json);
     float value;
     uint8_t online = true;
     float minimal_voltage;
 
-    if (error) {
+    auto& retMsg = response->getRoot();
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (root.containsKey("online")) {
         online = root["online"].as<bool>();
@@ -164,12 +140,9 @@ void WebApiHuaweiClass::onPost(AsyncWebServerRequest* request)
         }
     }
 
-    retMsg["type"] = "success";
+    WebApi.writeConfig(retMsg);
-    retMsg["message"] = "Settings saved!";
-    retMsg["code"] = WebApiError::GenericSuccess;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 
@@ -186,12 +159,17 @@ void WebApiHuaweiClass::onAdminGet(AsyncWebServerRequest* request)
     const CONFIG_T& config = Configuration.get();
 
     root["enabled"] = config.Huawei.Enabled;
+    root["verbose_logging"] = config.Huawei.VerboseLogging;
     root["can_controller_frequency"] = config.Huawei.CAN_Controller_Frequency;
     root["auto_power_enabled"] = config.Huawei.Auto_Power_Enabled;
+    root["auto_power_batterysoc_limits_enabled"] = config.Huawei.Auto_Power_BatterySoC_Limits_Enabled;
+    root["emergency_charge_enabled"] = config.Huawei.Emergency_Charge_Enabled;
     root["voltage_limit"] = static_cast<int>(config.Huawei.Auto_Power_Voltage_Limit * 100) / 100.0;
     root["enable_voltage_limit"] = static_cast<int>(config.Huawei.Auto_Power_Enable_Voltage_Limit * 100) / 100.0;
     root["lower_power_limit"] = config.Huawei.Auto_Power_Lower_Power_Limit;
     root["upper_power_limit"] = config.Huawei.Auto_Power_Upper_Power_Limit;   
+    root["stop_batterysoc_threshold"] = config.Huawei.Auto_Power_Stop_BatterySoC_Threshold;
+    root["target_power_consumption"] = config.Huawei.Auto_Power_Target_Power_Consumption;
 
     response->setLength();
     request->send(response);
@@ -204,41 +182,17 @@ void WebApiHuaweiClass::onAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("enabled")) ||
         !(root.containsKey("can_controller_frequency")) ||
         !(root.containsKey("auto_power_enabled")) ||
+        !(root.containsKey("emergency_charge_enabled")) ||
         !(root.containsKey("voltage_limit")) ||
         !(root.containsKey("lower_power_limit")) ||
         !(root.containsKey("upper_power_limit"))) {
@@ -251,17 +205,21 @@ void WebApiHuaweiClass::onAdminPost(AsyncWebServerRequest* request)
 
     CONFIG_T& config = Configuration.get();
     config.Huawei.Enabled = root["enabled"].as<bool>();
+    config.Huawei.VerboseLogging = root["verbose_logging"];
     config.Huawei.CAN_Controller_Frequency = root["can_controller_frequency"].as<uint32_t>();
     config.Huawei.Auto_Power_Enabled = root["auto_power_enabled"].as<bool>();
+    config.Huawei.Auto_Power_BatterySoC_Limits_Enabled = root["auto_power_batterysoc_limits_enabled"].as<bool>();
+    config.Huawei.Emergency_Charge_Enabled = root["emergency_charge_enabled"].as<bool>();
     config.Huawei.Auto_Power_Voltage_Limit = root["voltage_limit"].as<float>();
     config.Huawei.Auto_Power_Enable_Voltage_Limit = root["enable_voltage_limit"].as<float>();
     config.Huawei.Auto_Power_Lower_Power_Limit = root["lower_power_limit"].as<float>();
     config.Huawei.Auto_Power_Upper_Power_Limit = root["upper_power_limit"].as<float>();    
+    config.Huawei.Auto_Power_Stop_BatterySoC_Threshold = root["stop_batterysoc_threshold"];
+    config.Huawei.Auto_Power_Target_Power_Consumption = root["target_power_consumption"];
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     // TODO(schlimmchen): HuaweiCan has no real concept of the fact that the
     // config might change. at least not regarding CAN parameters. until that

src/WebApi_battery.cpp

@@ -59,43 +59,17 @@ void WebApiBatteryClass::onAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!root.containsKey("enabled") || !root.containsKey("provider")) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -110,8 +84,7 @@ void WebApiBatteryClass::onAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     Battery.updateSettings();
     MqttHandleBatteryHass.forceUpdate();

src/WebApi_config.cpp

@@ -40,6 +40,7 @@ void WebApiConfigClass::onConfigGet(AsyncWebServerRequest* request)
             requestFile = name;
         } else {
             request->send(404);
+            return;
         }
     }
 
@@ -53,51 +54,24 @@ void WebApiConfigClass::onConfigDelete(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("delete"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["delete"].as<bool>() == false) {
         retMsg["message"] = "Not deleted anything!";
         retMsg["code"] = WebApiError::ConfigNotDeleted;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -105,8 +79,7 @@ void WebApiConfigClass::onConfigDelete(AsyncWebServerRequest* request)
     retMsg["message"] = "Configuration resettet. Rebooting now...";
     retMsg["code"] = WebApiError::ConfigSuccess;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     Utils::removeAllFiles();
     Utils::restartDtu();
@@ -120,7 +93,7 @@ void WebApiConfigClass::onConfigListGet(AsyncWebServerRequest* request)
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-    auto data = root.createNestedArray("configs");
+    auto data = root["configs"].to<JsonArray>();
 
     File rootfs = LittleFS.open("/");
     File file = rootfs.openNextFile();
@@ -128,15 +101,14 @@ void WebApiConfigClass::onConfigListGet(AsyncWebServerRequest* request)
         if (file.isDirectory()) {
             continue;
         }
-        JsonObject obj = data.createNestedObject();
+        JsonObject obj = data.add<JsonObject>();
         obj["name"] = String(file.name());
 
         file = rootfs.openNextFile();
     }
     file.close();
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiConfigClass::onConfigUploadFinish(AsyncWebServerRequest* request)

src/WebApi_device.cpp

@@ -26,15 +26,15 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
     const CONFIG_T& config = Configuration.get();
     const PinMapping_t& pin = PinMapping.get();
 
-    auto curPin = root.createNestedObject("curPin");
+    auto curPin = root["curPin"].to<JsonObject>();
     curPin["name"] = config.Dev_PinMapping;
 
-    auto nrfPinObj = curPin.createNestedObject("nrf24");
+    auto nrfPinObj = curPin["nrf24"].to<JsonObject>();
     nrfPinObj["clk"] = pin.nrf24_clk;
     nrfPinObj["cs"] = pin.nrf24_cs;
     nrfPinObj["en"] = pin.nrf24_en;
@@ -42,7 +42,7 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
     nrfPinObj["miso"] = pin.nrf24_miso;
     nrfPinObj["mosi"] = pin.nrf24_mosi;
 
-    auto cmtPinObj = curPin.createNestedObject("cmt");
+    auto cmtPinObj = curPin["cmt"].to<JsonObject>();
     cmtPinObj["clk"] = pin.cmt_clk;
     cmtPinObj["cs"] = pin.cmt_cs;
     cmtPinObj["fcs"] = pin.cmt_fcs;
@@ -50,7 +50,7 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
     cmtPinObj["gpio2"] = pin.cmt_gpio2;
     cmtPinObj["gpio3"] = pin.cmt_gpio3;
 
-    auto ethPinObj = curPin.createNestedObject("eth");
+    auto ethPinObj = curPin["eth"].to<JsonObject>();
     ethPinObj["enabled"] = pin.eth_enabled;
     ethPinObj["phy_addr"] = pin.eth_phy_addr;
     ethPinObj["power"] = pin.eth_power;
@@ -59,19 +59,19 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
     ethPinObj["type"] = pin.eth_type;
     ethPinObj["clk_mode"] = pin.eth_clk_mode;
 
-    auto displayPinObj = curPin.createNestedObject("display");
+    auto displayPinObj = curPin["display"].to<JsonObject>();
     displayPinObj["type"] = pin.display_type;
     displayPinObj["data"] = pin.display_data;
     displayPinObj["clk"] = pin.display_clk;
     displayPinObj["cs"] = pin.display_cs;
     displayPinObj["reset"] = pin.display_reset;
 
-    auto ledPinObj = curPin.createNestedObject("led");
+    auto ledPinObj = curPin["led"].to<JsonObject>();
     for (uint8_t i = 0; i < PINMAPPING_LED_COUNT; i++) {
         ledPinObj["led" + String(i)] = pin.led[i];
     }
 
-    auto display = root.createNestedObject("display");
+    auto display = root["display"].to<JsonObject>();
     display["rotation"] = config.Display.Rotation;
     display["power_safe"] = config.Display.PowerSafe;
     display["screensaver"] = config.Display.ScreenSaver;
@@ -80,25 +80,25 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
     display["diagramduration"] = config.Display.Diagram.Duration;
     display["diagrammode"] = config.Display.Diagram.Mode;
 
-    auto leds = root.createNestedArray("led");
+    auto leds = root["led"].to<JsonArray>();
     for (uint8_t i = 0; i < PINMAPPING_LED_COUNT; i++) {
-        auto led = leds.createNestedObject();
+        auto led = leds.add<JsonObject>();
         led["brightness"] = config.Led_Single[i].Brightness;
     }
 
-    auto victronPinObj = curPin.createNestedObject("victron");
+    auto victronPinObj = curPin["victron"].to<JsonObject>();
     victronPinObj["rx"] = pin.victron_rx;
     victronPinObj["tx"] = pin.victron_tx;
     victronPinObj["rx2"] = pin.victron_rx2;
     victronPinObj["tx2"] = pin.victron_tx2;
 
-    JsonObject batteryPinObj = curPin.createNestedObject("battery");
+    auto batteryPinObj = curPin["battery"].to<JsonObject>();
     batteryPinObj["rx"] = pin.battery_rx;
     batteryPinObj["rxen"] = pin.battery_rxen;
     batteryPinObj["tx"] = pin.battery_tx;
     batteryPinObj["txen"] = pin.battery_txen;
 
-    JsonObject huaweiPinObj = curPin.createNestedObject("huawei");
+    auto huaweiPinObj = curPin["huawei"].to<JsonObject>();
     huaweiPinObj["miso"] = pin.huawei_miso;
     huaweiPinObj["mosi"] = pin.huawei_mosi;
     huaweiPinObj["clk"] = pin.huawei_clk;
@@ -106,8 +106,7 @@ void WebApiDeviceClass::onDeviceAdminGet(AsyncWebServerRequest* request)
     huaweiPinObj["cs"] = pin.huawei_cs;
     huaweiPinObj["power"] = pin.huawei_power;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiDeviceClass::onDeviceAdminPost(AsyncWebServerRequest* request)
@@ -116,45 +115,19 @@ void WebApiDeviceClass::onDeviceAdminPost(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > MQTT_JSON_DOC_SIZE) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(MQTT_JSON_DOC_SIZE);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("curPin")
             || root.containsKey("display"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -162,8 +135,7 @@ void WebApiDeviceClass::onDeviceAdminPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Pin mapping must between 1 and " STR(DEV_MAX_MAPPING_NAME_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::HardwarePinMappingLength;
         retMsg["param"]["max"] = DEV_MAX_MAPPING_NAME_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -194,8 +166,7 @@ void WebApiDeviceClass::onDeviceAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     if (performRestart) {
         Utils::restartDtu();

src/WebApi_devinfo.cpp

@@ -23,13 +23,7 @@ void WebApiDevInfoClass::onDevInfoStatus(AsyncWebServerRequest* request)
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-
+    auto serial = WebApi.parseSerialFromRequest(request);
-    uint64_t serial = 0;
-    if (request->hasParam("inv")) {
-        String s = request->getParam("inv")->value();
-        serial = strtoll(s.c_str(), NULL, 16);
-    }
-
     auto inv = Hoymiles.getInverterBySerial(serial);
 
     if (inv != nullptr) {
@@ -43,6 +37,5 @@ void WebApiDevInfoClass::onDevInfoStatus(AsyncWebServerRequest* request)
         root["fw_build_datetime"] = inv->DevInfo()->getFwBuildDateTimeStr();
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_dtu.cpp

@@ -63,10 +63,10 @@ void WebApiDtuClass::onDtuAdminGet(AsyncWebServerRequest* request)
     root["cmt_country"] = config.Dtu.Cmt.CountryMode;
     root["cmt_chan_width"] = Hoymiles.getRadioCmt()->getChannelWidth();
 
-    auto data = root.createNestedArray("country_def");
+    auto data = root["country_def"].to<JsonArray>();
     auto countryDefs = Hoymiles.getRadioCmt()->getCountryFrequencyList();
     for (const auto& definition : countryDefs) {
-        auto obj = data.createNestedObject();
+        auto obj = data.add<JsonObject>();
         obj["freq_default"] = definition.definition.Freq_Default;
         obj["freq_min"] = definition.definition.Freq_Min;
         obj["freq_max"] = definition.definition.Freq_Max;
@@ -74,8 +74,7 @@ void WebApiDtuClass::onDtuAdminGet(AsyncWebServerRequest* request)
         obj["freq_legal_max"] = definition.definition.Freq_Legal_Max;
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
@@ -85,37 +84,12 @@ void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("serial") 
             && root.containsKey("pollinterval")
@@ -126,8 +100,7 @@ void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
             && root.containsKey("cmt_country"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -137,40 +110,35 @@ void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
     if (serial == 0) {
         retMsg["message"] = "Serial cannot be zero!";
         retMsg["code"] = WebApiError::DtuSerialZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["pollinterval"].as<uint32_t>() == 0) {
         retMsg["message"] = "Poll interval must be greater zero!";
         retMsg["code"] = WebApiError::DtuPollZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["nrf_palevel"].as<uint8_t>() > 3) {
         retMsg["message"] = "Invalid power level setting!";
         retMsg["code"] = WebApiError::DtuInvalidPowerLevel;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["cmt_palevel"].as<int8_t>() < -10 || root["cmt_palevel"].as<int8_t>() > 20) {
         retMsg["message"] = "Invalid power level setting!";
         retMsg["code"] = WebApiError::DtuInvalidPowerLevel;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["cmt_country"].as<uint8_t>() >= CountryModeId_t::CountryModeId_Max) {
         retMsg["message"] = "Invalid country setting!";
         retMsg["code"] = WebApiError::DtuInvalidCmtCountry;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -183,8 +151,7 @@ void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::DtuInvalidCmtFrequency;
         retMsg["param"]["min"] = FrequencyDefinition.Freq_Min;
         retMsg["param"]["max"] = FrequencyDefinition.Freq_Max;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -200,8 +167,8 @@ void WebApiDtuClass::onDtuAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     _applyDataTask.enable();
+    _applyDataTask.restart();
 }

src/WebApi_eventlog.cpp

@@ -20,14 +20,9 @@ void WebApiEventlogClass::onEventlogStatus(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 2048);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-
+    auto serial = WebApi.parseSerialFromRequest(request);
-    uint64_t serial = 0;
-    if (request->hasParam("inv")) {
-        String s = request->getParam("inv")->value();
-        serial = strtoll(s.c_str(), NULL, 16);
-    }
 
     AlarmMessageLocale_t locale = AlarmMessageLocale_t::EN;
     if (request->hasParam("locale")) {
@@ -47,10 +42,10 @@ void WebApiEventlogClass::onEventlogStatus(AsyncWebServerRequest* request)
         uint8_t logEntryCount = inv->EventLog()->getEntryCount();
 
         root["count"] = logEntryCount;
-        JsonArray eventsArray = root.createNestedArray("events");
+        JsonArray eventsArray = root["events"].to<JsonArray>();
 
         for (uint8_t logEntry = 0; logEntry < logEntryCount; logEntry++) {
-            JsonObject eventsObject = eventsArray.createNestedObject();
+            JsonObject eventsObject = eventsArray.add<JsonObject>();
 
             AlarmLogEntry_t entry;
             inv->EventLog()->getLogEntry(logEntry, entry, locale);
@@ -62,6 +57,5 @@ void WebApiEventlogClass::onEventlogStatus(AsyncWebServerRequest* request)
         }
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_gridprofile.cpp

@@ -21,32 +21,26 @@ void WebApiGridProfileClass::onGridProfileStatus(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 8192);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-
+    auto serial = WebApi.parseSerialFromRequest(request);
-    uint64_t serial = 0;
-    if (request->hasParam("inv")) {
-        String s = request->getParam("inv")->value();
-        serial = strtoll(s.c_str(), NULL, 16);
-    }
-
     auto inv = Hoymiles.getInverterBySerial(serial);
 
     if (inv != nullptr) {
         root["name"] = inv->GridProfile()->getProfileName();
         root["version"] = inv->GridProfile()->getProfileVersion();
 
-        auto jsonSections = root.createNestedArray("sections");
+        auto jsonSections = root["sections"].to<JsonArray>();
         auto profSections = inv->GridProfile()->getProfile();
 
         for (auto &profSection : profSections) {
-            auto jsonSection = jsonSections.createNestedObject();
+            auto jsonSection = jsonSections.add<JsonObject>();
             jsonSection["name"] = profSection.SectionName;
 
-            auto jsonItems = jsonSection.createNestedArray("items");
+            auto jsonItems = jsonSection["items"].to<JsonArray>();
 
             for (auto &profItem : profSection.items) {
-                auto jsonItem = jsonItems.createNestedObject();
+                auto jsonItem = jsonItems.add<JsonObject>();
 
                 jsonItem["n"] = profItem.Name;
                 jsonItem["u"] = profItem.Unit;
@@ -55,8 +49,7 @@ void WebApiGridProfileClass::onGridProfileStatus(AsyncWebServerRequest* request)
         }
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiGridProfileClass::onGridProfileRawdata(AsyncWebServerRequest* request)
@@ -65,24 +58,17 @@ void WebApiGridProfileClass::onGridProfileRawdata(AsyncWebServerRequest* request
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 4096);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-
+    auto serial = WebApi.parseSerialFromRequest(request);
-    uint64_t serial = 0;
-    if (request->hasParam("inv")) {
-        String s = request->getParam("inv")->value();
-        serial = strtoll(s.c_str(), NULL, 16);
-    }
-
     auto inv = Hoymiles.getInverterBySerial(serial);
 
     if (inv != nullptr) {
-        auto raw = root.createNestedArray("raw");
+        auto raw = root["raw"].to<JsonArray>();
         auto data = inv->GridProfile()->getRawData();
 
         copyArray(&data[0], data.size(), raw);
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_inverter.cpp

@@ -29,15 +29,15 @@ void WebApiInverterClass::onInverterList(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 768 * INV_MAX_COUNT);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
-    JsonArray data = root.createNestedArray("inverter");
+    JsonArray data = root["inverter"].to<JsonArray>();
 
     const CONFIG_T& config = Configuration.get();
 
     for (uint8_t i = 0; i < INV_MAX_COUNT; i++) {
         if (config.Inverter[i].Serial > 0) {
-            JsonObject obj = data.createNestedObject();
+            JsonObject obj = data.add<JsonObject>();
             obj["id"] = i;
             obj["name"] = String(config.Inverter[i].Name);
             obj["order"] = config.Inverter[i].Order;
@@ -67,9 +67,9 @@ void WebApiInverterClass::onInverterList(AsyncWebServerRequest* request)
                 max_channels = inv->Statistics()->getChannelsByType(TYPE_DC).size();
             }
 
-            JsonArray channel = obj.createNestedArray("channel");
+            JsonArray channel = obj["channel"].to<JsonArray>();
             for (uint8_t c = 0; c < max_channels; c++) {
-                JsonObject chanData = channel.createNestedObject();
+                JsonObject chanData = channel.add<JsonObject>();
                 chanData["name"] = config.Inverter[i].channel[c].Name;
                 chanData["max_power"] = config.Inverter[i].channel[c].MaxChannelPower;
                 chanData["yield_total_offset"] = config.Inverter[i].channel[c].YieldTotalOffset;
@@ -77,8 +77,7 @@ void WebApiInverterClass::onInverterList(AsyncWebServerRequest* request)
         }
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
@@ -88,44 +87,18 @@ void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("serial")
             && root.containsKey("name"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -135,8 +108,7 @@ void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
     if (serial == 0) {
         retMsg["message"] = "Serial must be a number > 0!";
         retMsg["code"] = WebApiError::InverterSerialZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -144,8 +116,7 @@ void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
         retMsg["message"] = "Name must between 1 and " STR(INV_MAX_NAME_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::InverterNameLength;
         retMsg["param"]["max"] = INV_MAX_NAME_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -155,8 +126,7 @@ void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
         retMsg["message"] = "Only " STR(INV_MAX_COUNT) " inverters are supported!";
         retMsg["code"] = WebApiError::InverterCount;
         retMsg["param"]["max"] = INV_MAX_COUNT;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -167,8 +137,7 @@ void WebApiInverterClass::onInverterAdd(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg, WebApiError::InverterAdded, "Inverter created!");
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     auto inv = Hoymiles.addInverter(inverter->Name, inverter->Serial);
 
@@ -188,51 +157,24 @@ void WebApiInverterClass::onInverterEdit(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("id") && root.containsKey("serial") && root.containsKey("name") && root.containsKey("channel"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["id"].as<uint8_t>() > INV_MAX_COUNT - 1) {
         retMsg["message"] = "Invalid ID specified!";
         retMsg["code"] = WebApiError::InverterInvalidId;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -242,8 +184,7 @@ void WebApiInverterClass::onInverterEdit(AsyncWebServerRequest* request)
     if (serial == 0) {
         retMsg["message"] = "Serial must be a number > 0!";
         retMsg["code"] = WebApiError::InverterSerialZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -251,8 +192,7 @@ void WebApiInverterClass::onInverterEdit(AsyncWebServerRequest* request)
         retMsg["message"] = "Name must between 1 and " STR(INV_MAX_NAME_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::InverterNameLength;
         retMsg["param"]["max"] = INV_MAX_NAME_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -260,8 +200,7 @@ void WebApiInverterClass::onInverterEdit(AsyncWebServerRequest* request)
     if (channelArray.size() == 0 || channelArray.size() > INV_MAX_CHAN_COUNT) {
         retMsg["message"] = "Invalid amount of max channel setting given!";
         retMsg["code"] = WebApiError::InverterInvalidMaxChannel;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -293,8 +232,7 @@ void WebApiInverterClass::onInverterEdit(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg, WebApiError::InverterChanged, "Inverter changed!");
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     std::shared_ptr<InverterAbstract> inv = Hoymiles.getInverterBySerial(old_serial);
 
@@ -333,51 +271,24 @@ void WebApiInverterClass::onInverterDelete(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("id"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["id"].as<uint8_t>() > INV_MAX_COUNT - 1) {
         retMsg["message"] = "Invalid ID specified!";
         retMsg["code"] = WebApiError::InverterInvalidId;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -390,8 +301,7 @@ void WebApiInverterClass::onInverterDelete(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg, WebApiError::InverterDeleted, "Inverter deleted!");
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     MqttHandleHass.forceUpdate();
 }
@@ -403,43 +313,17 @@ void WebApiInverterClass::onInverterOrder(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("order"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -457,6 +341,5 @@ void WebApiInverterClass::onInverterOrder(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg, WebApiError::InverterOrdered, "Inverter order saved!");
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_limit.cpp

@@ -47,8 +47,7 @@ void WebApiLimitClass::onLimitStatus(AsyncWebServerRequest* request)
         root[serial]["limit_set_status"] = limitStatus;
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
@@ -58,45 +57,19 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("serial")
             && root.containsKey("limit_value")
             && root.containsKey("limit_type"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -106,8 +79,7 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
     if (serial == 0) {
         retMsg["message"] = "Serial must be a number > 0!";
         retMsg["code"] = WebApiError::LimitSerialZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -115,8 +87,7 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Limit must between 0 and " STR(MAX_INVERTER_LIMIT) "!";
         retMsg["code"] = WebApiError::LimitInvalidLimit;
         retMsg["param"]["max"] = MAX_INVERTER_LIMIT;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -127,8 +98,7 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
 
         retMsg["message"] = "Invalid type specified!";
         retMsg["code"] = WebApiError::LimitInvalidType;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -139,8 +109,7 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
     if (inv == nullptr) {
         retMsg["message"] = "Invalid inverter specified!";
         retMsg["code"] = WebApiError::LimitInvalidInverter;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -150,6 +119,5 @@ void WebApiLimitClass::onLimitPost(AsyncWebServerRequest* request)
     retMsg["message"] = "Settings saved!";
     retMsg["code"] = WebApiError::GenericSuccess;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_maintenance.cpp

@@ -22,44 +22,18 @@ void WebApiMaintenanceClass::onRebootPost(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > MQTT_JSON_DOC_SIZE) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(MQTT_JSON_DOC_SIZE);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("reboot"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -68,14 +42,12 @@ void WebApiMaintenanceClass::onRebootPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Reboot triggered!";
         retMsg["code"] = WebApiError::MaintenanceRebootTriggered;
 
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         Utils::restartDtu();
     } else {
         retMsg["message"] = "Reboot cancled!";
         retMsg["code"] = WebApiError::MaintenanceRebootCancled;
 
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
     }
 }

src/WebApi_mqtt.cpp

@@ -30,7 +30,7 @@ void WebApiMqttClass::onMqttStatus(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
     const CONFIG_T& config = Configuration.get();
 
@@ -55,8 +55,7 @@ void WebApiMqttClass::onMqttStatus(AsyncWebServerRequest* request)
     root["mqtt_hass_topic"] = config.Mqtt.Hass.Topic;
     root["mqtt_hass_individualpanels"] = config.Mqtt.Hass.IndividualPanels;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiMqttClass::onMqttAdminGet(AsyncWebServerRequest* request)
@@ -65,7 +64,7 @@ void WebApiMqttClass::onMqttAdminGet(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
     const CONFIG_T& config = Configuration.get();
 
@@ -94,8 +93,7 @@ void WebApiMqttClass::onMqttAdminGet(AsyncWebServerRequest* request)
     root["mqtt_hass_topic"] = config.Mqtt.Hass.Topic;
     root["mqtt_hass_individualpanels"] = config.Mqtt.Hass.IndividualPanels;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
@@ -104,38 +102,13 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
         return;
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, MQTT_JSON_DOC_SIZE);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > MQTT_JSON_DOC_SIZE) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(MQTT_JSON_DOC_SIZE);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("mqtt_enabled")
             && root.containsKey("mqtt_verbose_logging")
@@ -162,8 +135,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             && root.containsKey("mqtt_hass_individualpanels"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -172,8 +144,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "MqTT Server must between 1 and " STR(MQTT_MAX_HOSTNAME_STRLEN) " characters long!";
             retMsg["code"] = WebApiError::MqttHostnameLength;
             retMsg["param"]["max"] = MQTT_MAX_HOSTNAME_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -181,48 +152,42 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "Username must not be longer than " STR(MQTT_MAX_USERNAME_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttUsernameLength;
             retMsg["param"]["max"] = MQTT_MAX_USERNAME_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
         if (root["mqtt_password"].as<String>().length() > MQTT_MAX_PASSWORD_STRLEN) {
             retMsg["message"] = "Password must not be longer than " STR(MQTT_MAX_PASSWORD_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttPasswordLength;
             retMsg["param"]["max"] = MQTT_MAX_PASSWORD_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
         if (root["mqtt_topic"].as<String>().length() > MQTT_MAX_TOPIC_STRLEN) {
             retMsg["message"] = "Topic must not be longer than " STR(MQTT_MAX_TOPIC_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttTopicLength;
             retMsg["param"]["max"] = MQTT_MAX_TOPIC_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
         if (root["mqtt_topic"].as<String>().indexOf(' ') != -1) {
             retMsg["message"] = "Topic must not contain space characters!";
             retMsg["code"] = WebApiError::MqttTopicCharacter;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
         if (!root["mqtt_topic"].as<String>().endsWith("/")) {
             retMsg["message"] = "Topic must end with a slash (/)!";
             retMsg["code"] = WebApiError::MqttTopicTrailingSlash;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
         if (root["mqtt_port"].as<uint>() == 0 || root["mqtt_port"].as<uint>() > 65535) {
             retMsg["message"] = "Port must be a number between 1 and 65535!";
             retMsg["code"] = WebApiError::MqttPort;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -232,8 +197,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "Certificates must not be longer than " STR(MQTT_MAX_CERT_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttCertificateLength;
             retMsg["param"]["max"] = MQTT_MAX_CERT_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -241,16 +205,14 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "LWT topic must not be longer than " STR(MQTT_MAX_TOPIC_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttLwtTopicLength;
             retMsg["param"]["max"] = MQTT_MAX_TOPIC_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
         if (root["mqtt_lwt_topic"].as<String>().indexOf(' ') != -1) {
             retMsg["message"] = "LWT topic must not contain space characters!";
             retMsg["code"] = WebApiError::MqttLwtTopicCharacter;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -258,8 +220,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "LWT online value must not be longer than " STR(MQTT_MAX_LWTVALUE_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttLwtOnlineLength;
             retMsg["param"]["max"] = MQTT_MAX_LWTVALUE_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -267,8 +228,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "LWT offline value must not be longer than " STR(MQTT_MAX_LWTVALUE_STRLEN) " characters!";
             retMsg["code"] = WebApiError::MqttLwtOfflineLength;
             retMsg["param"]["max"] = MQTT_MAX_LWTVALUE_STRLEN;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -276,8 +236,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["message"] = "LWT QoS must not be greater than " STR(2) "!";
             retMsg["code"] = WebApiError::MqttLwtQos;
             retMsg["param"]["max"] = 2;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -286,8 +245,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
             retMsg["code"] = WebApiError::MqttPublishInterval;
             retMsg["param"]["min"] = 5;
             retMsg["param"]["max"] = 65535;
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
 
@@ -296,16 +254,14 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
                 retMsg["message"] = "Hass topic must not be longer than " STR(MQTT_MAX_TOPIC_STRLEN) " characters!";
                 retMsg["code"] = WebApiError::MqttHassTopicLength;
                 retMsg["param"]["max"] = MQTT_MAX_TOPIC_STRLEN;
-                response->setLength();
+                WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-                request->send(response);
                 return;
             }
 
             if (root["mqtt_hass_topic"].as<String>().indexOf(' ') != -1) {
                 retMsg["message"] = "Hass topic must not contain space characters!";
                 retMsg["code"] = WebApiError::MqttHassTopicCharacter;
-                response->setLength();
+                WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-                request->send(response);
                 return;
             }
         }
@@ -339,8 +295,7 @@ void WebApiMqttClass::onMqttAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     MqttSettings.performReconnect();
     MqttHandleHass.forceUpdate();

src/WebApi_network.cpp

@@ -46,8 +46,7 @@ void WebApiNetworkClass::onNetworkStatus(AsyncWebServerRequest* request)
     root["ap_mac"] = WiFi.softAPmacAddress();
     root["ap_stationnum"] = WiFi.softAPgetStationNum();
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiNetworkClass::onNetworkAdminGet(AsyncWebServerRequest* request)
@@ -72,8 +71,7 @@ void WebApiNetworkClass::onNetworkAdminGet(AsyncWebServerRequest* request)
     root["aptimeout"] = config.WiFi.ApTimeout;
     root["mdnsenabled"] = config.Mdns.Enabled;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiNetworkClass::onNetworkAdminPost(AsyncWebServerRequest* request)
@@ -83,37 +81,12 @@ void WebApiNetworkClass::onNetworkAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("ssid")
             && root.containsKey("password")
@@ -127,8 +100,7 @@ void WebApiNetworkClass::onNetworkAdminPost(AsyncWebServerRequest* request)
             && root.containsKey("aptimeout"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -136,68 +108,59 @@ void WebApiNetworkClass::onNetworkAdminPost(AsyncWebServerRequest* request)
     if (!ipaddress.fromString(root["ipaddress"].as<String>())) {
         retMsg["message"] = "IP address is invalid!";
         retMsg["code"] = WebApiError::NetworkIpInvalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     IPAddress netmask;
     if (!netmask.fromString(root["netmask"].as<String>())) {
         retMsg["message"] = "Netmask is invalid!";
         retMsg["code"] = WebApiError::NetworkNetmaskInvalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     IPAddress gateway;
     if (!gateway.fromString(root["gateway"].as<String>())) {
         retMsg["message"] = "Gateway is invalid!";
         retMsg["code"] = WebApiError::NetworkGatewayInvalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     IPAddress dns1;
     if (!dns1.fromString(root["dns1"].as<String>())) {
         retMsg["message"] = "DNS Server IP 1 is invalid!";
         retMsg["code"] = WebApiError::NetworkDns1Invalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     IPAddress dns2;
     if (!dns2.fromString(root["dns2"].as<String>())) {
         retMsg["message"] = "DNS Server IP 2 is invalid!";
         retMsg["code"] = WebApiError::NetworkDns2Invalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
     if (root["hostname"].as<String>().length() == 0 || root["hostname"].as<String>().length() > WIFI_MAX_HOSTNAME_STRLEN) {
         retMsg["message"] = "Hostname must between 1 and " STR(WIFI_MAX_HOSTNAME_STRLEN) " characters long!";
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     if (NetworkSettings.NetworkMode() == network_mode::WiFi) {
         if (root["ssid"].as<String>().length() == 0 || root["ssid"].as<String>().length() > WIFI_MAX_SSID_STRLEN) {
             retMsg["message"] = "SSID must between 1 and " STR(WIFI_MAX_SSID_STRLEN) " characters long!";
-            response->setLength();
+            WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-            request->send(response);
             return;
         }
     }
     if (root["password"].as<String>().length() > WIFI_MAX_PASSWORD_STRLEN - 1) {
         retMsg["message"] = "Password must not be longer than " STR(WIFI_MAX_PASSWORD_STRLEN) " characters long!";
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
     if (root["aptimeout"].as<uint>() > 99999) {
         retMsg["message"] = "ApTimeout must be a number between 0 and 99999!";
         retMsg["code"] = WebApiError::NetworkApTimeoutInvalid;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -235,8 +198,7 @@ void WebApiNetworkClass::onNetworkAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     NetworkSettings.enableAdminMode();
     NetworkSettings.applyConfig();

src/WebApi_ntp.cpp

@@ -63,8 +63,7 @@ void WebApiNtpClass::onNtpStatus(AsyncWebServerRequest* request)
     root["sun_isSunsetAvailable"] = SunPosition.isSunsetAvailable();
     root["sun_isDayPeriod"] = SunPosition.isDayPeriod();
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiNtpClass::onNtpAdminGet(AsyncWebServerRequest* request)
@@ -84,8 +83,7 @@ void WebApiNtpClass::onNtpAdminGet(AsyncWebServerRequest* request)
     root["latitude"] = config.Ntp.Latitude;
     root["sunsettype"] = config.Ntp.SunsetType;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
@@ -95,37 +93,12 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("ntp_server")
             && root.containsKey("ntp_timezone")
@@ -134,8 +107,7 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
             && root.containsKey("sunsettype"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -143,8 +115,7 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
         retMsg["message"] = "NTP Server must between 1 and " STR(NTP_MAX_SERVER_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::NtpServerLength;
         retMsg["param"]["max"] = NTP_MAX_SERVER_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -152,8 +123,7 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Timezone must between 1 and " STR(NTP_MAX_TIMEZONE_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::NtpTimezoneLength;
         retMsg["param"]["max"] = NTP_MAX_TIMEZONE_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -161,8 +131,7 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Timezone description must between 1 and " STR(NTP_MAX_TIMEZONEDESCR_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::NtpTimezoneDescriptionLength;
         retMsg["param"]["max"] = NTP_MAX_TIMEZONEDESCR_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -176,8 +145,7 @@ void WebApiNtpClass::onNtpAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 
     NtpSettings.setServer();
     NtpSettings.setTimezone();
@@ -208,8 +176,7 @@ void WebApiNtpClass::onNtpTimeGet(AsyncWebServerRequest* request)
     root["minute"] = timeinfo.tm_min;
     root["second"] = timeinfo.tm_sec;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
@@ -219,37 +186,12 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("year")
             && root.containsKey("month")
@@ -259,8 +201,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
             && root.containsKey("second"))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -269,8 +210,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpYearInvalid;
         retMsg["param"]["min"] = 2022;
         retMsg["param"]["max"] = 2100;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -279,8 +219,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpMonthInvalid;
         retMsg["param"]["min"] = 1;
         retMsg["param"]["max"] = 12;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -289,8 +228,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpDayInvalid;
         retMsg["param"]["min"] = 1;
         retMsg["param"]["max"] = 31;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -299,8 +237,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpHourInvalid;
         retMsg["param"]["min"] = 0;
         retMsg["param"]["max"] = 23;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -309,8 +246,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpMinuteInvalid;
         retMsg["param"]["min"] = 0;
         retMsg["param"]["max"] = 59;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -319,8 +255,7 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
         retMsg["code"] = WebApiError::NtpSecondInvalid;
         retMsg["param"]["min"] = 0;
         retMsg["param"]["max"] = 59;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -341,6 +276,5 @@ void WebApiNtpClass::onNtpTimePost(AsyncWebServerRequest* request)
     retMsg["message"] = "Time updated!";
     retMsg["code"] = WebApiError::NtpTimeUpdated;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_power.cpp

@@ -40,8 +40,7 @@ void WebApiPowerClass::onPowerStatus(AsyncWebServerRequest* request)
         root[inv->serialString()]["power_set_status"] = limitStatus;
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiPowerClass::onPowerPost(AsyncWebServerRequest* request)
@@ -51,45 +50,19 @@ void WebApiPowerClass::onPowerPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("serial")
             && (root.containsKey("power")
                 || root.containsKey("restart")))) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -99,8 +72,7 @@ void WebApiPowerClass::onPowerPost(AsyncWebServerRequest* request)
     if (serial == 0) {
         retMsg["message"] = "Serial must be a number > 0!";
         retMsg["code"] = WebApiError::PowerSerialZero;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -108,8 +80,7 @@ void WebApiPowerClass::onPowerPost(AsyncWebServerRequest* request)
     if (inv == nullptr) {
         retMsg["message"] = "Invalid inverter specified!";
         retMsg["code"] = WebApiError::PowerInvalidInverter;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -126,6 +97,5 @@ void WebApiPowerClass::onPowerPost(AsyncWebServerRequest* request)
     retMsg["message"] = "Settings saved!";
     retMsg["code"] = WebApiError::GenericSuccess;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_powerlimiter.cpp

@@ -27,10 +27,9 @@ void WebApiPowerLimiterClass::init(AsyncWebServer& server, Scheduler& scheduler)
 
 void WebApiPowerLimiterClass::onStatus(AsyncWebServerRequest* request)
 {
-    auto const& config = Configuration.get();
+    AsyncJsonResponse* response = new AsyncJsonResponse();
-
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 512);
     auto& root = response->getRoot();
+    auto const& config = Configuration.get();
 
     root["enabled"] = config.PowerLimiter.Enabled;
     root["verbose_logging"] = config.PowerLimiter.VerboseLogging;
@@ -44,6 +43,7 @@ void WebApiPowerLimiterClass::onStatus(AsyncWebServerRequest* request)
     root["target_power_consumption"] = config.PowerLimiter.TargetPowerConsumption;
     root["target_power_consumption_hysteresis"] = config.PowerLimiter.TargetPowerConsumptionHysteresis;
     root["lower_power_limit"] = config.PowerLimiter.LowerPowerLimit;
+    root["base_load_limit"] = config.PowerLimiter.BaseLoadLimit;
     root["upper_power_limit"] = config.PowerLimiter.UpperPowerLimit;
     root["ignore_soc"] = config.PowerLimiter.IgnoreSoc;
     root["battery_soc_start_threshold"] = config.PowerLimiter.BatterySocStartThreshold;
@@ -56,8 +56,7 @@ void WebApiPowerLimiterClass::onStatus(AsyncWebServerRequest* request)
     root["full_solar_passthrough_start_voltage"] = static_cast<int>(config.PowerLimiter.FullSolarPassThroughStartVoltage * 100 + 0.5) / 100.0;
     root["full_solar_passthrough_stop_voltage"] = static_cast<int>(config.PowerLimiter.FullSolarPassThroughStopVoltage * 100 + 0.5) / 100.0;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiPowerLimiterClass::onMetaData(AsyncWebServerRequest* request)
@@ -71,14 +70,14 @@ void WebApiPowerLimiterClass::onMetaData(AsyncWebServerRequest* request)
         if (config.Inverter[i].Serial != 0) { ++invAmount; }
     }
 
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 256 + 256 * invAmount);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
 
     root["power_meter_enabled"] = config.PowerMeter.Enabled;
     root["battery_enabled"] = config.Battery.Enabled;
     root["charge_controller_enabled"] = config.Vedirect.Enabled;
 
-    JsonObject inverters = root.createNestedObject("inverters");
+    JsonObject inverters = root["inverters"].to<JsonObject>();
     for (uint8_t i = 0; i < INV_MAX_COUNT; i++) {
         if (config.Inverter[i].Serial == 0) { continue; }
 
@@ -86,7 +85,7 @@ void WebApiPowerLimiterClass::onMetaData(AsyncWebServerRequest* request)
         // rather than the hex represenation as used when handling the inverter
         // serial elsewhere in the web application, because in this case, the
         // serial is actually not displayed but only used as a value/index.
-        JsonObject obj = inverters.createNestedObject(String(config.Inverter[i].Serial));
+        JsonObject obj = inverters[String(config.Inverter[i].Serial)].to<JsonObject>();
         obj["pos"] = i;
         obj["name"] = String(config.Inverter[i].Name);
         obj["poll_enable"] = config.Inverter[i].Poll_Enable;
@@ -104,8 +103,7 @@ void WebApiPowerLimiterClass::onMetaData(AsyncWebServerRequest* request)
         }
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiPowerLimiterClass::onAdminGet(AsyncWebServerRequest* request)
@@ -124,34 +122,12 @@ void WebApiPowerLimiterClass::onAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     // we were not actually checking for all the keys we (unconditionally)
     // access below for a long time, and it is technically not needed if users
@@ -188,6 +164,7 @@ void WebApiPowerLimiterClass::onAdminPost(AsyncWebServerRequest* request)
     config.PowerLimiter.TargetPowerConsumption = root["target_power_consumption"].as<int32_t>();
     config.PowerLimiter.TargetPowerConsumptionHysteresis = root["target_power_consumption_hysteresis"].as<int32_t>();
     config.PowerLimiter.LowerPowerLimit = root["lower_power_limit"].as<int32_t>();
+    config.PowerLimiter.BaseLoadLimit = root["base_load_limit"].as<int32_t>();
     config.PowerLimiter.UpperPowerLimit = root["upper_power_limit"].as<int32_t>();
 
     if (config.Battery.Enabled) {

src/WebApi_powermeter.cpp

@@ -28,9 +28,24 @@ void WebApiPowerMeterClass::init(AsyncWebServer& server, Scheduler& scheduler)
     _server->on("/api/powermeter/testhttprequest", HTTP_POST, std::bind(&WebApiPowerMeterClass::onTestHttpRequest, this, _1));
 }
 
+void WebApiPowerMeterClass::decodeJsonPhaseConfig(JsonObject const& json, PowerMeterHttpConfig& config) const
+{
+    config.Enabled = json["enabled"].as<bool>();
+    strlcpy(config.Url, json["url"].as<String>().c_str(), sizeof(config.Url));
+    config.AuthType = json["auth_type"].as<PowerMeterHttpConfig::Auth>();
+    strlcpy(config.Username, json["username"].as<String>().c_str(), sizeof(config.Username));
+    strlcpy(config.Password, json["password"].as<String>().c_str(), sizeof(config.Password));
+    strlcpy(config.HeaderKey, json["header_key"].as<String>().c_str(), sizeof(config.HeaderKey));
+    strlcpy(config.HeaderValue, json["header_value"].as<String>().c_str(), sizeof(config.HeaderValue));
+    config.Timeout = json["timeout"].as<uint16_t>();
+    strlcpy(config.JsonPath, json["json_path"].as<String>().c_str(), sizeof(config.JsonPath));
+    config.PowerUnit = json["unit"].as<PowerMeterHttpConfig::Unit>();
+    config.SignInverted = json["sign_inverted"].as<bool>();
+}
+
 void WebApiPowerMeterClass::onStatus(AsyncWebServerRequest* request)
 {
-    AsyncJsonResponse* response = new AsyncJsonResponse(false, 2048);
+    AsyncJsonResponse* response = new AsyncJsonResponse();
     auto& root = response->getRoot();
     const CONFIG_T& config = Configuration.get();
 
@@ -45,10 +60,10 @@ void WebApiPowerMeterClass::onStatus(AsyncWebServerRequest* request)
     root["sdmaddress"] = config.PowerMeter.SdmAddress;
     root["http_individual_requests"] = config.PowerMeter.HttpIndividualRequests;
 
-    JsonArray httpPhases = root.createNestedArray("http_phases");
+    auto httpPhases = root["http_phases"].to<JsonArray>();
  
     for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) {
-        JsonObject phaseObject = httpPhases.createNestedObject();
+        auto phaseObject = httpPhases.add<JsonObject>();
   
         phaseObject["index"] = i + 1;
         phaseObject["enabled"] = config.PowerMeter.Http_Phase[i].Enabled;
@@ -58,12 +73,13 @@ void WebApiPowerMeterClass::onStatus(AsyncWebServerRequest* request)
         phaseObject["password"] = String(config.PowerMeter.Http_Phase[i].Password);
         phaseObject["header_key"] = String(config.PowerMeter.Http_Phase[i].HeaderKey);
         phaseObject["header_value"] = String(config.PowerMeter.Http_Phase[i].HeaderValue);
-        phaseObject["json_path"] = String(config.PowerMeter.Http_Phase[i].JsonPath);
         phaseObject["timeout"] = config.PowerMeter.Http_Phase[i].Timeout;
+        phaseObject["json_path"] = String(config.PowerMeter.Http_Phase[i].JsonPath);
+        phaseObject["unit"] = config.PowerMeter.Http_Phase[i].PowerUnit;
+        phaseObject["sign_inverted"] = config.PowerMeter.Http_Phase[i].SignInverted;
     }
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiPowerMeterClass::onAdminGet(AsyncWebServerRequest* request)
@@ -82,34 +98,12 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 4096) {
-        retMsg["message"] = "Data too large!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(4096);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!(root.containsKey("enabled") && root.containsKey("source"))) {
         retMsg["message"] = "Values are missing!";
@@ -137,7 +131,7 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request)
                     return;
                 }
 
-                if ((phase["auth_type"].as<Auth>() != Auth::none)
+                if ((phase["auth_type"].as<uint8_t>() != PowerMeterHttpConfig::Auth::None)
                     && ( phase["username"].as<String>().length() == 0 ||  phase["password"].as<String>().length() == 0)) {
                     retMsg["message"] = "Username or password must not be empty!";
                     response->setLength();
@@ -178,23 +172,14 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request)
 
     JsonArray http_phases = root["http_phases"];
     for (uint8_t i = 0; i < http_phases.size(); i++) {
-        JsonObject phase = http_phases[i].as<JsonObject>();
+        decodeJsonPhaseConfig(http_phases[i].as<JsonObject>(), config.PowerMeter.Http_Phase[i]);
-
-        config.PowerMeter.Http_Phase[i].Enabled = (i == 0 ? true : phase["enabled"].as<bool>());
-        strlcpy(config.PowerMeter.Http_Phase[i].Url, phase["url"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].Url));
-        config.PowerMeter.Http_Phase[i].AuthType = phase["auth_type"].as<Auth>();
-        strlcpy(config.PowerMeter.Http_Phase[i].Username, phase["username"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].Username));
-        strlcpy(config.PowerMeter.Http_Phase[i].Password, phase["password"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].Password));
-        strlcpy(config.PowerMeter.Http_Phase[i].HeaderKey, phase["header_key"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].HeaderKey));
-        strlcpy(config.PowerMeter.Http_Phase[i].HeaderValue, phase["header_value"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].HeaderValue));
-        config.PowerMeter.Http_Phase[i].Timeout = phase["timeout"].as<uint16_t>();
-        strlcpy(config.PowerMeter.Http_Phase[i].JsonPath, phase["json_path"].as<String>().c_str(), sizeof(config.PowerMeter.Http_Phase[i].JsonPath));
     }
+    config.PowerMeter.Http_Phase[0].Enabled = true;
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
+
 
     // reboot requiered as per https://github.com/helgeerbe/OpenDTU-OnBattery/issues/565#issuecomment-1872552559
     yield();
@@ -210,34 +195,12 @@ void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* asyncJsonResponse = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, asyncJsonResponse, root)) {
+        return;
+    }
+
     auto& retMsg = asyncJsonResponse->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        asyncJsonResponse->setLength();
-        request->send(asyncJsonResponse);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 2048) {
-        retMsg["message"] = "Data too large!";
-        asyncJsonResponse->setLength();
-        request->send(asyncJsonResponse);
-        return;
-    }
-
-    DynamicJsonDocument root(2048);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        asyncJsonResponse->setLength();
-        request->send(asyncJsonResponse);
-        return;
-    }
 
     if (!root.containsKey("url") || !root.containsKey("auth_type") || !root.containsKey("username") || !root.containsKey("password") 
             || !root.containsKey("header_key") || !root.containsKey("header_value")
@@ -252,11 +215,10 @@ void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request)
     char response[256];
 
     int phase = 0;//"absuing" index 0 of the float power[3] in HttpPowerMeter to store the result
-    if (HttpPowerMeter.queryPhase(phase, root[F("url")].as<String>().c_str(),
+    PowerMeterHttpConfig phaseConfig;
-            root[F("auth_type")].as<Auth>(), root[F("username")].as<String>().c_str(), root[F("password")].as<String>().c_str(),
+    decodeJsonPhaseConfig(root.as<JsonObject>(), phaseConfig);
-            root[F("header_key")].as<String>().c_str(), root[F("header_value")].as<String>().c_str(), root[F("timeout")].as<uint16_t>(),
+    if (HttpPowerMeter.queryPhase(phase, phaseConfig)) {
-            root[F("json_path")].as<String>().c_str())) {
+        retMsg["type"] = "success";
-        retMsg[F("type")] = F("success");
         snprintf_P(response, sizeof(response), "Success! Power: %5.2fW", HttpPowerMeter.getPower(phase + 1));
     } else {
         snprintf_P(response, sizeof(response), "%s", HttpPowerMeter.httpPowerMeterError);

src/WebApi_security.cpp

@@ -31,8 +31,7 @@ void WebApiSecurityClass::onSecurityGet(AsyncWebServerRequest* request)
     root["password"] = config.Security.Password;
     root["allow_readonly"] = config.Security.AllowReadonly;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiSecurityClass::onSecurityPost(AsyncWebServerRequest* request)
@@ -42,44 +41,18 @@ void WebApiSecurityClass::onSecurityPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    const String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    const DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!root.containsKey("password")
         && root.containsKey("allow_readonly")) {
         retMsg["message"] = "Values are missing!";
         retMsg["code"] = WebApiError::GenericValueMissing;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -87,8 +60,7 @@ void WebApiSecurityClass::onSecurityPost(AsyncWebServerRequest* request)
         retMsg["message"] = "Password must between 8 and " STR(WIFI_MAX_PASSWORD_STRLEN) " characters long!";
         retMsg["code"] = WebApiError::SecurityPasswordLength;
         retMsg["param"]["max"] = WIFI_MAX_PASSWORD_STRLEN;
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
         return;
     }
 
@@ -98,8 +70,7 @@ void WebApiSecurityClass::onSecurityPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }
 
 void WebApiSecurityClass::onAuthenticateGet(AsyncWebServerRequest* request)
@@ -114,6 +85,5 @@ void WebApiSecurityClass::onAuthenticateGet(AsyncWebServerRequest* request)
     retMsg["message"] = "Authentication successful!";
     retMsg["code"] = WebApiError::SecurityAuthSuccess;
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_sysstatus.cpp

@@ -81,6 +81,5 @@ void WebApiSysstatusClass::onSystemStatus(AsyncWebServerRequest* request)
     root["cmt_configured"] = PinMapping.isValidCmt2300Config();
     root["cmt_connected"] = Hoymiles.getRadioCmt()->isConnected();
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
 }

src/WebApi_vedirect.cpp

@@ -66,37 +66,12 @@ void WebApiVedirectClass::onVedirectAdminPost(AsyncWebServerRequest* request)
     }
 
     AsyncJsonResponse* response = new AsyncJsonResponse();
+    JsonDocument root;
+    if (!WebApi.parseRequestData(request, response, root)) {
+        return;
+    }
+
     auto& retMsg = response->getRoot();
-    retMsg["type"] = "warning";
-
-    if (!request->hasParam("data", true)) {
-        retMsg["message"] = "No values found!";
-        retMsg["code"] = WebApiError::GenericNoValueFound;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    String json = request->getParam("data", true)->value();
-
-    if (json.length() > 1024) {
-        retMsg["message"] = "Data too large!";
-        retMsg["code"] = WebApiError::GenericDataTooLarge;
-        response->setLength();
-        request->send(response);
-        return;
-    }
-
-    DynamicJsonDocument root(1024);
-    DeserializationError error = deserializeJson(root, json);
-
-    if (error) {
-        retMsg["message"] = "Failed to parse data!";
-        retMsg["code"] = WebApiError::GenericParseError;
-        response->setLength();
-        request->send(response);
-        return;
-    }
 
     if (!root.containsKey("vedirect_enabled") ||
             !root.containsKey("verbose_logging") ||
@@ -115,8 +90,8 @@ void WebApiVedirectClass::onVedirectAdminPost(AsyncWebServerRequest* request)
 
     WebApi.writeConfig(retMsg);
 
-    response->setLength();
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-    request->send(response);
+
 
     VictronMppt.updateSettings();
 

src/WebApi_ws_Huawei.cpp

@@ -59,22 +59,15 @@ void WebApiWsHuaweiLiveClass::sendDataTaskCb()
 
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        DynamicJsonDocument root(1024);
+        JsonDocument root;
-        if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
         JsonVariant var = root;
-            generateJsonResponse(var);
 
-            if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+        generateCommonJsonResponse(var);
 
+        if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
             String buffer;
             serializeJson(root, buffer);
 
-            if (Configuration.get().Security.AllowReadonly) {
-                _ws.setAuthentication("", "");
-            } else {
-                _ws.setAuthentication(AUTH_USERNAME, Configuration.get().Security.Password);
-            }
-
             _ws.textAll(buffer);
         }
     } catch (std::bad_alloc& bad_alloc) {
@@ -84,7 +77,7 @@ void WebApiWsHuaweiLiveClass::sendDataTaskCb()
     }
 }
 
-void WebApiWsHuaweiLiveClass::generateJsonResponse(JsonVariant& root)
+void WebApiWsHuaweiLiveClass::generateCommonJsonResponse(JsonVariant& root)
 {
     const RectifierParameters_t * rp = HuaweiCan.get();
 
@@ -134,13 +127,13 @@ void WebApiWsHuaweiLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
     }
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        AsyncJsonResponse* response = new AsyncJsonResponse(false, 1024U);
+        AsyncJsonResponse* response = new AsyncJsonResponse();
         auto& root = response->getRoot();
 
-        generateJsonResponse(root);
+        generateCommonJsonResponse(root);
+
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
 
-        response->setLength();
-        request->send(response);
     } catch (std::bad_alloc& bad_alloc) {
         MessageOutput.printf("Calling /api/huaweilivedata/status has temporarily run out of resources. Reason: \"%s\".\r\n", bad_alloc.what());
         WebApi.sendTooManyRequests(request);

src/WebApi_ws_battery.cpp

@@ -62,12 +62,12 @@ void WebApiWsBatteryLiveClass::sendDataTaskCb()
 
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        DynamicJsonDocument root(_responseSize);
+        JsonDocument root;
-         if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
         JsonVariant var = root;
-            generateJsonResponse(var);
         
-            if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+        generateCommonJsonResponse(var);
+
+        if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
 
             // battery provider does not generate a card, e.g., MQTT provider
             if (root.isNull()) { return; }
@@ -90,7 +90,7 @@ void WebApiWsBatteryLiveClass::sendDataTaskCb()
     }
 }
 
-void WebApiWsBatteryLiveClass::generateJsonResponse(JsonVariant& root)
+void WebApiWsBatteryLiveClass::generateCommonJsonResponse(JsonVariant& root)
 {
     Battery.getStats()->getLiveViewData(root);
 }
@@ -111,12 +111,11 @@ void WebApiWsBatteryLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
     }
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        AsyncJsonResponse* response = new AsyncJsonResponse(false, _responseSize);
+        AsyncJsonResponse* response = new AsyncJsonResponse();
         auto& root = response->getRoot();
-        generateJsonResponse(root);
+        generateCommonJsonResponse(root);
 
-        response->setLength();
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
-        request->send(response);
     } catch (std::bad_alloc& bad_alloc) {
         MessageOutput.printf("Calling /api/batterylivedata/status has temporarily run out of resources. Reason: \"%s\".\r\n", bad_alloc.what());
         WebApi.sendTooManyRequests(request);

src/WebApi_ws_live.cpp

@@ -61,11 +61,11 @@ void WebApiWsLiveClass::generateOnBatteryJsonResponse(JsonVariant& root, bool al
 
     auto victronAge = VictronMppt.getDataAgeMillis();
     if (all || (victronAge > 0 && (millis() - _lastPublishVictron) > victronAge)) {
-        JsonObject vedirectObj = root.createNestedObject("vedirect");
+        auto vedirectObj = root["vedirect"].to<JsonObject>();
         vedirectObj["enabled"] = config.Vedirect.Enabled;
 
         if (config.Vedirect.Enabled) {
-            JsonObject totalVeObj = vedirectObj.createNestedObject("total");
+            auto totalVeObj = vedirectObj["total"].to<JsonObject>();
             addTotalField(totalVeObj, "Power", VictronMppt.getPanelPowerWatts(), "W", 1);
             addTotalField(totalVeObj, "YieldDay", VictronMppt.getYieldDay() * 1000, "Wh", 0);
             addTotalField(totalVeObj, "YieldTotal", VictronMppt.getYieldTotal(), "kWh", 2);
@@ -75,12 +75,12 @@ void WebApiWsLiveClass::generateOnBatteryJsonResponse(JsonVariant& root, bool al
     }
 
     if (all || (HuaweiCan.getLastUpdate() - _lastPublishHuawei) < halfOfAllMillis ) {
-        JsonObject huaweiObj = root.createNestedObject("huawei");
+        auto huaweiObj = root["huawei"].to<JsonObject>();
         huaweiObj["enabled"] = config.Huawei.Enabled;
 
         if (config.Huawei.Enabled) {
             const RectifierParameters_t * rp = HuaweiCan.get();
-            addTotalField(huaweiObj, "Power", rp->output_power, "W", 2);
+            addTotalField(huaweiObj, "Power", rp->input_power, "W", 2);
         }
 
         if (!all) { _lastPublishHuawei = millis(); }
@@ -88,7 +88,7 @@ void WebApiWsLiveClass::generateOnBatteryJsonResponse(JsonVariant& root, bool al
 
     auto spStats = Battery.getStats();
     if (all || spStats->updateAvailable(_lastPublishBattery)) {
-        JsonObject batteryObj = root.createNestedObject("battery");
+        auto batteryObj = root["battery"].to<JsonObject>();
         batteryObj["enabled"] = config.Battery.Enabled;
 
         if (config.Battery.Enabled) {
@@ -99,7 +99,7 @@ void WebApiWsLiveClass::generateOnBatteryJsonResponse(JsonVariant& root, bool al
     }
 
     if (all || (PowerMeter.getLastPowerMeterUpdate() - _lastPublishPowerMeter) < halfOfAllMillis) {
-        JsonObject powerMeterObj = root.createNestedObject("power_meter");
+        auto powerMeterObj = root["power_meter"].to<JsonObject>();
         powerMeterObj["enabled"] = config.PowerMeter.Enabled;
 
         if (config.PowerMeter.Enabled) {
@@ -112,9 +112,7 @@ void WebApiWsLiveClass::generateOnBatteryJsonResponse(JsonVariant& root, bool al
 
 void WebApiWsLiveClass::sendOnBatteryStats()
 {
-    DynamicJsonDocument root(1024);
+    JsonDocument root;
-    if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) { return; }
-
     JsonVariant var = root;
 
     bool all = (millis() - _lastPublishOnBatteryFull) > 10 * 1000;
@@ -123,12 +121,12 @@ void WebApiWsLiveClass::sendOnBatteryStats()
 
     if (root.isNull()) { return; }
 
-    if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+    if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-
         String buffer;
         serializeJson(root, buffer);
 
-    _ws.textAll(buffer);
+        _ws.textAll(buffer);;
+    }
 }
 
 void WebApiWsLiveClass::sendDataTaskCb()
@@ -156,19 +154,20 @@ void WebApiWsLiveClass::sendDataTaskCb()
 
         try {
             std::lock_guard<std::mutex> lock(_mutex);
-            DynamicJsonDocument root(4096);
+            JsonDocument root;
-            if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
-                continue;
-            }
             JsonVariant var = root;
 
-            auto invArray = var.createNestedArray("inverters");
+            auto invArray = var["inverters"].to<JsonArray>();
-            auto invObject = invArray.createNestedObject();
+            auto invObject = invArray.add<JsonObject>();
 
             generateCommonJsonResponse(var);
             generateInverterCommonJsonResponse(invObject, inv);
             generateInverterChannelJsonResponse(invObject, inv);
 
+            if (!Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
+                continue;
+            }
+
             String buffer;
             serializeJson(root, buffer);
 
@@ -184,12 +183,12 @@ void WebApiWsLiveClass::sendDataTaskCb()
 
 void WebApiWsLiveClass::generateCommonJsonResponse(JsonVariant& root)
 {
-    JsonObject totalObj = root.createNestedObject("total");
+    auto totalObj = root["total"].to<JsonObject>();
     addTotalField(totalObj, "Power", Datastore.getTotalAcPowerEnabled(), "W", Datastore.getTotalAcPowerDigits());
     addTotalField(totalObj, "YieldDay", Datastore.getTotalAcYieldDayEnabled(), "Wh", Datastore.getTotalAcYieldDayDigits());
     addTotalField(totalObj, "YieldTotal", Datastore.getTotalAcYieldTotalEnabled(), "kWh", Datastore.getTotalAcYieldTotalDigits());
 
-    JsonObject hintObj = root.createNestedObject("hints");
+    JsonObject hintObj = root["hints"].to<JsonObject>();
     struct tm timeinfo;
     hintObj["time_sync"] = !getLocalTime(&timeinfo, 5);
     hintObj["radio_problem"] = (Hoymiles.getRadioNrf()->isInitialized() && (!Hoymiles.getRadioNrf()->isConnected() || !Hoymiles.getRadioNrf()->isPVariant())) || (Hoymiles.getRadioCmt()->isInitialized() && (!Hoymiles.getRadioCmt()->isConnected()));
@@ -227,7 +226,7 @@ void WebApiWsLiveClass::generateInverterChannelJsonResponse(JsonObject& root, st
 
     // Loop all channels
     for (auto& t : inv->Statistics()->getChannelTypes()) {
-        JsonObject chanTypeObj = root.createNestedObject(inv->Statistics()->getChannelTypeName(t));
+        auto chanTypeObj = root[inv->Statistics()->getChannelTypeName(t)].to<JsonObject>();
         for (auto& c : inv->Statistics()->getChannelsByType(t)) {
             if (t == TYPE_DC) {
                 chanTypeObj[String(static_cast<uint8_t>(c))]["name"]["u"] = inv_cfg->channel[c].Name;
@@ -304,21 +303,15 @@ void WebApiWsLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
 
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        AsyncJsonResponse* response = new AsyncJsonResponse(false, 4096);
+        AsyncJsonResponse* response = new AsyncJsonResponse();
         auto& root = response->getRoot();
-
+        auto invArray = root["inverters"].to<JsonArray>();
-        JsonArray invArray = root.createNestedArray("inverters");
+        auto serial = WebApi.parseSerialFromRequest(request);
-
-        uint64_t serial = 0;
-        if (request->hasParam("inv")) {
-            String s = request->getParam("inv")->value();
-            serial = strtoll(s.c_str(), NULL, 16);
-        }
 
         if (serial > 0) {
             auto inv = Hoymiles.getInverterBySerial(serial);
             if (inv != nullptr) {
-                JsonObject invObject = invArray.createNestedObject();
+                JsonObject invObject = invArray.add<JsonObject>();
                 generateInverterCommonJsonResponse(invObject, inv);
                 generateInverterChannelJsonResponse(invObject, inv);
             }
@@ -330,7 +323,7 @@ void WebApiWsLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
                     continue;
                 }
 
-                JsonObject invObject = invArray.createNestedObject();
+                JsonObject invObject = invArray.add<JsonObject>();
                 generateInverterCommonJsonResponse(invObject, inv);
             }
         }
@@ -339,8 +332,9 @@ void WebApiWsLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
 
         generateOnBatteryJsonResponse(root, true);
 
-        response->setLength();
+        generateOnBatteryJsonResponse(root, true);
-        request->send(response);
+
+    WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
     
     } catch (const std::bad_alloc& bad_alloc) {
         MessageOutput.printf("Calling /api/livedata/status has temporarily run out of resources. Reason: \"%s\".\r\n", bad_alloc.what());

src/WebApi_ws_vedirect_live.cpp

@@ -86,25 +86,17 @@ void WebApiWsVedirectLiveClass::sendDataTaskCb()
     if (fullUpdate || updateAvailable) {
         try {
             std::lock_guard<std::mutex> lock(_mutex);
-            DynamicJsonDocument root(responseSize());
+            JsonDocument root;
-            if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
             JsonVariant var = root;
-                generateJsonResponse(var, fullUpdate);
 
-                if (Utils::checkJsonOverflow(root, __FUNCTION__, __LINE__)) { return; }
+            generateCommonJsonResponse(var, fullUpdate);
 
+            if (Utils::checkJsonAlloc(root, __FUNCTION__, __LINE__)) {
                 String buffer;
                 serializeJson(root, buffer);
 
-                if (Configuration.get().Security.AllowReadonly) {
+                _ws.textAll(buffer);;
-                    _ws.setAuthentication("", "");
-                } else {
-                    _ws.setAuthentication(AUTH_USERNAME, Configuration.get().Security.Password);
             }
-
-                _ws.textAll(buffer);
-            }
-
         } catch (std::bad_alloc& bad_alloc) {
             MessageOutput.printf("Calling /api/vedirectlivedata/status has temporarily run out of resources. Reason: \"%s\".\r\n", bad_alloc.what());
         } catch (const std::exception& exc) {
@@ -117,30 +109,27 @@ void WebApiWsVedirectLiveClass::sendDataTaskCb()
     }
 }
 
-void WebApiWsVedirectLiveClass::generateJsonResponse(JsonVariant& root, bool fullUpdate)
+void WebApiWsVedirectLiveClass::generateCommonJsonResponse(JsonVariant& root, bool fullUpdate)
 {
-    const JsonObject &array = root["vedirect"].createNestedObject("instances");
+    auto array = root["vedirect"]["instances"].to<JsonObject>();
     root["vedirect"]["full_update"] = fullUpdate;
 
     for (size_t idx = 0; idx < VictronMppt.controllerAmount(); ++idx) {
-        std::optional<VeDirectMpptController::spData_t> spOptMpptData = VictronMppt.getData(idx);
+        auto optMpptData = VictronMppt.getData(idx);
-        if (!spOptMpptData.has_value()) {
+        if (!optMpptData.has_value()) { continue; }
-            continue;
-        }
 
         if (!fullUpdate && !hasUpdate(idx)) { continue; }
 
-        VeDirectMpptController::spData_t &spMpptData = spOptMpptData.value();
+        String serial(optMpptData->serialNr_SER);
-
-        String serial(spMpptData->SER);
         if (serial.isEmpty()) { continue; } // serial required as index
 
-        const JsonObject &nested = array.createNestedObject(serial);
+        JsonObject nested = array[serial].to<JsonObject>();
         nested["data_age_ms"] = VictronMppt.getDataAgeMillis(idx);
-        populateJson(nested, spMpptData);
+        populateJson(nested, *optMpptData);
-        _lastPublish = millis();
     }
 
+    _lastPublish = millis();
+
     // power limiter state
     root["dpl"]["PLSTATE"] = -1;
     if (Configuration.get().PowerLimiter.Enabled)
@@ -148,58 +137,70 @@ void WebApiWsVedirectLiveClass::generateJsonResponse(JsonVariant& root, bool ful
     root["dpl"]["PLLIMIT"] = PowerLimiter.getLastRequestedPowerLimit();
 }
 
-void WebApiWsVedirectLiveClass::populateJson(const JsonObject &root, const VeDirectMpptController::spData_t &spMpptData) {
+void WebApiWsVedirectLiveClass::populateJson(const JsonObject &root, const VeDirectMpptController::data_t &mpptData) {
-    // device info
+    root["product_id"] = mpptData.getPidAsString();
-    root["device"]["PID"] = spMpptData->getPidAsString();
+    root["firmware_version"] = String(mpptData.firmwareNr_FW);
-    root["device"]["SER"] = spMpptData->SER;
-    root["device"]["FW"] = spMpptData->FW;
-    root["device"]["LOAD"] = spMpptData->LOAD ? "ON" : "OFF";
-    root["device"]["CS"] = spMpptData->getCsAsString();
-    root["device"]["ERR"] = spMpptData->getErrAsString();
-    root["device"]["OR"] = spMpptData->getOrAsString();
-    root["device"]["MPPT"] = spMpptData->getMpptAsString();
-    root["device"]["HSDS"]["v"] = spMpptData->HSDS;
-    root["device"]["HSDS"]["u"] = "d";
 
-    // battery info
+    const JsonObject values = root["values"].to<JsonObject>();
-    root["output"]["P"]["v"] = spMpptData->P;
-    root["output"]["P"]["u"] = "W";
-    root["output"]["P"]["d"] = 0;
-    root["output"]["V"]["v"] = spMpptData->V;
-    root["output"]["V"]["u"] = "V";
-    root["output"]["V"]["d"] = 2;
-    root["output"]["I"]["v"] = spMpptData->I;
-    root["output"]["I"]["u"] = "A";
-    root["output"]["I"]["d"] = 2;
-    root["output"]["E"]["v"] = spMpptData->E;
-    root["output"]["E"]["u"] = "%";
-    root["output"]["E"]["d"] = 1;
 
-    // panel info
+    const JsonObject device = values["device"].to<JsonObject>();
-    root["input"]["PPV"]["v"] = spMpptData->PPV;
+    device["LOAD"] = mpptData.loadOutputState_LOAD ? "ON" : "OFF";
-    root["input"]["PPV"]["u"] = "W";
+    device["CS"] = mpptData.getCsAsString();
-    root["input"]["PPV"]["d"] = 0;
+    device["MPPT"] = mpptData.getMpptAsString();
-    root["input"]["VPV"]["v"] = spMpptData->VPV;
+    device["OR"] = mpptData.getOrAsString();
-    root["input"]["VPV"]["u"] = "V";
+    device["ERR"] = mpptData.getErrAsString();
-    root["input"]["VPV"]["d"] = 2;
+    device["HSDS"]["v"] = mpptData.daySequenceNr_HSDS;
-    root["input"]["IPV"]["v"] = spMpptData->IPV;
+    device["HSDS"]["u"] = "d";
-    root["input"]["IPV"]["u"] = "A";
+    if (mpptData.MpptTemperatureMilliCelsius.first > 0) {
-    root["input"]["IPV"]["d"] = 2;
+        device["MpptTemperature"]["v"] = mpptData.MpptTemperatureMilliCelsius.second / 1000.0;
-    root["input"]["YieldToday"]["v"] = spMpptData->H20;
+        device["MpptTemperature"]["u"] = "°C";
-    root["input"]["YieldToday"]["u"] = "kWh";
+        device["MpptTemperature"]["d"] = "1";
-    root["input"]["YieldToday"]["d"] = 3;
+    }
-    root["input"]["YieldYesterday"]["v"] = spMpptData->H22;
+
-    root["input"]["YieldYesterday"]["u"] = "kWh";
+    const JsonObject output = values["output"].to<JsonObject>();
-    root["input"]["YieldYesterday"]["d"] = 3;
+    output["P"]["v"] = mpptData.batteryOutputPower_W;
-    root["input"]["YieldTotal"]["v"] = spMpptData->H19;
+    output["P"]["u"] = "W";
-    root["input"]["YieldTotal"]["u"] = "kWh";
+    output["P"]["d"] = 0;
-    root["input"]["YieldTotal"]["d"] = 3;
+    output["V"]["v"] = mpptData.batteryVoltage_V_mV / 1000.0;
-    root["input"]["MaximumPowerToday"]["v"] = spMpptData->H21;
+    output["V"]["u"] = "V";
-    root["input"]["MaximumPowerToday"]["u"] = "W";
+    output["V"]["d"] = 2;
-    root["input"]["MaximumPowerToday"]["d"] = 0;
+    output["I"]["v"] = mpptData.batteryCurrent_I_mA / 1000.0;
-    root["input"]["MaximumPowerYesterday"]["v"] = spMpptData->H23;
+    output["I"]["u"] = "A";
-    root["input"]["MaximumPowerYesterday"]["u"] = "W";
+    output["I"]["d"] = 2;
-    root["input"]["MaximumPowerYesterday"]["d"] = 0;
+    output["E"]["v"] = mpptData.mpptEfficiency_Percent;
+    output["E"]["u"] = "%";
+    output["E"]["d"] = 1;
+
+    const JsonObject input = values["input"].to<JsonObject>();
+    if (mpptData.NetworkTotalDcInputPowerMilliWatts.first > 0) {
+        input["NetworkPower"]["v"] = mpptData.NetworkTotalDcInputPowerMilliWatts.second / 1000.0;
+        input["NetworkPower"]["u"] = "W";
+        input["NetworkPower"]["d"] = "0";
+    }
+    input["PPV"]["v"] = mpptData.panelPower_PPV_W;
+    input["PPV"]["u"] = "W";
+    input["PPV"]["d"] = 0;
+    input["VPV"]["v"] = mpptData.panelVoltage_VPV_mV / 1000.0;
+    input["VPV"]["u"] = "V";
+    input["VPV"]["d"] = 2;
+    input["IPV"]["v"] = mpptData.panelCurrent_mA / 1000.0;
+    input["IPV"]["u"] = "A";
+    input["IPV"]["d"] = 2;
+    input["YieldToday"]["v"] = mpptData.yieldToday_H20_Wh / 1000.0;
+    input["YieldToday"]["u"] = "kWh";
+    input["YieldToday"]["d"] = 2;
+    input["YieldYesterday"]["v"] = mpptData.yieldYesterday_H22_Wh / 1000.0;
+    input["YieldYesterday"]["u"] = "kWh";
+    input["YieldYesterday"]["d"] = 2;
+    input["YieldTotal"]["v"] = mpptData.yieldTotal_H19_Wh / 1000.0;
+    input["YieldTotal"]["u"] = "kWh";
+    input["YieldTotal"]["d"] = 2;
+    input["MaximumPowerToday"]["v"] = mpptData.maxPowerToday_H21_W;
+    input["MaximumPowerToday"]["u"] = "W";
+    input["MaximumPowerToday"]["d"] = 0;
+    input["MaximumPowerYesterday"]["v"] = mpptData.maxPowerYesterday_H23_W;
+    input["MaximumPowerYesterday"]["u"] = "W";
+    input["MaximumPowerYesterday"]["d"] = 0;
 }
 
 void WebApiWsVedirectLiveClass::onWebsocketEvent(AsyncWebSocket* server, AsyncWebSocketClient* client, AwsEventType type, void* arg, uint8_t* data, size_t len)
@@ -224,14 +225,12 @@ void WebApiWsVedirectLiveClass::onLivedataStatus(AsyncWebServerRequest* request)
     }
     try {
         std::lock_guard<std::mutex> lock(_mutex);
-        AsyncJsonResponse* response = new AsyncJsonResponse(false, responseSize());
+        AsyncJsonResponse* response = new AsyncJsonResponse();
         auto& root = response->getRoot();
 
-        generateJsonResponse(root, true/*fullUpdate*/);
+        generateCommonJsonResponse(root, true/*fullUpdate*/);
-
-        response->setLength();
-        request->send(response);
 
+        WebApi.sendJsonResponse(request, response, __FUNCTION__, __LINE__);
     } catch (std::bad_alloc& bad_alloc) {
         MessageOutput.printf("Calling /api/vedirectlivedata/status has temporarily run out of resources. Reason: \"%s\".\r\n", bad_alloc.what());
         WebApi.sendTooManyRequests(request);

webapp/.eslintrc.cjs

@@ -1,14 +0,0 @@
-/* eslint-env node */
-require('@rushstack/eslint-patch/modern-module-resolution')
-
-module.exports = {
-  root: true,
-  'extends': [
-    'plugin:vue/vue3-essential',
-    'eslint:recommended',
-    '@vue/eslint-config-typescript'
-  ],
-  parserOptions: {
-    ecmaVersion: 'latest'
-  }
-}

webapp/eslint.config.js

@@ -0,0 +1,36 @@
+/* eslint-env node */
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { FlatCompat } from "@eslint/eslintrc";
+import js from "@eslint/js";
+import pluginVue from 'eslint-plugin-vue'
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+});
+
+export default [
+    js.configs.recommended,
+    ...pluginVue.configs['flat/essential'],
+    ...compat.extends("@vue/eslint-config-typescript/recommended"),
+    {
+      files: [
+        "**/*.vue",
+        "**/*.js",
+        "**/*.jsx",
+        "**/*.cjs",
+        "**/*.mjs",
+        "**/*.ts",
+        "**/*.tsx",
+        "**/*.cts",
+        "**/*.mts",
+      ],
+      languageOptions: {
+        ecmaVersion: 'latest'
+      },
+    }
+  ]

webapp/package.json

@@ -9,7 +9,7 @@
     "preview": "vite preview --port 4173",
     "build-only": "vite build",
     "type-check": "vue-tsc --noEmit",
-    "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore"
+    "lint": "eslint ."
   },
   "dependencies": {
     "@popperjs/core": "^2.11.8",
@@ -18,32 +18,31 @@
     "mitt": "^3.0.1",
     "sortablejs": "^1.15.2",
     "spark-md5": "^3.0.2",
-    "vue": "^3.4.21",
+    "vue": "^3.4.25",
-    "vue-i18n": "^9.10.2",
+    "vue-i18n": "^9.13.1",
-    "vue-router": "^4.3.0"
+    "vue-router": "^4.3.2"
   },
   "devDependencies": {
-    "@intlify/unplugin-vue-i18n": "^3.0.1",
+    "@intlify/unplugin-vue-i18n": "^4.0.0",
-    "@rushstack/eslint-patch": "^1.8.0",
+    "@tsconfig/node18": "^18.2.4",
-    "@tsconfig/node18": "^18.2.2",
     "@types/bootstrap": "^5.2.10",
-    "@types/node": "^20.11.30",
+    "@types/node": "^20.12.7",
     "@types/pulltorefreshjs": "^0.1.7",
     "@types/sortablejs": "^1.15.8",
     "@types/spark-md5": "^3.0.4",
     "@vitejs/plugin-vue": "^5.0.4",
     "@vue/eslint-config-typescript": "^13.0.0",
     "@vue/tsconfig": "^0.5.1",
-    "eslint": "^8.57.0",
+    "eslint": "^9.1.1",
-    "eslint-plugin-vue": "^9.23.0",
+    "eslint-plugin-vue": "^9.25.0",
     "npm-run-all": "^4.1.5",
     "pulltorefreshjs": "^0.1.22",
-    "sass": "^1.72.0",
+    "sass": "^1.75.0",
-    "terser": "^5.29.2",
+    "terser": "^5.30.4",
-    "typescript": "^5.4.3",
+    "typescript": "^5.4.5",
-    "vite": "^5.2.3",
+    "vite": "^5.2.10",
     "vite-plugin-compression": "^0.5.1",
     "vite-plugin-css-injected-by-js": "^3.5.0",
-    "vue-tsc": "^2.0.7"
+    "vue-tsc": "^2.0.14"
   }
 }

webapp/public/zones.json

@@ -180,7 +180,7 @@
 "America/Santiago":"<-04>4<-03>,M9.1.6/24,M4.1.6/24",
 "America/Santo_Domingo":"AST4",
 "America/Sao_Paulo":"<-03>3",
-"America/Scoresbysund":"<-01>1<+00>,M3.5.0/0,M10.5.0/1",
+"America/Scoresbysund":"<-02>2<-01>,M3.5.0/-1,M10.5.0/0",
 "America/Sitka":"AKST9AKDT,M3.2.0,M11.1.0",
 "America/St_Barthelemy":"AST4",
 "America/St_Johns":"NST3:30NDT,M3.2.0,M11.1.0",
@@ -200,7 +200,7 @@
 "America/Winnipeg":"CST6CDT,M3.2.0,M11.1.0",
 "America/Yakutat":"AKST9AKDT,M3.2.0,M11.1.0",
 "America/Yellowknife":"MST7MDT,M3.2.0,M11.1.0",
-"Antarctica/Casey":"<+11>-11",
+"Antarctica/Casey":"<+08>-8",
 "Antarctica/Davis":"<+07>-7",
 "Antarctica/DumontDUrville":"<+10>-10",
 "Antarctica/Macquarie":"AEST-10AEDT,M10.1.0,M4.1.0/3",
@@ -210,10 +210,10 @@
 "Antarctica/Rothera":"<-03>3",
 "Antarctica/Syowa":"<+03>-3",
 "Antarctica/Troll":"<+00>0<+02>-2,M3.5.0/1,M10.5.0/3",
-"Antarctica/Vostok":"<+06>-6",
+"Antarctica/Vostok":"<+05>-5",
 "Arctic/Longyearbyen":"CET-1CEST,M3.5.0,M10.5.0/3",
 "Asia/Aden":"<+03>-3",
-"Asia/Almaty":"<+06>-6",
+"Asia/Almaty":"<+05>-5",
 "Asia/Amman":"<+03>-3",
 "Asia/Anadyr":"<+12>-12",
 "Asia/Aqtau":"<+05>-5",

webapp/src/components/BasePage.vue

@@ -48,15 +48,14 @@ export default defineComponent({
         showReload: { type: Boolean, required: false, default: false },
     },
     mounted() {
-        var self = this;
         console.log("init");
         PullToRefresh.init({
             mainElement: 'body', // above which element?
             instructionsPullToRefresh: this.$t('base.Pull'),
             instructionsReleaseToRefresh: this.$t('base.Release'),
             instructionsRefreshing: this.$t('base.Refreshing'),
-            onRefresh: function() {
+            onRefresh: () => {
-                self.$emit('reload');
+                this.$emit('reload');
             }
         });
     },

webapp/src/components/BootstrapAlert.vue

@@ -52,7 +52,7 @@ export default defineComponent({
             _countDownTimeout = undefined;
         };
 
-        var countDown = ref();
+        const countDown = ref();
         watch(() => props.modelValue, () => {
             countDown.value = parseCountDown(props.modelValue);
         });

webapp/src/components/FirmwareInfo.vue

@@ -87,10 +87,10 @@ export default defineComponent({
     },
     computed: {
         modelAllowVersionInfo: {
-            get(): any {
+            get(): boolean {
                 return !!this.allowVersionInfo;
             },
-            set(value: any) {
+            set(value: boolean) {
                 this.$emit('update:allowVersionInfo', value);
             },
         },

webapp/src/components/InputElement.vue

@@ -83,10 +83,12 @@ export default defineComponent({
     },
     computed: {
         model: {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
             get(): any {
                 if (this.type === 'checkbox') return !!this.modelValue;
                 return this.modelValue;
             },
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
             set(value: any) {
                 this.$emit('update:modelValue', value);
             },

webapp/src/components/InputSerial.vue

@@ -28,9 +28,11 @@ export default defineComponent({
     },
     computed: {
         model: {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
             get(): any {
                 return this.modelValue;
             },
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
             set(value: any) {
                 this.$emit('update:modelValue', value);
             },