From 74330a56173eb12ae66cc8bf22bebde33c7bbf95 Mon Sep 17 00:00:00 2001
From: Bernhard Kirchen <schlimmchen@posteo.net>
Date: Thu, 18 Apr 2024 20:03:49 +0200
Subject: [PATCH] Feature: restart unresponsive inverter

we found that the inverter sometimes stops responding to commands,
especially to the "start producing" command. we now count the number of
consecutive timeouts when trying to send a new limit or power state
commands. after two timeouts were recorded, every additional timeout
will send a restart command to the inverter.

as a last resort, if the counter keeps climbing, the DTU is restarted.

notice that this only targets unresponsive inverters which are
reachable. unreachable inverters are not restarted and do not cause a
DTU reboot. this is important for solar-driven inverters, which are
unreachable during the night. the DPL will not calculate a new limit and
hence the updateInverter() method will do nothing while the target
inverter is unreachable.

publish the timeout counter to MQTT for monitoring purposes.
---
 include/PowerLimiter.h         |  2 ++
 src/MqttHandlePowerLimiter.cpp |  2 ++
 src/PowerLimiter.cpp           | 29 ++++++++++++++++++++++++++++-
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/include/PowerLimiter.h b/include/PowerLimiter.h
index 043a7fc1..c5727fe5 100644
--- a/include/PowerLimiter.h
+++ b/include/PowerLimiter.h
@@ -41,6 +41,7 @@ public:
     };
 
     void init(Scheduler& scheduler);
+    uint8_t getInverterUpdateTimeouts() const { return _inverterUpdateTimeouts; }
     uint8_t getPowerLimiterState();
     int32_t getLastRequestedPowerLimit() { return _lastRequestedPowerLimit; }
 
@@ -77,6 +78,7 @@ private:
     uint32_t _nextCalculateCheck = 5000; // time in millis for next NTP check to calulate restart
     bool _fullSolarPassThroughEnabled = false;
     bool _verboseLogging = true;
+    uint8_t _inverterUpdateTimeouts = 0;
 
     frozen::string const& getStatusText(Status status);
     void announceStatus(Status status);
diff --git a/src/MqttHandlePowerLimiter.cpp b/src/MqttHandlePowerLimiter.cpp
index 411fa3f1..95f90db2 100644
--- a/src/MqttHandlePowerLimiter.cpp
+++ b/src/MqttHandlePowerLimiter.cpp
@@ -76,6 +76,8 @@ void MqttHandlePowerLimiterClass::loop()
     auto val = static_cast<unsigned>(PowerLimiter.getMode());
     MqttSettings.publish("powerlimiter/status/mode", String(val));
 
+    MqttSettings.publish("powerlimiter/status/inverter_update_timeouts", String(PowerLimiter.getInverterUpdateTimeouts()));
+
     // no thresholds are relevant for setups without a battery
     if (config.PowerLimiter.IsInverterSolarPowered) { return; }
 
diff --git a/src/PowerLimiter.cpp b/src/PowerLimiter.cpp
index 6ebbf309..7ce0d8bd 100644
--- a/src/PowerLimiter.cpp
+++ b/src/PowerLimiter.cpp
@@ -3,6 +3,7 @@
  * Copyright (C) 2022 Thomas Basler and others
  */
 
+#include "Utils.h"
 #include "Battery.h"
 #include "PowerMeter.h"
 #include "PowerLimiter.h"
@@ -537,15 +538,39 @@ bool PowerLimiterClass::updateInverter()
 
     if (nullptr == _inverter) { return reset(); }
 
+    // do not reset _inverterUpdateTimeouts below if no state change requested
+    if (!_oTargetPowerState.has_value() && !_oTargetPowerLimitWatts.has_value()) {
+        return reset();
+    }
+
     if (!_oUpdateStartMillis.has_value()) {
         _oUpdateStartMillis = millis();
     }
 
     if ((millis() - *_oUpdateStartMillis) > 30 * 1000) {
-        MessageOutput.printf("[DPL::updateInverter] timeout, "
+        ++_inverterUpdateTimeouts;
+        MessageOutput.printf("[DPL::updateInverter] timeout (%d in succession), "
                 "state transition pending: %s, limit pending: %s\r\n",
+                _inverterUpdateTimeouts,
                 (_oTargetPowerState.has_value()?"yes":"no"),
                 (_oTargetPowerLimitWatts.has_value()?"yes":"no"));
+
+        // NOTE that this is not always 5 minutes, since this counts timeouts,
+        // not absolute time. after any timeout, an update cycle ends. a new
+        // timeout can only happen after starting a new update cycle, which in
+        // turn is only started if the DPL did calculate a new limit, which in
+        // turn does not happen while the inverter is unreachable, no matter
+        // how long (a whole night) that might be.
+        if (_inverterUpdateTimeouts >= 10) {
+            MessageOutput.println("[DPL::loop] issuing inverter restart command after update timed out repeatedly");
+            _inverter->sendRestartControlRequest();
+        }
+
+        if (_inverterUpdateTimeouts >= 20) {
+            MessageOutput.println("[DPL::loop] restarting system since inverter is unresponsive");
+            Utils::restartDtu();
+        }
+
         return reset();
     }
 
@@ -648,6 +673,8 @@ bool PowerLimiterClass::updateInverter()
     // enable power production only after setting the desired limit
     if (switchPowerState(true)) { return true; }
 
+    _inverterUpdateTimeouts = 0;
+
     return reset();
 }