re-factoring of HttpPowerMeter

Added ability to deal with local host names (mDNS), remove use of FirebasedJson to save ~20kB build size, some changes to PowerLimiter to avoid setting new inverter power limits when not needed (=current limit as reported by inverter is within hysteresis)
This commit is contained in:
Fribur 2024-01-04 16:20:32 -05:00
parent e9def28f3e
commit f5c69060f5
4 changed files with 183 additions and 163 deletions

View File

@ -10,17 +10,23 @@ public:
void init();
bool updateValues();
float getPower(int8_t phase);
bool httpRequest(const char* url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout,
char* response, size_t responseSize, char* error, size_t errorSize);
float getFloatValueByJsonPath(const char* jsonString, const char* jsonPath, float &value);
char httpPowerMeterError[256];
bool queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password,
const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath);
void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri);
private:
void extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri);
void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue);
HTTPClient httpClient;
float power[POWERMETER_MAX_PHASES];
String sha256(const String& data);
private:
float power[POWERMETER_MAX_PHASES];
HTTPClient httpClient;
String httpResponse;
bool httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& urlUri, Auth authType, const char* username,
const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath);
String extractParam(String& authReq, const String& param, const char delimit);
void getcNonce(char* cNounce);
String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter);
bool tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath);
void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue);
String sha256(const String& data);
};
extern HttpPowerMeterClass HttpPowerMeter;

View File

@ -3,11 +3,12 @@
#include "HttpPowerMeter.h"
#include "MessageOutput.h"
#include <WiFiClientSecure.h>
#include <FirebaseJson.h>
#include <ArduinoJson.h>//saves 20kB to not use FirebaseJson as ArduinoJson is used already elsewhere (e.g. in WebApi_powermeter)
#include <Crypto.h>
#include <SHA256.h>
#include <base64.h>
#include <memory>
#include <ESPmDNS.h>
void HttpPowerMeterClass::init()
{
@ -20,13 +21,14 @@ float HttpPowerMeterClass::getPower(int8_t phase)
bool HttpPowerMeterClass::updateValues()
{
const CONFIG_T& config = Configuration.get();
char response[2000],
errorMessage[256];
const CONFIG_T& config = Configuration.get();
for (uint8_t i = 0; i < POWERMETER_MAX_PHASES; i++) {
POWERMETER_HTTP_PHASE_CONFIG_T phaseConfig = config.PowerMeter.Http_Phase[i];
String urlProtocol;
String urlHostname;
String urlUri;
extractUrlComponents(phaseConfig.Url, urlProtocol, urlHostname, urlUri);
if (!phaseConfig.Enabled) {
power[i] = 0.0;
@ -34,33 +36,42 @@ bool HttpPowerMeterClass::updateValues()
}
if (i == 0 || config.PowerMeter.HttpIndividualRequests) {
if (httpRequest(phaseConfig.Url, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout,
response, sizeof(response), errorMessage, sizeof(errorMessage))) {
if (!getFloatValueByJsonPath(response, phaseConfig.JsonPath, power[i])) {
MessageOutput.printf("[HttpPowerMeter] Couldn't find a value with Json query \"%s\"\r\n", phaseConfig.JsonPath);
return false;
}
} else {
MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed. Error: %s\r\n",
i + 1, errorMessage);
if (!queryPhase(i, urlProtocol, urlHostname, urlUri, phaseConfig.AuthType, phaseConfig.Username, phaseConfig.Password, phaseConfig.HeaderKey, phaseConfig.HeaderValue, phaseConfig.Timeout,
phaseConfig.JsonPath)) {
MessageOutput.printf("[HttpPowerMeter] Getting the power of phase %d failed.\r\n", i + 1);
MessageOutput.printf("%s\r\n", httpPowerMeterError);
return false;
}
}
}
return true;
}
bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char* username, const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout,
char* response, size_t responseSize, char* error, size_t errorSize)
bool HttpPowerMeterClass::queryPhase(int phase, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username, const char* password,
const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath)
{
String urlProtocol;
String urlHostname;
String urlUri;
extractUrlComponents(url, urlProtocol, urlHostname, urlUri);
//hostByName in WiFiGeneric fails to resolve local names. issue described in
//https://github.com/espressif/arduino-esp32/issues/3822
//and in depth analyzed in https://github.com/espressif/esp-idf/issues/2507#issuecomment-761836300
//in conclusion: we cannot rely on httpClient.begin(*wifiClient, url) to resolve IP adresses.
//have to do it manually here. Feels Hacky...
IPAddress ipaddr((uint32_t)0);
//first check if the urlHostname is already an IP adress
if (!ipaddr.fromString(urlHostname))
{
//no it is not, so try to resolve the IP adress
const bool mdnsEnabled = Configuration.get().Mdns.Enabled;
if (!mdnsEnabled) {
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Enable mDNS in Network Settings"));
return false;
}
response[0] = '\0';
error[0] = '\0';
ipaddr = MDNS.queryHost(urlHostname);
if (ipaddr == INADDR_NONE){
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Error resolving url %s"), urlHostname.c_str());
return false;
}
}
// secureWifiClient MUST be created before HTTPClient
// see discussion: https://github.com/helgeerbe/OpenDTU-OnBattery/issues/381
@ -73,14 +84,21 @@ bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char
} else {
wifiClient = std::make_unique<WiFiClient>();
}
if (!httpClient.begin(*wifiClient, url)) {
snprintf_P(error, errorSize, "httpClient.begin(%s) failed", url);
return false;
return httpRequest(phase, *wifiClient, urlProtocol, ipaddr.toString(), uri, authType, username, password, httpHeader, httpValue, timeout, jsonPath);
}
bool HttpPowerMeterClass::httpRequest(int phase, WiFiClient &wifiClient, const String& urlProtocol, const String& urlHostname, const String& uri, Auth authType, const char* username,
const char* password, const char* httpHeader, const char* httpValue, uint32_t timeout, const char* jsonPath)
{
int port = 80;
if (urlProtocol == "https") {
port = 443;
}
prepareRequest(timeout, httpHeader, httpValue);
if(!httpClient.begin(wifiClient, urlHostname, port, uri)){
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s"), urlProtocol.c_str(), urlHostname.c_str());
return false;
}
prepareRequest(timeout, httpHeader, httpValue);
if (authType == Auth::digest) {
const char *headers[1] = {"WWW-Authenticate"};
httpClient.collectHeaders(headers, 1);
@ -92,111 +110,98 @@ bool HttpPowerMeterClass::httpRequest(const char* url, Auth authType, const char
auth.concat(base64::encode(authString));
httpClient.addHeader("Authorization", auth);
}
int httpCode = httpClient.GET();
if (httpCode == HTTP_CODE_UNAUTHORIZED && authType == Auth::digest) {
// Handle authentication challenge
char realm[256]; // Buffer to store the realm received from the server
char nonce[256]; // Buffer to store the nonce received from the server
if (httpClient.hasHeader("WWW-Authenticate")) {
String authHeader = httpClient.header("WWW-Authenticate");
if (authHeader.indexOf("Digest") != -1) {
int realmIndex = authHeader.indexOf("realm=\"");
int nonceIndex = authHeader.indexOf("nonce=\"");
if (realmIndex != -1 && nonceIndex != -1) {
int realmEndIndex = authHeader.indexOf("\"", realmIndex + 7);
int nonceEndIndex = authHeader.indexOf("\"", nonceIndex + 7);
if (realmEndIndex != -1 && nonceEndIndex != -1) {
authHeader.substring(realmIndex + 7, realmEndIndex).toCharArray(realm, sizeof(realm));
authHeader.substring(nonceIndex + 7, nonceEndIndex).toCharArray(nonce, sizeof(nonce));
}
}
String cnonce = String(random(1000)); // Generate client nonce
String str = username;
str += ":";
str += realm;
str += ":";
str += password;
String ha1 = sha256(str);
str = "GET:";
str += urlUri;
String ha2 = sha256(str);
str = ha1;
str += ":";
str += nonce;
str += ":00000001:";
str += cnonce;
str += ":auth:";
str += ha2;
String response = sha256(str);
String authorization = "Digest username=\"";
authorization += username;
authorization += "\", realm=\"";
authorization += realm;
authorization += "\", nonce=\"";
authorization += nonce;
authorization += "\", uri=\"";
authorization += urlUri;
authorization += "\", cnonce=\"";
authorization += cnonce;
authorization += "\", nc=00000001, qop=auth, response=\"";
authorization += response;
authorization += "\", algorithm=SHA-256";
httpClient.end();
if (!httpClient.begin(*wifiClient, url)) {
snprintf_P(error, errorSize, "httpClient.begin(%s) for digest auth failed", url);
return false;
}
prepareRequest(timeout, httpHeader, httpValue);
httpClient.addHeader("Authorization", authorization);
httpCode = httpClient.GET();
String authReq = httpClient.header("WWW-Authenticate");
String authorization = getDigestAuth(authReq, String(username), String(password), "GET", String(uri), 1);
httpClient.end();
if(!httpClient.begin(wifiClient, urlHostname, port, uri)){
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("httpClient.begin() failed for %s://%s using digest auth"), urlProtocol.c_str(), urlHostname.c_str());
return false;
}
}
prepareRequest(timeout, httpHeader, httpValue);
httpClient.addHeader("Authorization", authorization);
httpCode = httpClient.GET();
}
}
bool result = tryGetFloatValueForPhase(phase, httpCode, jsonPath);
httpClient.end();
return result;
}
String HttpPowerMeterClass::extractParam(String& authReq, const String& param, const char delimit) {
int _begin = authReq.indexOf(param);
if (_begin == -1) { return ""; }
return authReq.substring(_begin + param.length(), authReq.indexOf(delimit, _begin + param.length()));
}
void HttpPowerMeterClass::getcNonce(char* cNounce) {
static const char alphanum[] = "0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
auto len=sizeof(cNounce);
for (int i = 0; i < len; ++i) { cNounce[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; }
}
String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter) {
// extracting required parameters for RFC 2069 simpler Digest
String realm = extractParam(authReq, "realm=\"", '"');
String nonce = extractParam(authReq, "nonce=\"", '"');
char cNonce[8];
getcNonce(cNonce);
char nc[9];
snprintf(nc, sizeof(nc), "%08x", counter);
// parameters for the Digest
// sha256 of the user:realm:user
char h1Prep[sizeof(username)+sizeof(realm)+sizeof(password)+2];
snprintf(h1Prep, sizeof(h1Prep), "%s:%s:%s", username.c_str(),realm.c_str(), password.c_str());
String ha1 = sha256(h1Prep);
//sha256 of method:uri
char h2Prep[sizeof(method) + sizeof(uri) + 1];
snprintf(h2Prep, sizeof(h2Prep), "%s:%s", method.c_str(),uri.c_str());
String ha2 = sha256(h2Prep);
//md5 of h1:nonce:nc:cNonce:auth:h2
char responsePrep[sizeof(ha1)+sizeof(nc)+sizeof(cNonce)+4+sizeof(ha2) + 5];
snprintf(responsePrep, sizeof(responsePrep), "%s:%s:%s:%s:auth:%s", ha1.c_str(),nonce.c_str(), nc, cNonce,ha2.c_str());
String response = sha256(responsePrep);
//Final authorization String;
char authorization[17 + sizeof(username) + 10 + sizeof(realm) + 10 + sizeof(nonce) + 8 + sizeof(uri) + 34 + sizeof(nc) + 10 + sizeof(cNonce) + 13 + sizeof(response)];
snprintf(authorization, sizeof(authorization), "Digest username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", algorithm=SHA-256, qop=auth, nc=%s, cnonce=\"%s\", response=\"%s\"", username.c_str(), realm.c_str(), nonce.c_str(), uri.c_str(), nc, cNonce, response.c_str());
return authorization;
}
bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, int httpCode, const char* jsonPath)
{
bool success = false;
if (httpCode == HTTP_CODE_OK) {
String responseBody = httpClient.getString();
if (responseBody.length() > (responseSize - 1)) {
snprintf_P(error, errorSize, "Response too large! Response length: %d Body start: %s",
httpClient.getSize(), responseBody.c_str());
} else {
snprintf(response, responseSize, responseBody.c_str());
httpResponse = httpClient.getString(); //very unfortunate that we cannot parse WifiClient stream directly
StaticJsonDocument<2048> json; //however creating these allocations on stack should be fine to avoid heap fragmentation
deserializeJson(json, httpResponse);
if(!json.containsKey(jsonPath))
{
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("[HttpPowerMeter] Couldn't find a value for phase %i with Json query \"%s\""), phase, jsonPath);
}else {
power[phase] = json[jsonPath].as<long>();
//MessageOutput.printf("Power for Phase %i: %5.2fW\r\n", phase, power[phase]);
success = true;
}
} else if (httpCode <= 0) {
snprintf_P(error, errorSize, "Error(%s): %s", url, httpClient.errorToString(httpCode).c_str());
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("HTTP Error %s"), httpClient.errorToString(httpCode).c_str());
} else if (httpCode != HTTP_CODE_OK) {
snprintf_P(error, errorSize, "Bad HTTP code: %d", httpCode);
}
httpClient.end();
if (error[0] != '\0') {
return false;
}
return true;
snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("Bad HTTP code: %d"), httpCode);
}
return success;
}
float HttpPowerMeterClass::getFloatValueByJsonPath(const char* jsonString, const char* jsonPath, float& value)
{
FirebaseJson firebaseJson;
firebaseJson.setJsonData(jsonString);
FirebaseJsonData firebaseJsonResult;
if (!firebaseJson.get(firebaseJsonResult, jsonPath)) {
return false;
}
value = firebaseJsonResult.to<float>();
firebaseJson.clear();
return true;
}
void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri) {
void HttpPowerMeterClass::extractUrlComponents(const String& url, String& protocol, String& hostname, String& uri) {
// Find protocol delimiter
int protocolEndIndex = url.indexOf(":");
if (protocolEndIndex != -1) {
@ -247,7 +252,6 @@ String HttpPowerMeterClass::sha256(const String& data) {
return hashStr;
}
void HttpPowerMeterClass::prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue) {
httpClient.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
httpClient.setUserAgent("OpenDTU-OnBattery");

View File

@ -187,7 +187,8 @@ void PowerLimiterClass::loop()
// a calculated power limit will always be limited to the reported
// device's max power. that upper limit is only known after the first
// DevInfoSimpleCommand succeeded.
if (_inverter->DevInfo()->getMaxPower() <= 0) {
auto maxPower = _inverter->DevInfo()->getMaxPower();
if (maxPower <= 0) {
return announceStatus(Status::InverterDevInfoPending);
}
@ -199,12 +200,13 @@ void PowerLimiterClass::loop()
// the normal mode of operation requires a valid
// power meter reading to calculate a power limit
if (!config.PowerMeter.Enabled) {
shutdown(Status::PowerMeterDisabled);
//instead of shutting down completelty, how about setting alternativly to a save "low production" mode?
//Could be usefull when PowerMeter fails but we know for sure house consumption will never fall below a certain limit (say 200W)
shutdown(Status::PowerMeterDisabled);
return;
}
if (millis() - PowerMeter.getLastPowerMeterUpdate() > (30 * 1000)) {
shutdown(Status::PowerMeterTimeout);
return;
}
@ -222,6 +224,7 @@ void PowerLimiterClass::loop()
if (_inverter->Statistics()->getLastUpdate() <= settlingEnd) {
return announceStatus(Status::InverterStatsPending);
}
if (PowerMeter.getLastPowerMeterUpdate() <= settlingEnd) {
return announceStatus(Status::PowerMeterPending);
@ -545,8 +548,8 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr<InverterAbstract> inver
dcTotalChnls, dcProdChnls);
effPowerLimit = round(effPowerLimit * static_cast<float>(dcTotalChnls) / dcProdChnls);
}
effPowerLimit = std::min<int32_t>(effPowerLimit, inverter->DevInfo()->getMaxPower());
auto maxPower = inverter->DevInfo()->getMaxPower();
effPowerLimit = std::min<int32_t>(effPowerLimit, maxPower);
// Check if the new value is within the limits of the hysteresis
auto diff = std::abs(effPowerLimit - _lastRequestedPowerLimit);
@ -556,16 +559,23 @@ bool PowerLimiterClass::setNewPowerLimit(std::shared_ptr<InverterAbstract> inver
// staleness in case a power limit update was not received by the inverter.
auto ageMillis = millis() - _lastPowerLimitMillis;
if (diff < hysteresis && ageMillis < 60 * 1000) {
//instead pushing limit to inverter every 60 seconds no matter what,
//why not query instead the currenty configured limit...and do nothing if not needed
int currentLimit = round(inverter->SystemConfigPara()->getLimitPercent() * maxPower / 100);
auto currentDiff = std::abs(effPowerLimit - currentLimit );
if (diff < hysteresis && currentDiff < hysteresis ){
//if (diff < hysteresis && ageMillis < 60 * 1000) {
//MessageOutput.printf("Keep limit: %d W, current limit %d W\r\n", effPowerLimit, currentLimit);
if (_verboseLogging) {
MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms\r\n",
newPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis);
MessageOutput.printf("[DPL::setNewPowerLimit] Keep current limit. (new calculated: %d W, last limit: %d W, diff: %d W, hysteresis: %d W, age: %ld ms)\r\n",
effPowerLimit, _lastRequestedPowerLimit, diff, hysteresis, ageMillis);
}
return false;
}
//if we end up here, it we will set new limit
if (_verboseLogging) {
MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, (re-)sending limit: %d W\r\n",
MessageOutput.printf("[DPL::setNewPowerLimit] requested: %d W, sending limit: %d W\r\n",
newPowerLimit, effPowerLimit);
}

View File

@ -203,10 +203,11 @@ void WebApiPowerMeterClass::onAdminPost(AsyncWebServerRequest* request)
response->setLength();
request->send(response);
yield();
delay(1000);
yield();
ESP.restart();
// why reboot..WebApi_powerlimiter is also not rebooting
// yield();
// delay(1000);
// yield();
// ESP.restart();
}
void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request)
@ -254,25 +255,24 @@ void WebApiPowerMeterClass::onTestHttpRequest(AsyncWebServerRequest* request)
return;
}
char powerMeterResponse[2000],
errorMessage[256];
char response[200];
if (HttpPowerMeter.httpRequest(root[F("url")].as<String>().c_str(),
char response[256];
String urlProtocol;
String urlHostname;
String urlUri;
HttpPowerMeter.extractUrlComponents(root[F("url")].as<String>().c_str(), urlProtocol, urlHostname, urlUri);
int phase = 0;//"absuing" index 0 of the float power[3] in HttpPowerMeter to store the result
if (HttpPowerMeter.queryPhase(phase, urlProtocol, urlHostname, urlUri,
root[F("auth_type")].as<Auth>(), root[F("username")].as<String>().c_str(), root[F("password")].as<String>().c_str(),
root[F("header_key")].as<String>().c_str(), root[F("header_value")].as<String>().c_str(), root[F("timeout")].as<uint16_t>(),
powerMeterResponse, sizeof(powerMeterResponse), errorMessage, sizeof(errorMessage))) {
float power;
if (HttpPowerMeter.getFloatValueByJsonPath(powerMeterResponse,
root[F("json_path")].as<String>().c_str(), power)) {
retMsg[F("type")] = F("success");
snprintf_P(response, sizeof(response), "Success! Power: %5.2fW", power);
} else {
snprintf_P(response, sizeof(response), "Error: Could not find value for JSON path!");
}
root[F("json_path")].as<String>().c_str())) {
retMsg[F("type")] = F("success");
snprintf_P(response, sizeof(response), "Success! Power: %5.2fW", HttpPowerMeter.getPower(phase + 1));
} else {
snprintf_P(response, sizeof(response), errorMessage);
snprintf_P(response, sizeof(response), "%s", HttpPowerMeter.httpPowerMeterError);
}
retMsg[F("message")] = F(response);