From 1dd64a57fdd76e96b2cdcfea10b1f09b5d5ba49e Mon Sep 17 00:00:00 2001 From: Bernhard Kirchen Date: Sun, 5 May 2024 22:05:08 +0200 Subject: [PATCH 1/3] remove FirebaseJson lib from firmware (save 17.5k of flash) we used this library solely to interpret the answer of an HTTP web server as JSON and find a particular value using a path expression in the HTTP power meter implementation. since we ran out of flash memory on non-S3 ESP32, we need to cut some corners. removing FirebaseJson is the last low-hanging fruit that we currently know of. we can get rid of it by using ArduinoJson (which is already integral part of the firmware) and implementing a custom logic to extract a value based on a path expression. other than the FirebaseJson path "finder", the new implementation only knows how to access sub-keys delimited by a forward slash. in particular, accessing array members is not supported any more. I am hoping that this is simply not an issue. if so, we will have users complaining and we can add this functionality in a later release. --- include/HttpPowerMeter.h | 2 +- platformio.ini | 1 - src/HttpPowerMeter.cpp | 41 +++++++++++++++++++----- webapp/src/locales/de.json | 2 +- webapp/src/locales/en.json | 2 +- webapp/src/views/PowerMeterAdminView.vue | 6 ++-- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/include/HttpPowerMeter.h b/include/HttpPowerMeter.h index 25e627cd..8f703bba 100644 --- a/include/HttpPowerMeter.h +++ b/include/HttpPowerMeter.h @@ -26,7 +26,7 @@ private: String extractParam(String& authReq, const String& param, const char delimit); String getcNonce(const int len); String getDigestAuth(String& authReq, const String& username, const String& password, const String& method, const String& uri, unsigned int counter); - bool tryGetFloatValueForPhase(int phase, const char* jsonPath, Unit_t unit, bool signInverted); + bool tryGetFloatValueForPhase(int phase, String jsonPath, Unit_t unit, bool signInverted); void prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue); String sha256(const String& data); }; diff --git a/platformio.ini b/platformio.ini index b8b248a6..f5bab679 100644 --- a/platformio.ini +++ b/platformio.ini @@ -47,7 +47,6 @@ lib_deps = https://github.com/coryjfowler/MCP_CAN_lib plerup/EspSoftwareSerial @ ^8.0.1 https://github.com/dok-net/ghostl @ ^1.0.1 - mobizt/FirebaseJson @ ^3.0.6 rweather/Crypto@^0.4.0 extra_scripts = diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index cdb1253f..429d317e 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -3,7 +3,7 @@ #include "HttpPowerMeter.h" #include "MessageOutput.h" #include -#include +#include #include #include #include @@ -219,18 +219,43 @@ String HttpPowerMeterClass::getDigestAuth(String& authReq, const String& usernam return authorization; } -bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, const char* jsonPath, Unit_t unit, bool signInverted) +bool HttpPowerMeterClass::tryGetFloatValueForPhase(int phase, String jsonPath, Unit_t unit, bool signInverted) { - FirebaseJson json; - json.setJsonData(httpResponse); - FirebaseJsonData value; - if (!json.get(value, jsonPath)) { - snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), PSTR("[HttpPowerMeter] Couldn't find a value for phase %i with Json query \"%s\""), phase, jsonPath); + JsonDocument root; + const DeserializationError error = deserializeJson(root, httpResponse); + if (error) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), + PSTR("[HttpPowerMeter] Unable to parse server response as JSON")); + return false; + } + + constexpr char delimiter = '/'; + int start = 0; + int end = jsonPath.indexOf(delimiter); + auto value = root.as(); + + // NOTE: "Because ArduinoJson implements the Null Object Pattern, it is + // always safe to read the object: if the key doesn't exist, it returns an + // empty value." + while (end != -1) { + String key = jsonPath.substring(start, end); + value = value[key]; + start = end + 1; + end = jsonPath.indexOf(delimiter, start); + } + + String lastKey = jsonPath.substring(start); + value = value[lastKey]; + + if (value.isNull()) { + snprintf_P(httpPowerMeterError, sizeof(httpPowerMeterError), + PSTR("[HttpPowerMeter] Unable to find a value for phase %i with JSON path \"%s\""), + phase+1, jsonPath.c_str()); return false; } // this value is supposed to be in Watts and positive if energy is consumed. - power[phase] = value.to(); + power[phase] = value.as(); switch (unit) { case Unit_t::MilliWatts: diff --git a/webapp/src/locales/de.json b/webapp/src/locales/de.json index c9e363d3..2be16cef 100644 --- a/webapp/src/locales/de.json +++ b/webapp/src/locales/de.json @@ -577,7 +577,7 @@ "httpHeaderKeyDescription": "Ein individueller HTTP request header kann hier definiert werden. Das kann z.B. verwendet werden um einen eigenen Authorization header mitzugeben.", "httpHeaderValue": "Optional: HTTP request header - Wert", "httpJsonPath": "JSON Pfad", - "httpJsonPathDescription": "JSON Pfad um den Leistungswert zu finden. Es verwendet die Selektions-Syntax von mobizt/FirebaseJson. Beispiele gibt es oben.", + "httpJsonPathDescription": "Anwendungsspezifischer JSON-Pfad um den Leistungswert in the HTTP(S) Antwort zu finden, z.B. 'power/total/watts' oder nur 'total'.", "httpUnit": "Einheit", "httpSignInverted": "Vorzeichen umkehren", "httpSignInvertedHint": "Positive Werte werden als Leistungsabnahme aus dem Netz interpretiert. Diese Option muss aktiviert werden, wenn das Vorzeichen des Wertes die gegenteilige Bedeutung hat.", diff --git a/webapp/src/locales/en.json b/webapp/src/locales/en.json index c779dc43..ebfc0a37 100644 --- a/webapp/src/locales/en.json +++ b/webapp/src/locales/en.json @@ -582,7 +582,7 @@ "httpHeaderKeyDescription": "A custom HTTP request header can be defined. Might be useful if you have to send something like a custom Authorization header.", "httpHeaderValue": "Optional: HTTP request header - Value", "httpJsonPath": "JSON path", - "httpJsonPathDescription": "JSON path to find the power value in the response. This uses the JSON path query syntax from mobizt/FirebaseJson. See above for examples.", + "httpJsonPathDescription": "Application specific JSON path to find the power value in the HTTP(S) response, e.g., 'power/total/watts' or simply 'total'.", "httpUnit": "Unit", "httpSignInverted": "Change Sign", "httpSignInvertedHint": "Is is expected that positive values denote power usage from the grid. Check this option if the sign of this value has the opposite meaning.", diff --git a/webapp/src/views/PowerMeterAdminView.vue b/webapp/src/views/PowerMeterAdminView.vue index 8371d465..6744e08f 100644 --- a/webapp/src/views/PowerMeterAdminView.vue +++ b/webapp/src/views/PowerMeterAdminView.vue @@ -116,11 +116,9 @@

JSON path examples:

    -
  • total_power - { "othervalue": "blah", "total_power": 123.4 }
    • -
  • testarray/[2]/myvalue - { "testarray": [ {}, { "power": 123.4 } ] }
    • +
  • power/total/watts - Finds 123.4 in { "power": { "phase1": { "factor": 0.98, "watts": 42 }, "total": { "watts": 123.4 } } }
    • +
  • total - Finds 123.4 in { "othervalue": 66, "total": 123.4 }
- - More info: https://github.com/mobizt/FirebaseJson Date: Sun, 5 May 2024 23:46:20 +0200 Subject: [PATCH 2/3] remove rweather/Crypt lib from firmware (save 2.1k of flash) mbedtls is already integral part of the firmware. use it in favor of rweather/Crypto library to calculate a sha256 checksum of a string, as used in the HTTP power meter implementation. --- platformio.ini | 1 - src/HttpPowerMeter.cpp | 30 +++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/platformio.ini b/platformio.ini index f5bab679..498fa5ba 100644 --- a/platformio.ini +++ b/platformio.ini @@ -47,7 +47,6 @@ lib_deps = https://github.com/coryjfowler/MCP_CAN_lib plerup/EspSoftwareSerial @ ^8.0.1 https://github.com/dok-net/ghostl @ ^1.0.1 - rweather/Crypto@^0.4.0 extra_scripts = pre:pio-scripts/auto_firmware_version.py diff --git a/src/HttpPowerMeter.cpp b/src/HttpPowerMeter.cpp index 429d317e..dbe9aa79 100644 --- a/src/HttpPowerMeter.cpp +++ b/src/HttpPowerMeter.cpp @@ -4,8 +4,7 @@ #include "MessageOutput.h" #include #include -#include -#include +#include "mbedtls/sha256.h" #include #include #include @@ -324,27 +323,24 @@ bool HttpPowerMeterClass::extractUrlComponents(String url, String& _protocol, St return true; } -#define HASH_SIZE 32 - String HttpPowerMeterClass::sha256(const String& data) { - SHA256 sha256; - uint8_t hash[HASH_SIZE]; + uint8_t hash[32]; - sha256.reset(); - sha256.update(data.c_str(), data.length()); - sha256.finalize(hash, HASH_SIZE); + mbedtls_sha256_context ctx; + mbedtls_sha256_init(&ctx); + mbedtls_sha256_starts(&ctx, 0); // select SHA256 + mbedtls_sha256_update(&ctx, reinterpret_cast(data.c_str()), data.length()); + mbedtls_sha256_finish(&ctx, hash); + mbedtls_sha256_free(&ctx); - String hashStr = ""; - for (int i = 0; i < HASH_SIZE; i++) { - String hex = String(hash[i], HEX); - if (hex.length() == 1) { - hashStr += "0"; - } - hashStr += hex; + char res[sizeof(hash) * 2 + 1]; + for (int i = 0; i < sizeof(hash); i++) { + snprintf(res + (i*2), sizeof(res) - (i*2), "%02x", hash[i]); } - return hashStr; + return res; } + void HttpPowerMeterClass::prepareRequest(uint32_t timeout, const char* httpHeader, const char* httpValue) { httpClient.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); httpClient.setUserAgent("OpenDTU-OnBattery"); From 2f7e1f3f7038d2a62e8e00a2e1ab6e10efa08b8a Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 6 May 2024 11:12:25 +0200 Subject: [PATCH 3/3] Fix: solar_passthrough_losses setting persistence (#957) Due to a typo the value of `solar_passtrough_losses` was lost after a restart. --- src/Configuration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Configuration.cpp b/src/Configuration.cpp index 70d0ba81..d23ac9ba 100644 --- a/src/Configuration.cpp +++ b/src/Configuration.cpp @@ -179,7 +179,7 @@ bool ConfigurationClass::write() powerlimiter["enabled"] = config.PowerLimiter.Enabled; powerlimiter["verbose_logging"] = config.PowerLimiter.VerboseLogging; powerlimiter["solar_passtrough_enabled"] = config.PowerLimiter.SolarPassThroughEnabled; - powerlimiter["solar_passtrough_losses"] = config.PowerLimiter.SolarPassThroughLosses; + powerlimiter["solar_passthrough_losses"] = config.PowerLimiter.SolarPassThroughLosses; powerlimiter["battery_always_use_at_night"] = config.PowerLimiter.BatteryAlwaysUseAtNight; powerlimiter["interval"] = config.PowerLimiter.Interval; powerlimiter["is_inverter_behind_powermeter"] = config.PowerLimiter.IsInverterBehindPowerMeter; @@ -438,7 +438,7 @@ bool ConfigurationClass::read() config.PowerLimiter.Enabled = powerlimiter["enabled"] | POWERLIMITER_ENABLED; config.PowerLimiter.VerboseLogging = powerlimiter["verbose_logging"] | VERBOSE_LOGGING; config.PowerLimiter.SolarPassThroughEnabled = powerlimiter["solar_passtrough_enabled"] | POWERLIMITER_SOLAR_PASSTHROUGH_ENABLED; - config.PowerLimiter.SolarPassThroughLosses = powerlimiter["solar_passthrough_losses"] | POWERLIMITER_SOLAR_PASSTHROUGH_LOSSES; + config.PowerLimiter.SolarPassThroughLosses = powerlimiter["solar_passthrough_losses"] | powerlimiter["solar_passtrough_losses"] | POWERLIMITER_SOLAR_PASSTHROUGH_LOSSES; // solar_passthrough_losses was previously saved as solar_passtrough_losses. Be nice and also try mistyped key. config.PowerLimiter.BatteryAlwaysUseAtNight = powerlimiter["battery_always_use_at_night"] | POWERLIMITER_BATTERY_ALWAYS_USE_AT_NIGHT; if (powerlimiter["battery_drain_strategy"].as() == 1) { config.PowerLimiter.BatteryAlwaysUseAtNight = true; } // convert legacy setting config.PowerLimiter.Interval = powerlimiter["interval"] | POWERLIMITER_INTERVAL;