Merge remote-tracking branch 'origin/Database' into Database

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "C_Cpp.clang_format_style": "WebKit"
-}
+    "C_Cpp.clang_format_style": "WebKit",
+    "cmake.sourceDirectory": "C:/git/OpenDTU-Database/.pio/libdeps/generic_esp32/ArduinoJson/src"
+}

README.md

@@ -1,12 +1,35 @@
-# OpenDTU
+# OpenDTU-Database-Database
+# OpenDTU-Database
+
+One year OpenDTU-Database
+![Image](docs/screenshots/Screenshot_2024-05-23_131208.png)
 
 [![OpenDTU Build](https://github.com/tbnobody/OpenDTU/actions/workflows/build.yml/badge.svg)](https://github.com/tbnobody/OpenDTU/actions/workflows/build.yml)
 [![cpplint](https://github.com/tbnobody/OpenDTU/actions/workflows/cpplint.yml/badge.svg)](https://github.com/tbnobody/OpenDTU/actions/workflows/cpplint.yml)
 [![Yarn Linting](https://github.com/tbnobody/OpenDTU/actions/workflows/yarnlint.yml/badge.svg)](https://github.com/tbnobody/OpenDTU/actions/workflows/yarnlint.yml)
 
-## !! IMPORTANT UPGRADE NOTES !!
+OpenDTU-Database adds an ESP32 LittleFS Database and two energy charts, a column chart of the last 25 hours and a full calendar chart.
 
-If you are upgrading from a version before 15.03.2023 you have to upgrade the partition table of the ESP32. Please follow the [this](docs/UpgradePartition.md) documentation!
+![Screenshot](https://github.com/RaBa64/OpenDTU/blob/Database/docs/screenshots/23_Database.png)
+
+OpenDTU-Database adds an ESP32 LittleFS Database and two energy charts, a column chart of the last 25 hours and a full calendar chart.
+
+There are 3 new APIs available, returning JSON strings:
+
+| API               | returned values |
+|-------------------|-----------------|
+| /api/database     | returns all recored data points from the database with total energy value |
+| /api/databaseHour | returns the energy per hour for the last 25 hours |
+| /api/databaseDay  | returns the energy for each day |
+
+Each data point has the following format:
+[ _year (00-99)_, _month (1-12)_, _day (1-31)_, _hour (0-23)_, _energy (Wh)_ ]
+
+Example: [23,6,30,15,132.995605]
+
+The 192KB LittleFS in OpenDTU can store around 6 years of data, because each data point needs only 8 bytes of memory.
+
+![Screenshot](docs/screenshots/23_Database.png)
 
 ## Background
 

docs/screenshots/README.md

@@ -89,3 +89,7 @@ here are some screenshots of OpenDTU's web interface.
 ***
 
 ![Console](18_Console.png)
+
+***
+
+![Console](23_Database.png)

include/WebApi.h

@@ -23,6 +23,7 @@
 #include "WebApi_webapp.h"
 #include "WebApi_ws_console.h"
 #include "WebApi_ws_live.h"
+#include "WebApi_database.h"
 #include <AsyncJson.h>
 #include <ESPAsyncWebServer.h>
 #include <TaskSchedulerDeclarations.h>
@@ -68,6 +69,7 @@ private:
     WebApiWebappClass _webApiWebapp;
     WebApiWsConsoleClass _webApiWsConsole;
     WebApiWsLiveClass _webApiWsLive;
+    WebApiDatabaseClass _webApiWsDatabase;
 };
 
 extern WebApiClass WebApi;

include/WebApi_database.h

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+#include <Arduino.h>
+#include <ESPAsyncWebServer.h>
+#include <TaskSchedulerDeclarations.h>
+
+#define DATABASE_FILENAME "/database.bin"
+
+class WebApiDatabaseClass {
+public:
+    WebApiDatabaseClass();
+    void init(AsyncWebServer& server, Scheduler& scheduler);
+    bool write(float energy);
+
+    struct pvData {
+        uint8_t	tm_year;
+        uint8_t tm_mon;
+        uint8_t tm_mday;
+        uint8_t tm_hour;
+        float   energy;
+    };
+
+private:
+    void onDatabase(AsyncWebServerRequest* request);
+    void onDatabaseHour(AsyncWebServerRequest* request);
+    void onDatabaseDay(AsyncWebServerRequest* request);
+    static size_t readchunk(uint8_t* buffer, size_t maxLen, size_t index);
+    static size_t readchunk_log(uint8_t* buffer, size_t maxLen, size_t index);
+    static size_t readchunkHour(uint8_t* buffer, size_t maxLen, size_t index);
+    static size_t readchunkDay(uint8_t* buffer, size_t maxLen, size_t index);
+
+    AsyncWebServer* _server;
+
+    Task _sendDataTask;
+    void sendDataTaskCb();
+};

src/WebApi.cpp

@@ -36,6 +36,7 @@ void WebApiClass::init(Scheduler& scheduler)
     _webApiWebapp.init(_server, scheduler);
     _webApiWsConsole.init(_server, scheduler);
     _webApiWsLive.init(_server, scheduler);
+    _webApiWsDatabase.init(_server, scheduler);
 
     _server.begin();
 }

src/WebApi_database.cpp

@@ -0,0 +1,323 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "WebApi_database.h"
+#include "WebApi.h"
+#include "Datastore.h"
+#include "MessageOutput.h"
+#include "defaults.h"
+#include <Arduino.h>
+#include <AsyncJson.h>
+#include <LittleFS.h>
+
+
+WebApiDatabaseClass::WebApiDatabaseClass()
+    : _sendDataTask(1 * TASK_MINUTE, TASK_FOREVER, std::bind(&WebApiDatabaseClass::sendDataTaskCb, this))
+{
+}
+
+void WebApiDatabaseClass::init(AsyncWebServer& server, Scheduler& scheduler)
+{
+    using std::placeholders::_1;
+
+    server.on("/api/database", HTTP_GET, std::bind(&WebApiDatabaseClass::onDatabase, this, _1));
+    server.on("/api/databaseHour", HTTP_GET, std::bind(&WebApiDatabaseClass::onDatabaseHour, this, _1));
+    server.on("/api/databaseDay", HTTP_GET, std::bind(&WebApiDatabaseClass::onDatabaseDay, this, _1));
+
+    scheduler.addTask(_sendDataTask);
+    _sendDataTask.enable();
+}
+
+void WebApiDatabaseClass::sendDataTaskCb()
+{
+    if (!Hoymiles.isAllRadioIdle()) {
+        return;
+    }
+    write(Datastore.getTotalAcYieldTotalEnabled()); // write value to database
+}
+
+bool WebApiDatabaseClass::write(float energy)
+{
+    static uint8_t old_hour = 255;
+    static float old_energy = 0.0;
+
+    // LittleFS.remove(DATABASE_FILENAME);
+
+    // MessageOutput.println(energy, 6);
+
+    struct tm timeinfo;
+    if (!getLocalTime(&timeinfo, 5)) {
+        return false;
+    }
+    if (timeinfo.tm_hour == old_hour) // must be new hour
+        return (false);
+    if (old_hour == 255) { // don't write to database after reboot
+        old_hour = timeinfo.tm_hour;
+        return (false);
+    }
+    // MessageOutput.println("Next hour.");
+    if (energy <= old_energy) // enery must have increased
+        return (false);
+    // MessageOutput.println("Energy difference > 0");
+
+    struct pvData d;
+    d.tm_hour = timeinfo.tm_hour - 1;
+    old_hour = timeinfo.tm_hour;
+    d.tm_year = timeinfo.tm_year - 100; // year counting from 2000
+    d.tm_mon = timeinfo.tm_mon + 1;
+    d.tm_mday = timeinfo.tm_mday;
+    d.energy = old_energy = energy;
+
+    // create database file if it does not exist
+    // if (!LittleFS.exists(DATABASE_FILENAME)) {
+    //    MessageOutput.println("Database file does not exist.");
+    //    File f = LittleFS.open(DATABASE_FILENAME, "w", true);
+    //    f.flush();
+    //    f.close();
+    //    MessageOutput.println("New database file created.");
+    //}
+
+    File f = LittleFS.open(DATABASE_FILENAME, "a", true);
+    if (!f) {
+        MessageOutput.println("Failed to append the database.");
+        return (false);
+    }
+    f.write((const uint8_t*)&d, sizeof(pvData));
+    f.close();
+    // MessageOutput.println("Write data point.");
+    return (true);
+}
+
+// read chunk from database
+size_t WebApiDatabaseClass::readchunk(uint8_t* buffer, size_t maxLen, size_t index)
+{
+    static bool first = true;
+    static bool last = false;
+    static File f;
+    uint8_t* pr = buffer;
+    uint8_t* pre = pr + maxLen - 50;
+    size_t r;
+    struct pvData d;
+
+    if (first) {
+        f = LittleFS.open(DATABASE_FILENAME, "r", false);
+        if (!f) {
+            return (0);
+        }
+        *pr++ = '[';
+    }
+    while (true) {
+        r = f.read(reinterpret_cast<uint8_t*>(&d), sizeof(pvData)); // read from database
+        if (r <= 0) {
+            if (last) {
+                f.close();
+                first = true;
+                last = false;
+                return (0); // end transmission
+            }
+            last = true;
+            *pr++ = ']';
+            return (pr - buffer); // last chunk
+        }
+        if (first) {
+            first = false;
+        } else {
+            *pr++ = ',';
+        }
+        int len = snprintf(reinterpret_cast<char*>(pr), maxLen, "[%d,%d,%d,%d,%f]",
+            d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.energy * 1e3);
+        if (len >= 0) {
+            pr += len;
+        }
+        if (pr >= pre)
+            return (pr - buffer); // buffer full, return number of chars
+    }
+}
+
+size_t WebApiDatabaseClass::readchunk_log(uint8_t* buffer, size_t maxLen, size_t index)
+{
+    size_t x = readchunk(buffer, maxLen, index);
+    //MessageOutput.println("----------");
+    //MessageOutput.println(maxLen);
+    //MessageOutput.println(x);
+    return (x);
+}
+
+// read chunk from database for the last 25 hours
+size_t WebApiDatabaseClass::readchunkHour(uint8_t* buffer, size_t maxLen, size_t index)
+{
+    static bool first = true;
+    static bool last = false;
+    static bool valid = false;
+    static float oldenergy = 0.0;
+    static File f;
+    static bool fileopen = false;
+    union datehour {
+        uint32_t dh;
+        uint8_t dd[4];
+    };
+    static datehour startdate;
+    uint8_t* pr = buffer;
+    uint8_t* pre = pr + maxLen - 50;
+    size_t r;
+    struct pvData d;
+
+    if (!fileopen) {
+        time_t now;
+        struct tm sdate;
+        time(&now);
+        time_t stime = now - (60 * 60 * 25); // subtract 25h
+        localtime_r(&stime, &sdate);
+        if (sdate.tm_year <= (2016 - 1900)) {
+            return (false); // time not set
+        }
+        startdate.dd[3] = sdate.tm_year - 100;
+        startdate.dd[2] = sdate.tm_mon + 1;
+        startdate.dd[1] = sdate.tm_mday;
+        startdate.dd[0] = sdate.tm_hour;
+
+        f = LittleFS.open(DATABASE_FILENAME, "r", false);
+        if (!f) {
+            return (false);
+        }
+        fileopen = true;
+        *pr++ = '[';
+    }
+    while (true) {
+        r = f.read(reinterpret_cast<uint8_t*>(&d), sizeof(pvData)); // read from database
+        if (r <= 0) {
+            if (last) {
+                f.close();
+                fileopen = false;
+                first = true;
+                last = false;
+                valid = false;
+                startdate.dh = 0L;
+                return (0); // end transmission
+            }
+            last = true;
+            *pr++ = ']';
+            return (pr - buffer); // last chunk
+        }
+        if (!valid) {
+            datehour cd;
+            cd.dd[3] = d.tm_year;
+            cd.dd[2] = d.tm_mon;
+            cd.dd[1] = d.tm_mday;
+            cd.dd[0] = d.tm_hour;
+            // MessageOutput.println(cd,16);
+            if ((cd.dh >= startdate.dh) && (oldenergy > 0.0)) {
+                valid = true;
+            } else
+                oldenergy = d.energy;
+        }
+        if (valid) {
+            if (first) {
+                first = false;
+            } else {
+                *pr++ = ',';
+            }
+            int len = snprintf(reinterpret_cast<char*>(pr), maxLen, "[%d,%d,%d,%d,%f]",
+                d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour,
+                (d.energy - oldenergy) * 1e3);
+            oldenergy = d.energy;
+            if (len >= 0) {
+                pr += len;
+            }
+            if (pr >= pre) {
+                return (pr - buffer); // buffer full, return number of chars
+            }
+        }
+    }
+}
+
+// read chunk from database for calendar view
+size_t WebApiDatabaseClass::readchunkDay(uint8_t* buffer, size_t maxLen, size_t index)
+{
+    static bool first = true;
+    static bool last = false;
+    static float startenergy = 0.0;
+    static struct pvData endofday = { 0, 0, 0, 0, 0.0 };
+    static File f;
+    uint8_t* pr = buffer;
+    uint8_t* pre = pr + maxLen - 50;
+    size_t r;
+    struct pvData d;
+
+    if (first) {
+        f = LittleFS.open(DATABASE_FILENAME, "r", false);
+        if (!f) {
+            return (0);
+        }
+        *pr++ = '[';
+    }
+    while (true) {
+        r = f.read(reinterpret_cast<uint8_t*>(&d), sizeof(pvData)); // read from database
+        if (r <= 0) {
+            if (last) {
+                f.close();
+                first = true;
+                last = false;
+                endofday = { 0, 0, 0, 0, 0.0 };
+                startenergy = 0.0;
+                return (0); // end transmission
+            }
+            last = true;
+            if (!first)
+                *pr++ = ',';
+            int len = snprintf(reinterpret_cast<char*>(pr), maxLen, "[%d,%d,%d,%d,%f]",
+                endofday.tm_year, endofday.tm_mon, endofday.tm_mday, endofday.tm_hour,
+                (endofday.energy - startenergy) * 1e3);
+            pr += len;
+            *pr++ = ']';
+            return (pr - buffer); // last chunk
+        }
+        if (startenergy == 0.0) {
+            if (d.energy > 0.0) {
+                startenergy = d.energy;
+            }
+        } else {
+            if (endofday.tm_mday != d.tm_mday) { // next day
+                if (first) {
+                    first = false;
+                } else
+                    *pr++ = ',';
+                int len = snprintf(reinterpret_cast<char*>(pr), maxLen, "[%d,%d,%d,%d,%f]",
+                    endofday.tm_year, endofday.tm_mon, endofday.tm_mday, endofday.tm_hour,
+                    (endofday.energy - startenergy) * 1e3);
+                startenergy = endofday.energy;
+                if (len >= 0)
+                    pr += len;
+                if (pr >= pre)
+                    return (pr - buffer); // buffer full, return number of chars
+            }
+        }
+        endofday = d;
+    }
+}
+
+void WebApiDatabaseClass::onDatabase(AsyncWebServerRequest* request)
+{
+    if (!WebApi.checkCredentialsReadonly(request)) {
+        return;
+    }
+    AsyncWebServerResponse* response = request->beginChunkedResponse("application/json", readchunk);
+    request->send(response);
+}
+
+void WebApiDatabaseClass::onDatabaseHour(AsyncWebServerRequest* request)
+{
+    if (!WebApi.checkCredentialsReadonly(request)) {
+        return;
+    }
+    AsyncWebServerResponse* response = request->beginChunkedResponse("application/json", readchunkHour);
+    request->send(response);
+}
+
+void WebApiDatabaseClass::onDatabaseDay(AsyncWebServerRequest* request)
+{
+    if (!WebApi.checkCredentialsReadonly(request)) {
+        return;
+    }
+    AsyncWebServerResponse* response = request->beginChunkedResponse("application/json", readchunkDay);
+    request->send(response);
+}

webapp/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

webapp/src/components/BarChart.vue

@@ -0,0 +1,88 @@
+<template>
+    <div class="card row" v-if="dataLoaded">
+        <GChart type="ColumnChart" :data="chartData" :options="chartOptions" />
+    </div>
+</template>
+
+<script lang="ts">
+import { defineComponent } from 'vue';
+import { GChart } from 'vue-google-charts';
+import { authHeader, handleResponse } from '@/utils/authentication';
+//import { DatetimeFormat } from 'vue-i18n';
+var data_col: any;
+
+const options_col = {
+    height: 300,
+    chartArea: {
+        top: 25,
+        width: '85%',
+        height: '80%'
+    },
+    bar: {
+        groupWidth: '90%'
+    },
+    legend: {
+        position: 'none'
+    },
+    hAxis: {
+        format: 'HH',
+        minorGridlines: {
+            count: 0
+        }
+    },
+    vAxis: {
+        minValue: 0,
+        format: '# Wh'
+    }
+};
+export default defineComponent({
+    components: {
+        GChart,
+    },
+    data() {
+        return {
+            chartData: data_col,
+            chartOptions: options_col,
+            dataLoaded: false,
+        }
+    },
+    created() {
+        this.getInitialData();
+        this.startautorefresh();
+    },
+    methods: {
+        getInitialData() {
+            this.dataLoaded = false;
+            fetch("/api/databaseHour", { headers: authHeader() })
+                .then((response) => handleResponse(response, this.$emitter, this.$router))
+                .then((energy) => {
+                    if (energy) {
+                        this.chartData = [[{ type: 'date', id: 'Time' }, { type: 'number', id: 'Energy' }]];
+                        energy.forEach((x: any[]) => {
+                            var d = new Date(x[0] + 2000, x[1] - 1, x[2], x[3]);
+                            this.chartData.push([d, Math.round(x[4])])
+                        });
+                        this.dataLoaded = true;
+                    }
+                });
+
+            // var date_formatter = new google.visualization.DateFormat({
+            //     pattern: "dd.MM.YY HH:mm"
+            // });
+            // date_formatter.format(data, 0);
+        },
+        callEveryHour() {
+            this.getInitialData();
+            setInterval(this.getInitialData, 1000 * 60 * 60);   // refresh every hour
+        },
+        startautorefresh() {
+            var nextDate = new Date();
+            nextDate.setHours(nextDate.getHours() + 1);
+            nextDate.setMinutes(0);
+            nextDate.setSeconds(5);
+            var difference: number = nextDate.valueOf() - Date.now();
+            setTimeout(this.callEveryHour, difference);
+        },
+    }
+});
+</script>

webapp/src/components/CalendarChart.vue

@@ -0,0 +1,59 @@
+<template>
+    <div class="card row" v-if="dataLoaded">
+        <CalendarHeatmap :values="values" :round="1" :end-date="endDate" :style="{'font-size': '10px'}" tooltip-unit="Wh" />
+    </div>
+</template>
+
+<script lang="ts">
+import { defineComponent } from 'vue';
+import { CalendarHeatmap } from 'vue3-calendar-heatmap'
+import { authHeader, handleResponse } from '@/utils/authentication';
+var data: Array<{ date: Date; count: number; }>;
+
+export default defineComponent({
+    components: {
+        CalendarHeatmap,
+    },
+    data() {
+        return {
+            values: data,
+            endDate: new Date(),
+            dataLoaded: false,
+        }
+    },
+    created() {
+        this.getInitialData();
+        this.startautorefresh();
+    },
+    methods: {
+        getInitialData() {
+            this.dataLoaded = false;
+            fetch("/api/databaseDay", { headers: authHeader() })
+                .then((response) => handleResponse(response, this.$emitter, this.$router))
+                .then((energy) => {
+                    if (energy) {
+                        this.values = [];
+                        var d: Date;
+                        energy.forEach((x: any[]) => {
+                            d = new Date(x[0] + 2000, x[1] - 1, x[2], x[3])
+                            this.values.push({ date: d, count: Math.round(x[4]) })
+                        })
+                        this.dataLoaded = true;
+                    }
+                })
+        },
+        callEveryHour() {
+            this.getInitialData();
+            setInterval(this.getInitialData, 1000 * 60 * 60);   // refresh every hour
+        },
+        startautorefresh() {
+            var nextDate = new Date();
+            nextDate.setHours(nextDate.getHours() + 1);
+            nextDate.setMinutes(0);
+            nextDate.setSeconds(5);
+            var difference: number = nextDate.valueOf() - Date.now();
+            setTimeout(this.callEveryHour, difference);
+        },
+    }
+});
+</script>

webapp/src/components/MemoryInfo.vue

@@ -22,6 +22,7 @@
                         :total="systemStatus.psram_total"
                         :used="systemStatus.psram_used"
                     />
+                    <a href='/api/database'>Read Database</a>
                     <FsInfo
                         :name="$t('memoryinfo.LittleFs')"
                         :total="systemStatus.littlefs_total"

webapp/src/views/ConfigAdminView.vue

@@ -61,6 +61,7 @@
                             <option v-for="file in restoreList" :key="file.name" :value="file.name">
                                 {{ file.descr }}
                             </option>
+                            <option selected value="database.bin">Database (database.bin)</option>
                         </select>
                     </div>
                     <div class="col-sm">

webapp/src/views/HomeView.vue

@@ -9,6 +9,12 @@
     >
         <HintView :hints="liveData.hints" />
         <InverterTotalInfo :totalData="liveData.total" /><br />
+        <div class="card">
+            <div class="card-body">
+                <BarChart />
+                <CalendarChart />
+            </div>
+        </div>
         <div class="row gy-3">
             <div class="col-sm-3 col-md-2" :style="[inverterData.length == 1 ? { display: 'none' } : {}]">
                 <div
@@ -498,6 +504,8 @@ import DevInfo from '@/components/DevInfo.vue';
 import EventLog from '@/components/EventLog.vue';
 import GridProfile from '@/components/GridProfile.vue';
 import HintView from '@/components/HintView.vue';
+import BarChart from "@/components/BarChart.vue";
+import CalendarChart from "@/components/CalendarChart.vue";
 import InverterChannelInfo from '@/components/InverterChannelInfo.vue';
 import InverterTotalInfo from '@/components/InverterTotalInfo.vue';
 import ModalDialog from '@/components/ModalDialog.vue';
@@ -536,6 +544,8 @@ export default defineComponent({
         GridProfile,
         HintView,
         InverterChannelInfo,
+        BarChart,
+        CalendarChart,
         InverterTotalInfo,
         ModalDialog,
         BIconArrowCounterclockwise,

webapp/vite.config.ts

@@ -14,8 +14,8 @@ let proxy_target;
 try {
     // eslint-disable-next-line
     proxy_target = require('./vite.user.ts').proxy_target;
-} catch {
-    proxy_target = '192.168.20.110';
+} catch (error) {
+    proxy_target = '192.168.2.93';
 }
 
 // https://vitejs.dev/config/