danog/ecodash
1
0
Fork 0
forked from massivebox/ecodash
Code Pull requests Activity

Improve SQLite

Browse source 
- Now the cache isn't cleared and fetched from zero each update, but only when there's missing information for a day
- Fixed a bug that prevented new users from saving changes in the config

NOTE: Turns out that HomeAssistant's API only returns data for the last 10 days. Looks like we will have to improve the caching system to work around this.
This commit is contained in:
MassiveBox 2023-01-04 15:57:16 +01:00
parent e9125b783c
commit 52ba0ea4c1
4 changed files with 156 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
api.go
View file

@ -12,11 +12,16 @@ import (
"time"
)
type HistoryResult [][]struct {
type HistoryResult []struct {
State string `json:"state"`
LastUpdated time.Time `json:"last_updated"`
}
func dayStart(t time.Time) time.Time {
hours, minutes, seconds := t.Clock()
return t.Add(-(time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second))
}
func (config Config) queryHistory(entityID string, startTime, endTime time.Time) (HistoryResult, error) {
req, err := http.NewRequest("GET", config.HomeAssistant.BaseURL+
@ -46,13 +51,27 @@ func (config Config) queryHistory(entityID string, startTime, endTime time.Time)
return HistoryResult{}, err
}
var result HistoryResult
var result []HistoryResult
err = json.Unmarshal(body, &result)
if err != nil {
return result, err
return HistoryResult{}, err
}
return result, nil
if len(result) != 1 {
return HistoryResult{}, nil
}
return result[0], nil
}
// t can be any time during the desired day.
func (config Config) getDayHistory(entityID string, t time.Time) (HistoryResult, error) {
hours, minutes, seconds := t.Clock()
endTime := t.Add(time.Duration(23-hours)*time.Hour + time.Duration(59-minutes)*time.Minute + time.Duration(59-seconds)*time.Second)
return config.queryHistory(entityID, dayStart(t), endTime)
}
@ -65,7 +84,33 @@ type DayData struct {
Low float32
}
func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
func (config Config) historyAverageAndConvertToGreen(entityID string, t time.Time) (DayData, error) {
history, err := config.getDayHistory(entityID, t)
if err != nil {
return DayData{}, err
}
var day DayData
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
}
day.Value += float32(val)
day.Measurements++
}
day.Value = 100 - (day.Value / float32(day.Measurements))
return day, nil
}
func (config Config) historyBulkAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
history, err := config.queryHistory(entityID, startTime, endTime)
if err != nil {
@ -74,7 +119,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
var days []DayData
for _, historyChange := range history[0] {
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
@ -93,7 +138,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
if !found {
days = append(days, DayData{
DayNumber: dayNo,
DayTime: historyChange.LastUpdated.Local(),
DayTime: dayStart(historyChange.LastUpdated.Local()),
Measurements: 1,
Value: value,
})
@ -112,7 +157,38 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
}
func (config Config) historyDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
func (config Config) historyDelta(entityID string, t time.Time) (DayData, error) {
history, err := config.getDayHistory(entityID, t)
if err != nil {
return DayData{}, err
}
var day DayData
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
}
value := float32(val)
if value > day.High {
day.High = value
}
if value < day.Low || day.Low == 0 {
day.Low = value
}
}
day.Value = day.High - day.Low
return day, nil
}
func (config Config) historyBulkDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
history, err := config.queryHistory(entityID, startTime, endTime)
if err != nil {
@ -121,7 +197,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
var days []DayData
for _, historyChange := range history[0] {
for _, historyChange := range history {
if historyChange.State != "off" {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
@ -145,7 +221,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
if !found {
days = append(days, DayData{
DayNumber: dayNo,
DayTime: historyChange.LastUpdated.Local(),
DayTime: dayStart(historyChange.LastUpdated.Local()),
Value: value,
})
}
@ -185,7 +261,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
ret = append(ret, DayData{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: previousValue,
})
}
@ -210,7 +286,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
ret = append(ret, DayData{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: previousValue,
})
}
@ -224,7 +300,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
ret = append([]DayData{
{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: 0,
},
}, ret...)

60
cache.go
View file

@ -1,6 +1,7 @@
package main
import (
"errors"
"fmt"
"time"
)
@ -14,45 +15,72 @@ type CacheData []CacheEntry
func (config Config) updateCache() {
greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, time.Now())
if err != nil {
return
}
historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, time.Now())
if err != nil {
return
}
config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", dayStart(time.Now()).Unix(), greenEnergyPercentage.Value, historyPolledSmartEnergySummation.Value)
cached, err := config.readCache()
if err != nil {
return
}
if len(cached) != 8 && time.Now().Sub(config.HomeAssistant.InstallationDate) > 8*time.Hour*24 {
err := config.refreshCacheFromPast(time.Now().Add(-8 * time.Hour * 24))
if err != nil {
fmt.Println("Error refreshing cache", err.Error())
return
}
}
}
func (config Config) refreshCacheFromInstall() error {
return config.refreshCacheFromPast(config.HomeAssistant.InstallationDate)
}
func (config Config) refreshCacheFromPast(pastTime time.Time) error {
// in order to avoid querying and storing each day's data from 0001-01-01 in future versions
if config.HomeAssistant.InstallationDate.IsZero() {
return
return errors.New("installation date not set")
}
now := time.Now()
h, m, s := now.Clock()
start := now.AddDate(0, 0, -7).Add(-(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second))
greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, start, now)
greenEnergyPercentage, err := config.historyBulkAverageAndConvertToGreen(config.Sensors.FossilPercentage, pastTime, time.Now())
if err != nil {
fmt.Println("Error updating cached data for FossilPercentage -" + err.Error())
return
return err
}
historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, start, now)
historyPolledSmartEnergySummation, err := config.historyBulkDelta(config.Sensors.PolledSmartEnergySummation, pastTime, time.Now())
if err != nil {
fmt.Println("Error updating cached data for PolledSmartEnergySummation -" + err.Error())
return
return err
}
_, err = config.db.Exec("DELETE FROM cache")
if err != nil {
fmt.Println("Error deleting previous records to cache: -", err.Error())
return
return err
}
for key, day := range greenEnergyPercentage {
_, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value)
if err != nil {
fmt.Println("Error adding record to cache: -", err.Error())
return
return err
}
}
return nil
}
func (config Config) readCache() (CacheData, error) {
rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache")
start := dayStart(time.Now()).AddDate(0, 0, -8)
rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache WHERE time > ?", start.Unix())
if err != nil {
return CacheData{}, err
}

31
config.go
View file

@ -64,6 +64,21 @@ func formatURL(url string) (string, error) {
func loadConfig() (config Config, err error, isFirstRun bool) {
db, err := sql.Open("sqlite3", "./database.db")
if err != nil {
return Config{}, err, false
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" (
"time" NUMERIC NOT NULL,
"green_energy_percentage" REAL NOT NULL,
"energy_consumption" REAL NOT NULL,
PRIMARY KEY("time")
);`)
if err != nil {
return Config{}, err, false
}
var defaultConfig = Config{}
defaultConfig.Dashboard.Theme = "default"
defaultConfig.Dashboard.Name = "EcoDash"
@ -76,6 +91,7 @@ func loadConfig() (config Config, err error, isFirstRun bool) {
NewTab: true,
Primary: true,
})
defaultConfig.db = db
data, err := os.ReadFile("config.json")
if err != nil {
@ -96,23 +112,8 @@ func loadConfig() (config Config, err error, isFirstRun bool) {
if err != nil {
return Config{}, err, false
}
db, err := sql.Open("sqlite3", "./database.db")
if err != nil {
return Config{}, err, false
}
conf.db = db
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" (
"time" NUMERIC NOT NULL,
"green_energy_percentage" REAL NOT NULL,
"energy_consumption" REAL NOT NULL,
PRIMARY KEY("time")
);`)
if err != nil {
return Config{}, err, false
}
return conf, nil, false
}

10
http.go
View file

@ -118,7 +118,7 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
}
form := Config{
HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: parsedTime},
HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: dayStart(parsedTime)},
Sensors: Sensors{PolledSmartEnergySummation: c.FormValue("polled_smart_energy_summation"), FossilPercentage: c.FormValue("fossil_percentage")},
Administrator: Administrator{Username: c.FormValue("username") /*PasswordHash to be filled later*/},
Dashboard: Dashboard{Theme: c.FormValue("theme"), Name: c.FormValue("name"), HeaderLinks: config.Dashboard.HeaderLinks, FooterLinks: config.Dashboard.FooterLinks},
@ -140,12 +140,8 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
return errors.New("No changes from previous config.")
}
// in order to test if ha base URL, API key and entity IDs are correct we try fetching the devices history
_, err = form.queryHistory(form.Sensors.FossilPercentage, time.Now().Add(-5*time.Minute), time.Now())
if err != nil {
return err
}
_, err = form.queryHistory(form.Sensors.PolledSmartEnergySummation, time.Now().Add(-5*time.Minute), time.Now())
form.db = config.db
err = form.refreshCacheFromInstall()
if err != nil {
return err
}