Improve SQLite

- Now the cache isn't cleared and fetched from zero each update, but only when there's missing information for a day
- Fixed a bug that prevented new users from saving changes in the config

NOTE: Turns out that HomeAssistant's API only returns data for the last 10 days. Looks like we will have to improve the caching system to work around this.

api.go

@@ -12,11 +12,16 @@ import (
 	"time"
 )
 
-type HistoryResult [][]struct {
+type HistoryResult []struct {
 	State       string    `json:"state"`
 	LastUpdated time.Time `json:"last_updated"`
 }
 
+func dayStart(t time.Time) time.Time {
+	hours, minutes, seconds := t.Clock()
+	return t.Add(-(time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second))
+}
+
 func (config Config) queryHistory(entityID string, startTime, endTime time.Time) (HistoryResult, error) {
 
 	req, err := http.NewRequest("GET", config.HomeAssistant.BaseURL+
@@ -46,13 +51,27 @@ func (config Config) queryHistory(entityID string, startTime, endTime time.Time)
 		return HistoryResult{}, err
 	}
 
-	var result HistoryResult
+	var result []HistoryResult
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return result, err
+		return HistoryResult{}, err
 	}
 
-	return result, nil
+	if len(result) != 1 {
+		return HistoryResult{}, nil
+	}
+
+	return result[0], nil
+
+}
+
+// t can be any time during the desired day.
+func (config Config) getDayHistory(entityID string, t time.Time) (HistoryResult, error) {
+
+	hours, minutes, seconds := t.Clock()
+	endTime := t.Add(time.Duration(23-hours)*time.Hour + time.Duration(59-minutes)*time.Minute + time.Duration(59-seconds)*time.Second)
+
+	return config.queryHistory(entityID, dayStart(t), endTime)
 
 }
 
@@ -65,7 +84,33 @@ type DayData struct {
 	Low          float32
 }
 
-func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
+func (config Config) historyAverageAndConvertToGreen(entityID string, t time.Time) (DayData, error) {
+
+	history, err := config.getDayHistory(entityID, t)
+	if err != nil {
+		return DayData{}, err
+	}
+
+	var day DayData
+
+	for _, historyChange := range history {
+
+		val, err := strconv.ParseFloat(historyChange.State, 32)
+		if err != nil {
+			continue
+		}
+
+		day.Value += float32(val)
+		day.Measurements++
+
+	}
+
+	day.Value = 100 - (day.Value / float32(day.Measurements))
+	return day, nil
+
+}
+
+func (config Config) historyBulkAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
 
 	history, err := config.queryHistory(entityID, startTime, endTime)
 	if err != nil {
@@ -74,7 +119,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
 
 	var days []DayData
 
-	for _, historyChange := range history[0] {
+	for _, historyChange := range history {
 		val, err := strconv.ParseFloat(historyChange.State, 32)
 		if err != nil {
 			continue
@@ -93,7 +138,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
 		if !found {
 			days = append(days, DayData{
 				DayNumber:    dayNo,
-				DayTime:      historyChange.LastUpdated.Local(),
+				DayTime:      dayStart(historyChange.LastUpdated.Local()),
 				Measurements: 1,
 				Value:        value,
 			})
@@ -112,7 +157,38 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
 
 }
 
-func (config Config) historyDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
+func (config Config) historyDelta(entityID string, t time.Time) (DayData, error) {
+
+	history, err := config.getDayHistory(entityID, t)
+	if err != nil {
+		return DayData{}, err
+	}
+
+	var day DayData
+
+	for _, historyChange := range history {
+
+		val, err := strconv.ParseFloat(historyChange.State, 32)
+		if err != nil {
+			continue
+		}
+		value := float32(val)
+
+		if value > day.High {
+			day.High = value
+		}
+		if value < day.Low || day.Low == 0 {
+			day.Low = value
+		}
+
+	}
+
+	day.Value = day.High - day.Low
+	return day, nil
+
+}
+
+func (config Config) historyBulkDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
 
 	history, err := config.queryHistory(entityID, startTime, endTime)
 	if err != nil {
@@ -121,7 +197,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
 
 	var days []DayData
 
-	for _, historyChange := range history[0] {
+	for _, historyChange := range history {
 		if historyChange.State != "off" {
 			val, err := strconv.ParseFloat(historyChange.State, 32)
 			if err != nil {
@@ -145,7 +221,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
 			if !found {
 				days = append(days, DayData{
 					DayNumber: dayNo,
-					DayTime:   historyChange.LastUpdated.Local(),
+					DayTime:   dayStart(historyChange.LastUpdated.Local()),
 					Value:     value,
 				})
 			}
@@ -185,7 +261,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
 					fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
 					ret = append(ret, DayData{
 						DayNumber: fakeTime.Day(),
-						DayTime:   fakeTime,
+						DayTime:   dayStart(fakeTime),
 						Value:     previousValue,
 					})
 				}
@@ -210,7 +286,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
 			fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
 			ret = append(ret, DayData{
 				DayNumber: fakeTime.Day(),
-				DayTime:   fakeTime,
+				DayTime:   dayStart(fakeTime),
 				Value:     previousValue,
 			})
 		}
@@ -224,7 +300,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
 			ret = append([]DayData{
 				{
 					DayNumber: fakeTime.Day(),
-					DayTime:   fakeTime,
+					DayTime:   dayStart(fakeTime),
 					Value:     0,
 				},
 			}, ret...)

cache.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"time"
 )
@@ -14,45 +15,72 @@ type CacheData []CacheEntry
 
 func (config Config) updateCache() {
 
-	// in order to avoid querying and storing each day's data from 0001-01-01 in future versions
+	greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, time.Now())
-	if config.HomeAssistant.InstallationDate.IsZero() {
+	if err != nil {
+		return
+	}
+	historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, time.Now())
+	if err != nil {
 		return
 	}
 
-	now := time.Now()
+	config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", dayStart(time.Now()).Unix(), greenEnergyPercentage.Value, historyPolledSmartEnergySummation.Value)
-	h, m, s := now.Clock()
-	start := now.AddDate(0, 0, -7).Add(-(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second))
 
-	greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, start, now)
+	cached, err := config.readCache()
 	if err != nil {
-		fmt.Println("Error updating cached data for FossilPercentage -" + err.Error())
 		return
 	}
-	historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, start, now)
+	if len(cached) != 8 && time.Now().Sub(config.HomeAssistant.InstallationDate) > 8*time.Hour*24 {
+		err := config.refreshCacheFromPast(time.Now().Add(-8 * time.Hour * 24))
 		if err != nil {
-		fmt.Println("Error updating cached data for PolledSmartEnergySummation -" + err.Error())
+			fmt.Println("Error refreshing cache", err.Error())
-		return
-	}
-
-	_, err = config.db.Exec("DELETE FROM cache")
-	if err != nil {
-		fmt.Println("Error deleting previous records to cache: -", err.Error())
-		return
-	}
-
-	for key, day := range greenEnergyPercentage {
-		_, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value)
-		if err != nil {
-			fmt.Println("Error adding record to cache: -", err.Error())
 			return
 		}
 	}
 
 }
 
+func (config Config) refreshCacheFromInstall() error {
+	return config.refreshCacheFromPast(config.HomeAssistant.InstallationDate)
+}
+
+func (config Config) refreshCacheFromPast(pastTime time.Time) error {
+
+	// in order to avoid querying and storing each day's data from 0001-01-01 in future versions
+	if config.HomeAssistant.InstallationDate.IsZero() {
+		return errors.New("installation date not set")
+	}
+
+	greenEnergyPercentage, err := config.historyBulkAverageAndConvertToGreen(config.Sensors.FossilPercentage, pastTime, time.Now())
+	if err != nil {
+		return err
+	}
+	historyPolledSmartEnergySummation, err := config.historyBulkDelta(config.Sensors.PolledSmartEnergySummation, pastTime, time.Now())
+	if err != nil {
+		return err
+	}
+
+	_, err = config.db.Exec("DELETE FROM cache")
+	if err != nil {
+		return err
+	}
+
+	for key, day := range greenEnergyPercentage {
+		_, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
 func (config Config) readCache() (CacheData, error) {
 
-	rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache")
+	start := dayStart(time.Now()).AddDate(0, 0, -8)
+
+	rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache WHERE time > ?", start.Unix())
 	if err != nil {
 		return CacheData{}, err
 	}

config.go

@@ -64,6 +64,21 @@ func formatURL(url string) (string, error) {
 
 func loadConfig() (config Config, err error, isFirstRun bool) {
 
+	db, err := sql.Open("sqlite3", "./database.db")
+	if err != nil {
+		return Config{}, err, false
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" (
+		"time"	NUMERIC NOT NULL,
+		"green_energy_percentage"	REAL NOT NULL,
+		"energy_consumption"	REAL NOT NULL,
+		PRIMARY KEY("time")
+	);`)
+	if err != nil {
+		return Config{}, err, false
+	}
+
 	var defaultConfig = Config{}
 	defaultConfig.Dashboard.Theme = "default"
 	defaultConfig.Dashboard.Name = "EcoDash"
@@ -76,6 +91,7 @@ func loadConfig() (config Config, err error, isFirstRun bool) {
 		NewTab:      true,
 		Primary:     true,
 	})
+	defaultConfig.db = db
 
 	data, err := os.ReadFile("config.json")
 	if err != nil {
@@ -96,23 +112,8 @@ func loadConfig() (config Config, err error, isFirstRun bool) {
 	if err != nil {
 		return Config{}, err, false
 	}
-
-	db, err := sql.Open("sqlite3", "./database.db")
-	if err != nil {
-		return Config{}, err, false
-	}
 	conf.db = db
 
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" (
-		"time"	NUMERIC NOT NULL,
-		"green_energy_percentage"	REAL NOT NULL,
-		"energy_consumption"	REAL NOT NULL,
-		PRIMARY KEY("time")
-	);`)
-	if err != nil {
-		return Config{}, err, false
-	}
-
 	return conf, nil, false
 
 }

http.go

@@ -118,7 +118,7 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
 	}
 
 	form := Config{
-		HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: parsedTime},
+		HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: dayStart(parsedTime)},
 		Sensors:       Sensors{PolledSmartEnergySummation: c.FormValue("polled_smart_energy_summation"), FossilPercentage: c.FormValue("fossil_percentage")},
 		Administrator: Administrator{Username: c.FormValue("username") /*PasswordHash to be filled later*/},
 		Dashboard:     Dashboard{Theme: c.FormValue("theme"), Name: c.FormValue("name"), HeaderLinks: config.Dashboard.HeaderLinks, FooterLinks: config.Dashboard.FooterLinks},
@@ -140,12 +140,8 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
 		return errors.New("No changes from previous config.")
 	}
 
-	// in order to test if ha base URL, API key and entity IDs are correct we try fetching the devices history
+	form.db = config.db
-	_, err = form.queryHistory(form.Sensors.FossilPercentage, time.Now().Add(-5*time.Minute), time.Now())
+	err = form.refreshCacheFromInstall()
-	if err != nil {
-		return err
-	}
-	_, err = form.queryHistory(form.Sensors.PolledSmartEnergySummation, time.Now().Add(-5*time.Minute), time.Now())
 	if err != nil {
 		return err
 	}