From c63861230efb2b7b4d179ba169f1003f29e35ea1 Mon Sep 17 00:00:00 2001
From: makeworld <colecmac@protonmail.com>
Date: Mon, 17 Aug 2020 13:31:45 -0400
Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20Added=20updateAll?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 feeds/feeds.go   | 77 +++++++++++++++++++++++++++++++++++++++++++-----
 feeds/structs.go | 30 +++++++++++++++++--
 2 files changed, 96 insertions(+), 11 deletions(-)

diff --git a/feeds/feeds.go b/feeds/feeds.go
index d496a7d..6ae33f8 100644
--- a/feeds/feeds.go
+++ b/feeds/feeds.go
@@ -20,6 +20,9 @@ import (
 	"github.com/mmcdole/gofeed"
 )
 
+// TODO: Test for deadlocks and whether there should be more
+// goroutines for file writing or other things.
+
 var (
 	ErrSaving     = errors.New("couldn't save JSON to disk")
 	ErrNotSuccess = errors.New("status 20 not returned")
@@ -93,11 +96,9 @@ func writeJson() error {
 	enc.SetEscapeHTML(false)
 	enc.SetIndent("", "  ")
 
-	data.feedMu.Lock()
-	data.pageMu.Lock()
+	data.Lock()
 	err = enc.Encode(&data)
-	data.feedMu.Unlock()
-	data.pageMu.Unlock()
+	data.Unlock()
 
 	return err
 }
@@ -191,10 +192,15 @@ func updatePage(url string) error {
 	if _, err := io.Copy(h, res.Body); err != nil {
 		return err
 	}
+	newHash := fmt.Sprintf("%x", h.Sum(nil))
+
 	data.pageMu.Lock()
-	data.Pages[url] = &pageJson{
-		Hash:    fmt.Sprintf("%x", h.Sum(nil)),
-		Updated: time.Now().UTC(),
+	if data.Pages[url].Hash != newHash {
+		// Page content is different
+		data.Pages[url] = &pageJson{
+			Hash:    newHash,
+			Changed: time.Now().UTC(),
+		}
 	}
 	data.pageMu.Unlock()
 
@@ -211,7 +217,62 @@ func updatePage(url string) error {
 }
 
 // updateAll updates all feeds and pages.
-// It should run in goroutine at a regular interval.
 func updateAll() {
+	// TODO: Is two goroutines the right amount?
 
+	worker := func(jobs <-chan [2]string, wg *sync.WaitGroup) {
+		// Each job is: []string{<type>, "url"}
+		// where <type> is "feed" or "page"
+
+		defer wg.Done()
+		for j := range jobs {
+			if j[0] == "feed" {
+				updateFeed(j[1])
+			}
+			if j[0] == "page" {
+				updatePage(j[1])
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	data.RLock()
+	numJobs := len(data.Feeds) + len(data.Pages)
+	jobs := make(chan [2]string, numJobs)
+
+	// Start 2 workers, waiting for jobs
+	for w := 0; w < 2; w++ {
+		wg.Add(1)
+		go worker(jobs, &wg)
+	}
+
+	// Get map keys in a slice
+
+	feedKeys := make([]string, len(data.Feeds))
+	i := 0
+	for k := range data.Feeds {
+		feedKeys[i] = k
+		i++
+	}
+
+	pageKeys := make([]string, len(data.Pages))
+	i = 0
+	for k := range data.Pages {
+		pageKeys[i] = k
+		i++
+	}
+
+	data.RUnlock()
+
+	for j := 0; j < numJobs; j++ {
+		if j < len(feedKeys) {
+			jobs <- [2]string{"feed", feedKeys[j]}
+		} else {
+			// In the Pages
+			jobs <- [2]string{"page", pageKeys[j-len(feedKeys)]}
+		}
+	}
+
+	wg.Wait()
 }
diff --git a/feeds/structs.go b/feeds/structs.go
index 01e2ac4..2f38adf 100644
--- a/feeds/structs.go
+++ b/feeds/structs.go
@@ -17,11 +17,11 @@ Example JSON.
 	"pages": {
 		"url1": {
 			"hash": <hash>,
-			"updated": <time>
+			"changed": <time>
 		},
 		"url2": {
 			"hash": <hash>,
-			"updated": <time>
+			"changed": <time>
 		}
 	}
 }
@@ -39,9 +39,33 @@ type jsonData struct {
 	Pages  map[string]*pageJson    `json:"pages,omitempty"`
 }
 
+// Lock locks both feed and page mutexes.
+func (j *jsonData) Lock() {
+	j.feedMu.Lock()
+	j.pageMu.Lock()
+}
+
+// Unlock unlocks both feed and page mutexes.
+func (j *jsonData) Unlock() {
+	j.feedMu.Unlock()
+	j.pageMu.Unlock()
+}
+
+// RLock read-locks both feed and page mutexes.
+func (j *jsonData) RLock() {
+	j.feedMu.RLock()
+	j.pageMu.RLock()
+}
+
+// RUnlock read-unlocks both feed and page mutexes.
+func (j *jsonData) RUnlock() {
+	j.feedMu.RUnlock()
+	j.pageMu.RUnlock()
+}
+
 type pageJson struct {
 	Hash    string    `json:"hash"`
-	Updated time.Time `json:"updated"`
+	Changed time.Time `json:"changed"` // When the latest change happened
 }
 
 var data jsonData // Global instance of jsonData - loaded from JSON and used