Feeds: Don't forget entries or their unread status

Three important changes:

1) Visited URLs can be marked as "kept" so they will never be discarded due to old age.
2) Feed entries are not discarded from the database until they are removed from the source, and then become too old (six months).
3) Visited feed entry URLs are always flagged as kept, so the (un)read status will not be forgotten.

6 files changed, 96 insertions, 14 deletions

diff --git a/src/feeds.c b/src/feeds.c
index 86f33367..3417b4e9 100644
--- a/src/feeds.c
+++ b/src/feeds.c
@@ -317,10 +317,10 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
     /* Entries are removed from `incoming` if they are added to the Feeds entries array.
        Anything remaining in `incoming` will be deleted afterwards. */
     iBool gotNew = iFalse;
-    lock_Mutex(d->mtx);
     iTime now;
     initCurrent_Time(&now);
     if (isHeadings) {
+        lock_Mutex(d->mtx);
 //        printf("Updating sourceID %d...\n", sourceId);
         iStringSet *known = listHeadingEntriesFrom_Feeds_(d, sourceId);
 //        puts("  Known URLs:");
@@ -357,16 +357,22 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
 //        puts("Done.");
         iRelease(presentInSource);
         iRelease(known);
+        unlock_Mutex(d->mtx);
     }
     else {
+        /* All visited URLs still present in the source should be kept indefinitely so their
+           read status remains correct. The Kept flag will be cleared after the URL has been
+           discarded from the entry database and enough time has passed. */ {
+//            printf("updating entries from %d:\n", sourceId);
+            iForEach(PtrArray, i, incoming) {
+                const iFeedEntry *entry = i.ptr;
+//                printf("marking as kept: {%s}\n", cstr_String(&entry->url));
+                setUrlKept_Visited(visited_App(), &entry->url, iTrue);
+            }
+        }
+        lock_Mutex(d->mtx);
         iForEach(PtrArray, i, incoming) {
             iFeedEntry *entry = i.ptr;
-            /* Disregard old incoming entries. */
-            if (secondsSince_Time(&now, &entry->posted) >= maxAge_Visited) {
-                /* We don't remember this far back, so the unread status of the entry would
-                   be incorrect. */
-                continue;
-            }
             size_t pos;
             if (locate_SortedArray(&d->entries, &entry, &pos)) {
                 iFeedEntry *existing = *(iFeedEntry **) at_SortedArray(&d->entries, pos);
@@ -384,7 +390,8 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
                     changed = iTrue;
                 }
                 set_String(&existing->title, &entry->title);
-                existing->posted = entry->posted;
+                existing->posted     = entry->posted;
+                existing->discovered = entry->discovered; /* prevent discarding */
                 delete_FeedEntry(entry);
                 if (changed) {
                     /* TODO: better to use a new flag for read feed entries? */
@@ -398,8 +405,8 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
             }
             remove_PtrArrayIterator(&i);
         }
+        unlock_Mutex(d->mtx);
     }
-    unlock_Mutex(d->mtx);
     return gotNew;
 }
 
@@ -410,6 +417,8 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
     iZap(work);
     iBool gotNew = iFalse;
     postCommand_App("feeds.update.started");
+    const int totalJobs = size_PtrArray(&d->jobs);
+    int numFinishedJobs = 0;
     while (!d->stopWorker) {
         /* Start new jobs. */
         iForIndices(i, work) {
@@ -420,6 +429,7 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
         sleep_Thread(0.5); /* TODO: wait on a Condition so we can exit quickly */
         if (d->stopWorker) break;
         size_t ongoing = 0;
+        iBool doNotify = iFalse;
         iForIndices(i, work) {
             if (work[i]) {
                 if (isFinished_GmRequest(work[i]->request)) {
@@ -429,11 +439,15 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
                         d, work[i]->checkHeadings, work[i]->bookmarkId, &work[i]->results);
                     delete_FeedJob(work[i]);
                     work[i] = NULL;
+                    numFinishedJobs++;
+                    doNotify = iTrue;
                 }
                 else if (isTimedOut_FeedJob_(work[i])) {
                     /* Maybe we'll get it next time! */
                     delete_FeedJob(work[i]);
                     work[i] = NULL;
+                    numFinishedJobs++;
+                    doNotify = iTrue;
                 }
                 else {
                     ongoing++;
@@ -441,6 +455,9 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
                 /* TODO: abort job if it takes too long (> 15 seconds?) */
             }
         }
+        if (doNotify) {
+            postCommandf_App("feeds.update.progress arg:%d total:%d", numFinishedJobs, totalJobs);
+        }
         /* Stop if everything has finished. */
         if (ongoing == 0 && isEmpty_PtrArray(&d->jobs)) {
             break;
@@ -448,6 +465,24 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
     }
     initCurrent_Time(&d->lastRefreshedAt);
     save_Feeds_(d);
+    /* Check if there are visited URLs marked as Kept that can be cleared because they are no
+       longer present in the database. */ {
+        iStringSet *knownEntryUrls = new_StringSet();
+        lock_Mutex(d->mtx);
+        iConstForEach(Array, i, &d->entries.values) {
+            const iFeedEntry *entry = *(const iFeedEntry **) i.value;
+            insert_StringSet(knownEntryUrls, &entry->url);
+        }
+        unlock_Mutex(d->mtx);
+        iConstForEach(PtrArray, j, listKept_Visited(visited_App())) {
+            iVisitedUrl *visUrl = j.ptr;
+            if (!contains_StringSet(knownEntryUrls, &visUrl->url)) {
+                visUrl->flags &= ~kept_VisitedUrlFlag;
+//                printf("unkept: {%s}\n", cstr_String(&visUrl->url));
+            }
+        }
+        iRelease(knownEntryUrls);
+    }
     postCommandf_App("feeds.update.finished arg:%d unread:%zu", gotNew ? 1 : 0,
                      numUnread_Feeds());
     return 0;
diff --git a/src/gmutil.c b/src/gmutil.c
index 1d361875..971747d4 100644
--- a/src/gmutil.c
+++ b/src/gmutil.c
@@ -137,6 +137,16 @@ void stripDefaultUrlPort_String(iString *d) {
     }
 }
 
+const iString *urlQueryStripped_String(const iString *url) {
+    size_t pos = indexOfCStr_String(url, "?");
+    if (pos != iInvalidPos) {
+        iString *stripped = collect_String(copy_String(url));
+        truncate_Block(&stripped->chars, pos);
+        return stripped;
+    }
+    return url;
+}
+
 iBool isDataUrl_String(const iString *d) {
     return startsWithCase_String(d, "data:");
 }
diff --git a/src/gmutil.h b/src/gmutil.h
index 3c10d45b..c65a17e6 100644
--- a/src/gmutil.h
+++ b/src/gmutil.h
@@ -123,6 +123,7 @@ void            punyEncodeDomain_Rangecc(iRangecc domain, iString *encoded_out);
 void            punyEncodeUrlHost_String(iString *absoluteUrl);
 void            stripDefaultUrlPort_String(iString *);
 const iString * urlFragmentStripped_String(const iString *);
+const iString * urlQueryStripped_String (const iString *);
 void            urlDecodePath_String    (iString *);
 void            urlEncodePath_String    (iString *);
 iString *       makeFileUrl_String      (const iString *localFilePath);
diff --git a/src/ui/sidebarwidget.c b/src/ui/sidebarwidget.c
index 4b772eeb..fcdb5b9e 100644
--- a/src/ui/sidebarwidget.c
+++ b/src/ui/sidebarwidget.c
@@ -1463,7 +1463,7 @@ static iBool processEvent_SidebarWidget_(iSidebarWidget *d, const SDL_Event *ev)
                         removeUrl_Visited(vis, url);
                     }
                     else {
-                        visitUrl_Visited(vis, url, transient_VisitedUrlFlag);
+                        visitUrl_Visited(vis, url, transient_VisitedUrlFlag | kept_VisitedUrlFlag);
                     }
                     postCommand_App("visited.changed");
                     return iTrue;
diff --git a/src/visited.c b/src/visited.c
index e9f691c6..466add5b 100644
--- a/src/visited.c
+++ b/src/visited.c
@@ -109,11 +109,13 @@ void load_Visited(iVisited *d, const char *dirPath) {
             const char *urlStart = skipSpace_CStr(endp);
             iVisitedUrl item;
             item.when.ts = (struct timespec){ .tv_sec = ts };
-            if (secondsSince_Time(&now, &item.when) > maxAge_Visited) {
+            if (~flags & kept_VisitedUrlFlag &&
+                secondsSince_Time(&now, &item.when) > maxAge_Visited) {
                 continue; /* Too old. */
             }
             item.flags = flags;
             initRange_String(&item.url, (iRangecc){ urlStart, line.end });
+            set_String(&item.url, &item.url);
             insert_SortedArray(&d->visited, &item);
         }
         unlock_Mutex(d->mtx);
@@ -153,6 +155,9 @@ void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
     lock_Mutex(d->mtx);
     if (locate_SortedArray(&d->visited, &visit, &pos)) {
         iVisitedUrl *old = at_SortedArray(&d->visited, pos);
+        if (old->flags & kept_VisitedUrlFlag) {
+            visitFlags |= kept_VisitedUrlFlag; /* must continue to be kept */
+        }
         if (cmpNewer_VisitedUrl_(&visit, old)) {
             old->when = visit.when;
             old->flags = visitFlags;
@@ -165,6 +170,21 @@ void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
     unlock_Mutex(d->mtx);
 }
 
+void setUrlKept_Visited(iVisited *d, const iString *url, iBool isKept) {
+    if (isEmpty_String(url)) return;
+    iVisitedUrl visit;
+    init_VisitedUrl(&visit);
+    set_String(&visit.url, canonicalUrl_String(url));
+    size_t pos;
+    lock_Mutex(d->mtx);
+    if (locate_SortedArray(&d->visited, &visit, &pos)) {
+        iVisitedUrl *vis = at_SortedArray(&d->visited, pos);
+        iChangeFlags(vis->flags, kept_VisitedUrlFlag, isKept);
+    }
+    unlock_Mutex(d->mtx);
+    deinit_VisitedUrl(&visit);
+}
+
 void removeUrl_Visited(iVisited *d, const iString *url) {
     url = canonicalUrl_String(url);
     iGuardMutex(d->mtx, {
@@ -183,7 +203,7 @@ iTime urlVisitTime_Visited(const iVisited *d, const iString *url) {
     iVisitedUrl item;
     size_t pos;
     iZap(item);
-    initCopy_String(&item.url, url);
+    initCopy_String(&item.url, canonicalUrl_String(url));
     lock_Mutex(d->mtx);
     if (locate_SortedArray(&d->visited, &item, &pos)) {
         item.when = ((const iVisitedUrl *) constAt_SortedArray(&d->visited, pos))->when;
@@ -203,7 +223,7 @@ static int cmpWhenDescending_VisitedUrlPtr_(const void *a, const void *b) {
     return -cmp_Time(&s->when, &t->when);
 }
 
-const iArray *list_Visited(const iVisited *d, size_t count) {
+const iPtrArray *list_Visited(const iVisited *d, size_t count) {
     iPtrArray *urls = collectNew_PtrArray();
     iGuardMutex(d->mtx, {
         iConstForEach(Array, i, &d->visited.values) {
@@ -219,3 +239,16 @@ const iArray *list_Visited(const iVisited *d, size_t count) {
     }
     return urls;
 }
+
+const iPtrArray *listKept_Visited(const iVisited *d) {
+    iPtrArray *urls = collectNew_PtrArray();
+    iGuardMutex(d->mtx, {
+        iConstForEach(Array, i, &d->visited.values) {
+            const iVisitedUrl *vis = i.value;
+            if (vis->flags & kept_VisitedUrlFlag) {
+                pushBack_PtrArray(urls, vis);
+            }
+        }
+    });
+    return urls;
+}
diff --git a/src/visited.h b/src/visited.h
index fec0b4c3..1f2d4fcf 100644
--- a/src/visited.h
+++ b/src/visited.h
@@ -41,6 +41,7 @@ struct Impl_VisitedUrl {
 
 enum iVisitedUrlFlag {
     transient_VisitedUrlFlag = 0x1, /* redirected; don't show in history */
+    kept_VisitedUrlFlag      = 0x2, /* don't discard this even after max age */
 };
 
 iDeclareType(Visited)
@@ -52,7 +53,9 @@ void    save_Visited            (const iVisited *, const char *dirPath);
 
 iTime   urlVisitTime_Visited    (const iVisited *, const iString *url);
 void    visitUrl_Visited        (iVisited *, const iString *url, uint16_t visitFlags); /* adds URL to the visited URLs set */
+void    setUrlKept_Visited      (iVisited *, const iString *url, iBool isKept); /* URL is marked as (non)discardable */
 void    removeUrl_Visited       (iVisited *, const iString *url);
 iBool   containsUrl_Visited     (const iVisited *, const iString *url);
 
-const iPtrArray *  list_Visited (const iVisited *, size_t count); /* returns collected */
+const iPtrArray *   list_Visited        (const iVisited *, size_t count); /* returns collected */
+const iPtrArray *   listKept_Visited    (const iVisited *);