6 files changed, 96 insertions, 14 deletions

diff --git a/src/feeds.c b/src/feeds.c
index 86f33367..3417b4e9 100644
--- a/src/feeds.c
+++ b/src/feeds.c
@@ -317,10 +317,10 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
     /* Entries are removed from `incoming` if they are added to the Feeds entries array.
        Anything remaining in `incoming` will be deleted afterwards. */
     iBool gotNew = iFalse;
-    lock_Mutex(d->mtx);
     iTime now;
     initCurrent_Time(&now);
     if (isHeadings) {
+        lock_Mutex(d->mtx);
 //        printf("Updating sourceID %d...\n", sourceId);
         iStringSet *known = listHeadingEntriesFrom_Feeds_(d, sourceId);
 //        puts("  Known URLs:");
@@ -357,16 +357,22 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
 //        puts("Done.");
         iRelease(presentInSource);
         iRelease(known);
+        unlock_Mutex(d->mtx);
     }
     else {
+        /* All visited URLs still present in the source should be kept indefinitely so their
+           read status remains correct. The Kept flag will be cleared after the URL has been
+           discarded from the entry database and enough time has passed. */ {
+//            printf("updating entries from %d:\n", sourceId);
+            iForEach(PtrArray, i, incoming) {
+                const iFeedEntry *entry = i.ptr;
+//                printf("marking as kept: {%s}\n", cstr_String(&entry->url));
+                setUrlKept_Visited(visited_App(), &entry->url, iTrue);
+            }
+        }
+        lock_Mutex(d->mtx);
         iForEach(PtrArray, i, incoming) {
             iFeedEntry *entry = i.ptr;
-            /* Disregard old incoming entries. */
-            if (secondsSince_Time(&now, &entry->posted) >= maxAge_Visited) {
-                /* We don't remember this far back, so the unread status of the entry would
-                   be incorrect. */
-                continue;
-            }
             size_t pos;
             if (locate_SortedArray(&d->entries, &entry, &pos)) {
                 iFeedEntry *existing = *(iFeedEntry **) at_SortedArray(&d->entries, pos);
@@ -384,7 +390,8 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
                     changed = iTrue;
                 }
                 set_String(&existing->title, &entry->title);
-                existing->posted = entry->posted;
+                existing->posted     = entry->posted;
+                existing->discovered = entry->discovered; /* prevent discarding */
                 delete_FeedEntry(entry);
                 if (changed) {
                     /* TODO: better to use a new flag for read feed entries? */
@@ -398,8 +405,8 @@ static iBool updateEntries_Feeds_(iFeeds *d, iBool isHeadings, uint32_t sourceId
             }
             remove_PtrArrayIterator(&i);
         }
+        unlock_Mutex(d->mtx);
     }
-    unlock_Mutex(d->mtx);
     return gotNew;
 }
 
@@ -410,6 +417,8 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
     iZap(work);
     iBool gotNew = iFalse;
     postCommand_App("feeds.update.started");
+    const int totalJobs = size_PtrArray(&d->jobs);
+    int numFinishedJobs = 0;
     while (!d->stopWorker) {
         /* Start new jobs. */
         iForIndices(i, work) {
@@ -420,6 +429,7 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
         sleep_Thread(0.5); /* TODO: wait on a Condition so we can exit quickly */
         if (d->stopWorker) break;
         size_t ongoing = 0;
+        iBool doNotify = iFalse;
         iForIndices(i, work) {
             if (work[i]) {
                 if (isFinished_GmRequest(work[i]->request)) {
@@ -429,11 +439,15 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
                         d, work[i]->checkHeadings, work[i]->bookmarkId, &work[i]->results);
                     delete_FeedJob(work[i]);
                     work[i] = NULL;
+                    numFinishedJobs++;
+                    doNotify = iTrue;
                 }
                 else if (isTimedOut_FeedJob_(work[i])) {
                     /* Maybe we'll get it next time! */
                     delete_FeedJob(work[i]);
                     work[i] = NULL;
+                    numFinishedJobs++;
+                    doNotify = iTrue;
                 }
                 else {
                     ongoing++;
@@ -441,6 +455,9 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
                 /* TODO: abort job if it takes too long (> 15 seconds?) */
             }
         }
+        if (doNotify) {
+            postCommandf_App("feeds.update.progress arg:%d total:%d", numFinishedJobs, totalJobs);
+        }
         /* Stop if everything has finished. */
         if (ongoing == 0 && isEmpty_PtrArray(&d->jobs)) {
             break;
@@ -448,6 +465,24 @@ static iThreadResult fetch_Feeds_(iThread *thread) {
     }
     initCurrent_Time(&d->lastRefreshedAt);
     save_Feeds_(d);
+    /* Check if there are visited URLs marked as Kept that can be cleared because they are no
+       longer present in the database. */ {
+        iStringSet *knownEntryUrls = new_StringSet();
+        lock_Mutex(d->mtx);
+        iConstForEach(Array, i, &d->entries.values) {
+            const iFeedEntry *entry = *(const iFeedEntry **) i.value;
+            insert_StringSet(knownEntryUrls, &entry->url);
+        }
+        unlock_Mutex(d->mtx);
+        iConstForEach(PtrArray, j, listKept_Visited(visited_App())) {
+            iVisitedUrl *visUrl = j.ptr;
+            if (!contains_StringSet(knownEntryUrls, &visUrl->url)) {
+                visUrl->flags &= ~kept_VisitedUrlFlag;
+//                printf("unkept: {%s}\n", cstr_String(&visUrl->url));
+            }
+        }
+        iRelease(knownEntryUrls);
+    }
     postCommandf_App("feeds.update.finished arg:%d unread:%zu", gotNew ? 1 : 0,
                      numUnread_Feeds());
     return 0;
diff --git a/src/gmutil.c b/src/gmutil.c
index 1d361875..971747d4 100644
--- a/src/gmutil.c
+++ b/src/gmutil.c
@@ -137,6 +137,16 @@ void stripDefaultUrlPort_String(iString *d) {
     }
 }
 
+const iString *urlQueryStripped_String(const iString *url) {
+    size_t pos = indexOfCStr_String(url, "?");
+    if (pos != iInvalidPos) {
+        iString *stripped = collect_String(copy_String(url));
+        truncate_Block(&stripped->chars, pos);
+        return stripped;
+    }
+    return url;
+}
+
 iBool isDataUrl_String(const iString *d) {
     return startsWithCase_String(d, "data:");
 }
diff --git a/src/gmutil.h b/src/gmutil.h
index 3c10d45b..c65a17e6 100644
--- a/src/gmutil.h
+++ b/src/gmutil.h
@@ -123,6 +123,7 @@ void            punyEncodeDomain_Rangecc(iRangecc domain, iString *encoded_out);
 void            punyEncodeUrlHost_String(iString *absoluteUrl);
 void            stripDefaultUrlPort_String(iString *);
 const iString * urlFragmentStripped_String(const iString *);
+const iString * urlQueryStripped_String (const iString *);
 void            urlDecodePath_String    (iString *);
 void            urlEncodePath_String    (iString *);
 iString *       makeFileUrl_String      (const iString *localFilePath);
diff --git a/src/ui/sidebarwidget.c b/src/ui/sidebarwidget.c
index 4b772eeb..fcdb5b9e 100644
--- a/src/ui/sidebarwidget.c
+++ b/src/ui/sidebarwidget.c
@@ -1463,7 +1463,7 @@ static iBool processEvent_SidebarWidget_(iSidebarWidget *d, const SDL_Event *ev)
                         removeUrl_Visited(vis, url);
                     }
                     else {
-                        visitUrl_Visited(vis, url, transient_VisitedUrlFlag);
+                        visitUrl_Visited(vis, url, transient_VisitedUrlFlag | kept_VisitedUrlFlag);
                     }
                     postCommand_App("visited.changed");
                     return iTrue;
diff --git a/src/visited.c b/src/visited.c
index e9f691c6..466add5b 100644
--- a/src/visited.c
+++ b/src/visited.c
@@ -109,11 +109,13 @@ void load_Visited(iVisited *d, const char *dirPath) {
             const char *urlStart = skipSpace_CStr(endp);
             iVisitedUrl item;
             item.when.ts = (struct timespec){ .tv_sec = ts };
-            if (secondsSince_Time(&now, &item.when) > maxAge_Visited) {
+            if (~flags & kept_VisitedUrlFlag &&
+                secondsSince_Time(&now, &item.when) > maxAge_Visited) {
                 continue; /* Too old. */
             }
             item.flags = flags;
             initRange_String(&item.url, (iRangecc){ urlStart, line.end });
+            set_String(&item.url, &item.url);
             insert_SortedArray(&d->visited, &item);
         }
         unlock_Mutex(d->mtx);
@@ -153,6 +155,9 @@ void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
     lock_Mutex(d->mtx);
     if (locate_SortedArray(&d->visited, &visit, &pos)) {
         iVisitedUrl *old = at_SortedArray(&d->visited, pos);
+        if (old->flags & kept_VisitedUrlFlag) {
+            visitFlags |= kept_VisitedUrlFlag; /* must continue to be kept */
+        }
         if (cmpNewer_VisitedUrl_(&visit, old)) {
             old->when = visit.when;
             old->flags = visitFlags;
@@ -165,6 +170,21 @@ void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
     unlock_Mutex(d->mtx);
 }
 
+void setUrlKept_Visited(iVisited *d, const iString *url, iBool isKept) {
+    if (isEmpty_String(url)) return;
+    iVisitedUrl visit;
+    init_VisitedUrl(&visit);
+    set_String(&visit.url, canonicalUrl_String(url));
+    size_t pos;
+    lock_Mutex(d->mtx);
+    if (locate_SortedArray(&d->visited, &visit, &pos)) {
+        iVisitedUrl *vis = at_SortedArray(&d->visited, pos);
+        iChangeFlags(vis->flags, kept_VisitedUrlFlag, isKept);
+    }
+    unlock_Mutex(d->mtx);
+    deinit_VisitedUrl(&visit);
+}
+
 void removeUrl_Visited(iVisited *d, const iString *url) {
     url = canonicalUrl_String(url);
     iGuardMutex(d->mtx, {
@@ -183,7 +203,7 @@ iTime urlVisitTime_Visited(const iVisited *d, const iString *url) {
     iVisitedUrl item;
     size_t pos;
     iZap(item);
-    initCopy_String(&item.url, url);
+    initCopy_String(&item.url, canonicalUrl_String(url));
     lock_Mutex(d->mtx);
     if (locate_SortedArray(&d->visited, &item, &pos)) {
         item.when = ((const iVisitedUrl *) constAt_SortedArray(&d->visited, pos))->when;
@@ -203,7 +223,7 @@ static int cmpWhenDescending_VisitedUrlPtr_(const void *a, const void *b) {
     return -cmp_Time(&s->when, &t->when);
 }
 
-const iArray *list_Visited(const iVisited *d, size_t count) {
+const iPtrArray *list_Visited(const iVisited *d, size_t count) {
     iPtrArray *urls = collectNew_PtrArray();
     iGuardMutex(d->mtx, {
         iConstForEach(Array, i, &d->visited.values) {
@@ -219,3 +239,16 @@ const iArray *list_Visited(const iVisited *d, size_t count) {
     }
     return urls;
 }
+
+const iPtrArray *listKept_Visited(const iVisited *d) {
+    iPtrArray *urls = collectNew_PtrArray();
+    iGuardMutex(d->mtx, {
+        iConstForEach(Array, i, &d->visited.values) {
+            const iVisitedUrl *vis = i.value;
+            if (vis->flags & kept_VisitedUrlFlag) {
+                pushBack_PtrArray(urls, vis);
+            }
+        }
+    });
+    return urls;
+}
diff --git a/src/visited.h b/src/visited.h
index fec0b4c3..1f2d4fcf 100644
--- a/src/visited.h
+++ b/src/visited.h
@@ -41,6 +41,7 @@ struct Impl_VisitedUrl {
 
 enum iVisitedUrlFlag {
     transient_VisitedUrlFlag = 0x1, /* redirected; don't show in history */
+    kept_VisitedUrlFlag      = 0x2, /* don't discard this even after max age */
 };
 
 iDeclareType(Visited)
@@ -52,7 +53,9 @@ void    save_Visited            (const iVisited *, const char *dirPath);
 
 iTime   urlVisitTime_Visited    (const iVisited *, const iString *url);
 void    visitUrl_Visited        (iVisited *, const iString *url, uint16_t visitFlags); /* adds URL to the visited URLs set */
+void    setUrlKept_Visited      (iVisited *, const iString *url, iBool isKept); /* URL is marked as (non)discardable */
 void    removeUrl_Visited       (iVisited *, const iString *url);
 iBool   containsUrl_Visited     (const iVisited *, const iString *url);
 
-const iPtrArray *  list_Visited (const iVisited *, size_t count); /* returns collected */
+const iPtrArray *   list_Visited        (const iVisited *, size_t count); /* returns collected */
+const iPtrArray *   listKept_Visited    (const iVisited *);