Canonical URIs

Internally, all URIs should be converted to a canonical form so that they can be compared against each other.

The canonical form is an IRI with spaces and reserved characters percent-encoded.

12 files changed, 51 insertions, 21 deletions

diff --git a/src/app.c b/src/app.c
index b25a213e..2de4c370 100644
--- a/src/app.c
+++ b/src/app.c
@@ -2815,7 +2815,7 @@ iStringSet *listOpenURLs_App(void) {
     iStringSet *set = new_StringSet();
     iObjectList *docs = listDocuments_App(NULL);
     iConstForEach(ObjectList, i, docs) {
-        insert_StringSet(set, withSpacesEncoded_String(url_DocumentWidget(i.object)));
+        insert_StringSet(set, canonicalUrl_String(url_DocumentWidget(i.object)));
     }
     iRelease(docs);
     return set;
diff --git a/src/bookmarks.c b/src/bookmarks.c
index 1f887c98..65cc7982 100644
--- a/src/bookmarks.c
+++ b/src/bookmarks.c
@@ -158,6 +158,7 @@ void load_Bookmarks(iBookmarks *d, const char *dirPath) {
                     appendChar_String(&bm->url, '/');
                 }
                 stripDefaultUrlPort_String(&bm->url);
+                set_String(&bm->url, canonicalUrl_String(&bm->url));
             }
             nextSplit_Rangecc(src, "\n", &line);
             setRange_String(&bm->title, line);
@@ -201,7 +202,7 @@ uint32_t add_Bookmarks(iBookmarks *d, const iString *url, const iString *title,
                        iChar icon) {
     lock_Mutex(d->mtx);
     iBookmark *bm = new_Bookmark();
-    set_String(&bm->url, url);
+    set_String(&bm->url, canonicalUrl_String(url));
     set_String(&bm->title, title);
     if (tags) {
         set_String(&bm->tags, tags);
@@ -298,6 +299,7 @@ static iBool matchUrl_(void *url, const iBookmark *bm) {
 
 uint32_t findUrl_Bookmarks(const iBookmarks *d, const iString *url) {
     /* TODO: O(n), boo */
+    url = canonicalUrl_String(url);
     const iPtrArray *found = list_Bookmarks(d, NULL, matchUrl_, (void *) url);
     if (isEmpty_PtrArray(found)) return 0;
     return id_Bookmark(constFront_PtrArray(found));
@@ -448,7 +450,7 @@ void requestFinished_Bookmarks(iBookmarks *d, iGmRequest *req) {
                 const iRangecc url    = capturedRange_RegExpMatch(&m, 1);
                 const iRangecc title  = capturedRange_RegExpMatch(&m, 3);
                 iString *      urlStr = newRange_String(url);
-                const iString *absUrl = absoluteUrl_String(url_GmRequest(req), urlStr);
+                const iString *absUrl = canonicalUrl_String(absoluteUrl_String(url_GmRequest(req), urlStr));
                 if (!findUrl_Bookmarks(d, absUrl)) {
                     iString *titleStr = newRange_String(title);
                     if (isEmpty_String(titleStr)) {
diff --git a/src/feeds.c b/src/feeds.c
index 6b102fef..a68fd726 100644
--- a/src/feeds.c
+++ b/src/feeds.c
@@ -215,7 +215,7 @@ static void parseResult_FeedJob_(iFeedJob *d) {
                 entry->discovered = now;
                 entry->bookmarkId = d->bookmarkId;
                 setRange_String(&entry->url, url);
-                set_String(&entry->url, absoluteUrl_String(url_GmRequest(d->request), &entry->url));
+                set_String(&entry->url, canonicalUrl_String(absoluteUrl_String(url_GmRequest(d->request), &entry->url)));
                 setRange_String(&entry->title, title);
                 trimTitle_(&entry->title);
                 int year, month, day;
@@ -241,7 +241,7 @@ static void parseResult_FeedJob_(iFeedJob *d) {
                     entry->bookmarkId = d->bookmarkId;
                     iString *title = newRange_String(line);
                     set_String(&entry->title, title);
-                    set_String(&entry->url, &d->url);
+                    set_String(&entry->url, canonicalUrl_String(&d->url));                    
                     appendChar_String(&entry->url, '#');
                     append_String(&entry->url, collect_String(urlEncode_String(title)));
                     delete_String(title);
diff --git a/src/gempub.c b/src/gempub.c
index 1f5d58ce..23846414 100644
--- a/src/gempub.c
+++ b/src/gempub.c
@@ -246,10 +246,10 @@ size_t navSize_Gempub(const iGempub *d) {
 
 size_t navIndex_Gempub(const iGempub *d, const iString *url) {
     parseNavigationLinks_Gempub_(d);
-    const iString *normUrl = withSpacesEncoded_String(url);
+    const iString *canonUrl = withSpacesEncoded_String(url);
     iConstForEach(Array, i, d->navLinks) {
         const iGempubNavLink *nav = i.value;
-        if (equalCase_String(&nav->url, normUrl)) {
+        if (equalCase_String(&nav->url, canonUrl)) {
             return index_ArrayConstIterator(&i);
         }
     }
diff --git a/src/gmdocument.c b/src/gmdocument.c
index f15d9d1d..8832271a 100644
--- a/src/gmdocument.c
+++ b/src/gmdocument.c
@@ -1554,13 +1554,14 @@ static void normalize_GmDocument(iGmDocument *d) {
     printf("wasNormalized: %d\n", wasNormalized);
     fflush(stdout);
     set_String(&d->source, collect_String(normalized));
-    normalize_String(&d->source); /* NFC */
+    //normalize_String(&d->source); /* NFC */
     printf("orig:%zu norm:%zu\n", size_String(&d->unormSource), size_String(&d->source));
     /* normalized source has an extra newline at the end */
 //    iAssert(wasNormalized || equal_String(&d->unormSource, &d->source));
 }
 
 void setUrl_GmDocument(iGmDocument *d, const iString *url) {
+    url = canonicalUrl_String(url);
     set_String(&d->url, url);
     iUrl parts;
     init_Url(&parts, url);
diff --git a/src/gmutil.c b/src/gmutil.c
index 7a1ae938..846405c3 100644
--- a/src/gmutil.c
+++ b/src/gmutil.c
@@ -537,9 +537,29 @@ const iString *withSpacesEncoded_String(const iString *d) {
     if (isDataUrl_String(d)) {
         return d;
     }
-    iString *enc = copy_String(d);
-    urlEncodeSpaces_String(enc);
-    return collect_String(enc);
+    /* Only make a copy if we need to modify the URL. */
+    if (indexOfCStr_String(d, " ") != iInvalidPos) {
+        iString *enc = copy_String(d);
+        urlEncodeSpaces_String(enc);
+        return collect_String(enc);
+    }
+    return d;
+}
+
+const iString *canonicalUrl_String(const iString *d) {
+    /* The "canonical" form, used for internal storage and comparisons, is:
+       - all non-reserved characters decoded (i.e., it's an IRI)
+       - expect for spaces, which are always `%20`
+       This means a canonical URL can be used on a gemtext link line without modifications. */
+    iString *canon = maybeUrlDecodeExclude_String(d, "/?:;#&= ");
+    /* `canon` may now be NULL if nothing was decoded. */
+    if (indexOfCStr_String(canon ? canon : d, " ") != iInvalidPos) {
+        if (!canon) {
+            canon = copy_String(d);
+        }
+        urlEncodeSpaces_String(canon);
+    }
+    return canon ? collect_String(canon) : d;
 }
 
 iRangecc mediaTypeWithoutParameters_Rangecc(iRangecc mime) {
diff --git a/src/gmutil.h b/src/gmutil.h
index e7ff7cc5..be6b471e 100644
--- a/src/gmutil.h
+++ b/src/gmutil.h
@@ -123,6 +123,7 @@ const char *    makeFileUrl_CStr        (const char *localFilePath);
 iString *       localFilePathFromUrl_String(const iString *);
 void            urlEncodeSpaces_String  (iString *);
 const iString * withSpacesEncoded_String(const iString *);
+const iString * canonicalUrl_String     (const iString *);
 
 const char *    mediaType_Path                      (const iString *path);
 iRangecc        mediaTypeWithoutParameters_Rangecc  (iRangecc mime);
diff --git a/src/ui/documentwidget.c b/src/ui/documentwidget.c
index cb1fde28..509933af 100644
--- a/src/ui/documentwidget.c
+++ b/src/ui/documentwidget.c
@@ -1470,7 +1470,7 @@ static void updateDocument_DocumentWidget_(iDocumentWidget *d,
                                 baseName_Path(collect_String(newRange_String(parts.path))).start;
                         }
                         format_String(&str, "=> %s %s\n",
-                                      cstr_String(withSpacesEncoded_String(d->mod.url)),
+                                      cstr_String(canonicalUrl_String(d->mod.url)),
                                       linkTitle);
                         setData_Media(media_GmDocument(d->doc),
                                       imgLinkId,
@@ -1638,7 +1638,7 @@ static void updateFromCachedResponse_DocumentWidget_(iDocumentWidget *d, float n
 }
 
 static iBool updateFromHistory_DocumentWidget_(iDocumentWidget *d) {
-    const iRecentUrl *recent = findUrl_History(d->mod.history, withSpacesEncoded_String(d->mod.url));
+    const iRecentUrl *recent = findUrl_History(d->mod.history, canonicalUrl_String(d->mod.url));
     if (recent && recent->cachedResponse) {
         iChangeFlags(d->flags,
                      openedFromSidebar_DocumentWidgetFlag,
@@ -1845,8 +1845,11 @@ static void checkResponse_DocumentWidget_(iDocumentWidget *d) {
         d->sourceStatus = statusCode;
         switch (category_GmStatusCode(statusCode)) {
             case categoryInput_GmStatusCode: {
+                /* Let the navigation history know that we have been to this URL even though
+                   it is only displayed as an input dialog. */
+                visitUrl_Visited(visited_App(), d->mod.url, transient_VisitedUrlFlag);
                 iUrl parts;
-                init_Url(&parts, d->mod.url);
+                init_Url(&parts, d->mod.url);                
                 iWidget *dlg = makeValueInput_Widget(
                     as_Widget(d),
                     NULL,
@@ -2646,11 +2649,11 @@ static iBool handleCommand_DocumentWidget_(iDocumentWidget *d, const char *cmd)
     }
     else if (equal_Command(cmd, "document.copylink") && document_App() == d) {
         if (d->contextLink) {
-            SDL_SetClipboardText(cstr_String(withSpacesEncoded_String(absoluteUrl_String(
+            SDL_SetClipboardText(cstr_String(canonicalUrl_String(absoluteUrl_String(
                 d->mod.url, linkUrl_GmDocument(d->doc, d->contextLink->linkId)))));
         }
         else {
-            SDL_SetClipboardText(cstr_String(withSpacesEncoded_String(d->mod.url)));
+            SDL_SetClipboardText(cstr_String(canonicalUrl_String(d->mod.url)));
         }
         return iTrue;
     }
@@ -4870,6 +4873,7 @@ void deserializeState_DocumentWidget(iDocumentWidget *d, iStream *ins) {
 }
 
 static void setUrl_DocumentWidget_(iDocumentWidget *d, const iString *url) {
+    url = canonicalUrl_String(url);
     if (!equal_String(d->mod.url, url)) {
         d->flags |= urlChanged_DocumentWidgetFlag;
         set_String(d->mod.url, url);
diff --git a/src/ui/inputwidget.c b/src/ui/inputwidget.c
index b108ee17..8ea21cff 100644
--- a/src/ui/inputwidget.c
+++ b/src/ui/inputwidget.c
@@ -903,7 +903,7 @@ static iBool copy_InputWidget_(iInputWidget *d, iBool doCut) {
         iString *str = collect_String(newUnicodeN_String(constAt_Array(&d->text, m.start),
                                                          size_Range(&m)));
         SDL_SetClipboardText(
-            cstr_String(d->inFlags & isUrl_InputWidgetFlag ? withSpacesEncoded_String(str) : str));
+            cstr_String(d->inFlags & isUrl_InputWidgetFlag ? canonicalUrl_String(str) : str));
         if (doCut) {
             pushUndo_InputWidget_(d);
             deleteMarked_InputWidget_(d);
diff --git a/src/ui/root.c b/src/ui/root.c
index 5266978b..c78c9632 100644
--- a/src/ui/root.c
+++ b/src/ui/root.c
@@ -725,7 +725,7 @@ static iBool handleNavBarCommands_(iWidget *navBar, const char *cmd) {
                 const iString *urlStr = collect_String(suffix_Command(cmd, "url"));
                 trimCache_App();
                 trimMemory_App();
-                visitUrl_Visited(visited_App(), withSpacesEncoded_String(urlStr), 0); /* TODO: internal URI normalization */
+                visitUrl_Visited(visited_App(), urlStr, 0); /* TODO: internal URI normalization */
                 postCommand_App("visited.changed"); /* sidebar will update */
                 setText_InputWidget(url, urlStr);
                 checkLoadAnimation_Root_(get_Root());
diff --git a/src/ui/sidebarwidget.c b/src/ui/sidebarwidget.c
index c0a22e99..4c2a9d64 100644
--- a/src/ui/sidebarwidget.c
+++ b/src/ui/sidebarwidget.c
@@ -217,7 +217,7 @@ static void updateItems_SidebarWidget_(iSidebarWidget *d) {
     iBool isEmpty = iFalse; /* show blank? */
     switch (d->mode) {
         case feeds_SidebarMode: {
-            const iString *docUrl = withSpacesEncoded_String(url_DocumentWidget(document_App()));
+            const iString *docUrl = canonicalUrl_String(url_DocumentWidget(document_App()));
                                     /* TODO: internal URI normalization */
             iTime now;
             iDate on;
@@ -1111,7 +1111,7 @@ static iBool processEvent_SidebarWidget_(iSidebarWidget *d, const SDL_Event *ev)
         else if (isCommand_Widget(w, ev, "bookmark.copy")) {
             const iSidebarItem *item = d->contextItem;
             if (d->mode == bookmarks_SidebarMode && item) {
-                SDL_SetClipboardText(cstr_String(withSpacesEncoded_String(&item->url)));
+                SDL_SetClipboardText(cstr_String(canonicalUrl_String(&item->url)));
             }
             return iTrue;
         }
@@ -1374,7 +1374,7 @@ static iBool processEvent_SidebarWidget_(iSidebarWidget *d, const SDL_Event *ev)
         else if (isCommand_Widget(w, ev, "history.copy")) {
             const iSidebarItem *item = d->contextItem;
             if (item && !isEmpty_String(&item->url)) {
-                SDL_SetClipboardText(cstr_String(withSpacesEncoded_String(&item->url)));
+                SDL_SetClipboardText(cstr_String(canonicalUrl_String(&item->url)));
             }
             return iTrue;
         }
diff --git a/src/visited.c b/src/visited.c
index f4769e97..94cff492 100644
--- a/src/visited.c
+++ b/src/visited.c
@@ -144,6 +144,7 @@ static size_t find_Visited_(const iVisited *d, const iString *url) {
 
 void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
     if (isEmpty_String(url)) return;
+    url = canonicalUrl_String(url);
     iVisitedUrl visit;
     init_VisitedUrl(&visit);
     visit.flags = visitFlags;
@@ -165,6 +166,7 @@ void visitUrl_Visited(iVisited *d, const iString *url, uint16_t visitFlags) {
 }
 
 void removeUrl_Visited(iVisited *d, const iString *url) {
+    url = canonicalUrl_String(url);
     iGuardMutex(d->mtx, {
         size_t pos = find_Visited_(d, url);
         if (pos < size_SortedArray(&d->visited)) {