More robust URL processing

Fixed a number of special cases revealed by the client torture test.

4 files changed, 137 insertions, 71 deletions

diff --git a/src/gmdocument.c b/src/gmdocument.c
index 0b792d56..116e532a 100644
--- a/src/gmdocument.c
+++ b/src/gmdocument.c
@@ -171,18 +171,17 @@ static iRangecc addLink_GmDocument_(iGmDocument *d, iRangecc line, iGmLinkId *li
     if (matchRange_RegExp(pattern, line, &m)) {
         iGmLink *link = new_GmLink();
         setRange_String(&link->url, capturedRange_RegExpMatch(&m, 1));
+        set_String(&link->url, absoluteUrl_String(&d->url, &link->url));
         /* Check the URL. */ {
             iUrl parts;
             init_Url(&parts, &link->url);
-            /* Host name. */
-            if (!isEmpty_Range(&parts.host) &&
-                !equalCase_Rangecc(&parts.host, cstr_String(&d->localHost))) {
+            if (!equalCase_Rangecc(&parts.host, cstr_String(&d->localHost))) {
                 link->flags |= remote_GmLinkFlag;
             }
-            if (!isEmpty_Range(&parts.protocol) && !equalCase_Rangecc(&parts.protocol, "gemini")) {
-                link->flags |= remote_GmLinkFlag;
+            if (startsWithCase_Rangecc(&parts.protocol, "gemini")) {
+                link->flags |= gemini_GmLinkFlag;
             }
-            if (startsWithCase_Rangecc(&parts.protocol, "http")) {
+            else if (startsWithCase_Rangecc(&parts.protocol, "http")) {
                 link->flags |= http_GmLinkFlag;
             }
             else if (equalCase_Rangecc(&parts.protocol, "gopher")) {
@@ -191,6 +190,9 @@ static iRangecc addLink_GmDocument_(iGmDocument *d, iRangecc line, iGmLinkId *li
             else if (equalCase_Rangecc(&parts.protocol, "file")) {
                 link->flags |= file_GmLinkFlag;
             }
+            else if (equalCase_Rangecc(&parts.protocol, "data")) {
+                link->flags |= data_GmLinkFlag;
+            }
             /* Check the file name extension, if present. */
             if (!isEmpty_Range(&parts.path)) {
                 iString *path = newRange_String(parts.path);
@@ -206,11 +208,9 @@ static iRangecc addLink_GmDocument_(iGmDocument *d, iRangecc line, iGmLinkId *li
                 }
                 delete_String(path);
             }
-        }
-        /* Check if visited. */ {
-            const iString *absUrl = absoluteUrl_String(&d->url, &link->url);
-            if (cmpString_String(absUrl, &d->url)) {
-                link->when = urlVisitTime_History(history_App(), absUrl);
+            /* Check if visited. */
+            if (cmpString_String(&link->url, &d->url)) {
+                link->when = urlVisitTime_History(history_App(), &link->url);
                 if (isValid_Time(&link->when)) {
                     link->flags |= visited_GmLinkFlag;
                 }
@@ -401,8 +401,10 @@ static void doLayout_GmDocument_(iGmDocument *d) {
             if (link->flags & remote_GmLinkFlag) {
                 run.visBounds.pos.x -= gap_UI / 2;
             }
-
             run.color = linkColor_GmDocument(d, run.linkId);
+            if (link->flags & visited_GmLinkFlag) {
+                run.color--; /* darker */
+            }
             pushBack_Array(&d->layout, &run);
         }
         run.color = colors[type];
@@ -554,7 +556,7 @@ static void normalize_GmDocument(iGmDocument *d) {
                         appendCStrN_String(normalized, " ", 1);
                     }
                 }
-                else {
+                else if (*ch != '\r') {
                     appendCStrN_String(normalized, ch, 1);
                 }
             }
@@ -573,6 +575,7 @@ static void normalize_GmDocument(iGmDocument *d) {
         iBool isPrevSpace = iFalse;
         for (const char *ch = line.start; ch != line.end; ch++) {
             char c = *ch;
+            if (c == '\r') continue;
             if (isNormalizableSpace_(c)) {
                 if (isPrevSpace) {
                     continue; /* skip repeated spaces */
@@ -738,6 +741,9 @@ uint16_t linkImage_GmDocument(const iGmDocument *d, iGmLinkId linkId) {
 enum iColorId linkColor_GmDocument(const iGmDocument *d, iGmLinkId linkId) {
     const iGmLink *link = link_GmDocument_(d, linkId);
     if (link) {
+        if ((link->flags & supportedProtocol_GmLinkFlag) == 0) {
+            return red_ColorId;
+        }
         return link->flags & http_GmLinkFlag
                    ? orange_ColorId
                    : link->flags & gopher_GmLinkFlag ? blue_ColorId : cyan_ColorId;
diff --git a/src/gmdocument.h b/src/gmdocument.h
index 92b9e1cb..db5a5451 100644
--- a/src/gmdocument.h
+++ b/src/gmdocument.h
@@ -13,15 +13,18 @@ iDeclareType(GmRun)
 typedef uint16_t iGmLinkId;
 
 enum iGmLinkFlags {
-    userFriendly_GmLinkFlag       = iBit(1),
-    remote_GmLinkFlag             = iBit(2),
+    gemini_GmLinkFlag             = iBit(1),
+    gopher_GmLinkFlag             = iBit(2),
     http_GmLinkFlag               = iBit(3),
-    gopher_GmLinkFlag             = iBit(4),
-    file_GmLinkFlag               = iBit(5),
-    imageFileExtension_GmLinkFlag = iBit(6),
-    audioFileExtension_GmLinkFlag = iBit(7),
-    content_GmLinkFlag            = iBit(8), /* content visible below */
-    visited_GmLinkFlag            = iBit(9), /* in the history */
+    file_GmLinkFlag               = iBit(4),
+    data_GmLinkFlag               = iBit(5),
+    supportedProtocol_GmLinkFlag  = 0x1f,
+    remote_GmLinkFlag             = iBit(9),
+    userFriendly_GmLinkFlag       = iBit(10),
+    imageFileExtension_GmLinkFlag = iBit(11),
+    audioFileExtension_GmLinkFlag = iBit(12),
+    content_GmLinkFlag            = iBit(13),  /* content visible below */
+    visited_GmLinkFlag            = iBit(14), /* in the history */
 };
 
 iDeclareType(GmImageInfo)
diff --git a/src/gmutil.c b/src/gmutil.c
index af090574..2008ea36 100644
--- a/src/gmutil.c
+++ b/src/gmutil.c
@@ -6,30 +6,36 @@
 
 void init_Url(iUrl *d, const iString *text) {
     iRegExp *absPat =
-        new_RegExp("(.+)://([^/:?]*)(:[0-9]+)?([^?]*)(\\?.*)?", caseInsensitive_RegExpOption);
+        new_RegExp("([a-z]+:)?(//[^/:?]*)(:[0-9]+)?([^?]*)(\\?.*)?", caseInsensitive_RegExpOption);
     iRegExpMatch m;
     if (matchString_RegExp(absPat, text, &m)) {
         d->protocol = capturedRange_RegExpMatch(&m, 1);
-        d->host     = capturedRange_RegExpMatch(&m, 2);
-        d->port     = capturedRange_RegExpMatch(&m, 3);
+        d->host = capturedRange_RegExpMatch(&m, 2);
+        if (!isEmpty_Range(&d->host)) {
+            d->host.start += 2; /* skip the double slash */
+        }
+        d->port = capturedRange_RegExpMatch(&m, 3);
         if (!isEmpty_Range(&d->port)) {
-            /* Don't include the colon. */
-            d->port.start++;
+            d->port.start++; /* omit the colon */
         }
-        d->path  = capturedRange_RegExpMatch(&m, 4);
+        d->path = capturedRange_RegExpMatch(&m, 4);
         d->query = capturedRange_RegExpMatch(&m, 5);
     }
     else {
         /* Must be a relative path. */
         iZap(*d);
-        iRegExp *relPat = new_RegExp("([^?]*)(\\?.*)?", 0);
+        iRegExp *relPat = new_RegExp("([a-z]+:)?([^?]*)(\\?.*)?", 0);
         if (matchString_RegExp(relPat, text, &m)) {
-            d->path  = capturedRange_RegExpMatch(&m, 1);
-            d->query = capturedRange_RegExpMatch(&m, 2);
+            d->protocol = capturedRange_RegExpMatch(&m, 1);
+            d->path     = capturedRange_RegExpMatch(&m, 2);
+            d->query    = capturedRange_RegExpMatch(&m, 3);
         }
         iRelease(relPat);
     }
     iRelease(absPat);
+    if (!isEmpty_Range(&d->protocol)) {
+        d->protocol.end--; /* omit the colon */
+    }
 }
 
 static iRangecc dirPath_(iRangecc path) {
@@ -38,51 +44,97 @@ static iRangecc dirPath_(iRangecc path) {
     return (iRangecc){ path.start, path.start + pos };
 }
 
+iLocalDef iBool isDef_(iRangecc cc) {
+    return !isEmpty_Range(&cc);
+}
+
+static iRangecc prevPathSeg_(const char *end, const char *start) {
+    iRangecc seg = { end, end };
+    do {
+        seg.start--;
+    } while (*seg.start != '/' && seg.start != start);
+    return seg;
+}
+
+void cleanUrlPath_String(iString *d) {
+    iString clean;
+    init_String(&clean);
+    iUrl parts;
+    init_Url(&parts, d);
+    iRangecc seg = iNullRange;
+    while (nextSplit_Rangecc(&parts.path, "/", &seg)) {
+        if (equal_Rangecc(&seg, "..")) {
+            /* Back up one segment. */
+            iRangecc last = prevPathSeg_(constEnd_String(&clean), constBegin_String(&clean));
+            truncate_Block(&clean.chars, last.start - constBegin_String(&clean));
+        }
+        else if (equal_Rangecc(&seg, ".")) {
+            /* Skip it. */
+        }
+        else {
+            appendCStr_String(&clean, "/");
+            appendRange_String(&clean, seg);
+        }
+    }
+    if (endsWith_Rangecc(&parts.path, "/")) {
+        appendCStr_String(&clean, "/");
+    }
+    /* Replace with the new path. */
+    if (cmpCStrNSc_Rangecc(&parts.path, cstr_String(&clean), size_String(&clean), &iCaseSensitive)) {
+        const size_t pos = parts.path.start - constBegin_String(d);
+        remove_Block(&d->chars, pos, size_Range(&parts.path));
+        insertData_Block(&d->chars, pos, cstr_String(&clean), size_String(&clean));
+    }
+    deinit_String(&clean);
+}
+
 const iString *absoluteUrl_String(const iString *d, const iString *urlMaybeRelative) {
-    if (indexOfCStr_String(urlMaybeRelative, "://") != iInvalidPos) {
-        /* Already absolute. */
+    iUrl orig;
+    iUrl rel;
+    init_Url(&orig, d);
+    init_Url(&rel, urlMaybeRelative);
+    if (equalCase_Rangecc(&rel.protocol, "data")) {
+        /* Special case, the contents should be left unparsed. */
         return urlMaybeRelative;
     }
-    iUrl parts;
-    init_Url(&parts, d);
-    iString *absolute = new_String();
-    appendRange_String(absolute, parts.protocol);
-    appendCStr_String(absolute, "://");
-    appendRange_String(absolute, parts.host);
-    if (!isEmpty_Range(&parts.port)) {
-        appendCStr_String(absolute, ":");
-        appendRange_String(absolute, parts.port);
+    const iBool isRelative = !isDef_(rel.host);
+    iRangecc protocol = range_CStr("gemini");
+    if (isDef_(rel.protocol)) {
+        protocol = rel.protocol;
+    }
+    else if (isRelative && isDef_(orig.protocol)) {
+        protocol = orig.protocol;
     }
-    if (startsWith_String(urlMaybeRelative, "/")) {
-        append_String(absolute, urlMaybeRelative);
+    iString *absolute = collectNew_String();
+    appendRange_String(absolute, protocol);
+    appendCStr_String(absolute, "://"); {
+        const iUrl *selHost = isDef_(rel.host) ? &rel : &orig;
+        appendRange_String(absolute, selHost->host);
+        if (!isEmpty_Range(&selHost->port)) {
+            appendCStr_String(absolute, ":");
+            appendRange_String(absolute, selHost->port);
+        }
+    }
+    if (isDef_(rel.protocol) || isDef_(rel.host) || startsWith_Rangecc(&rel.path, "/")) {
+        appendRange_String(absolute, rel.path); /* absolute path */
     }
     else {
-        iRangecc relPath = range_String(urlMaybeRelative);
-        iRangecc dir = dirPath_(parts.path);
-        for (;;) {
-            if (equal_Rangecc(&relPath, ".")) {
-                relPath.start++;
-            }
-            else if (startsWith_Rangecc(&relPath, "./")) {
-                relPath.start += 2;
-            }
-            else if (equal_Rangecc(&relPath, "..")) {
-                relPath.start += 2;
-                dir = dirPath_(dir);
-            }
-            else if (startsWith_Rangecc(&relPath, "../")) {
-                relPath.start += 3;
-                dir = dirPath_(dir);
-            }
-            else break;
+        if (!endsWith_Rangecc(&orig.path, "/")) {
+            /* Referencing a file. */
+            appendRange_String(absolute, dirPath_(orig.path));
+        }
+        else {
+            /* Referencing a directory. */
+            appendRange_String(absolute, orig.path);
         }
-        appendRange_String(absolute, dir);
         if (!endsWith_String(absolute, "/")) {
             appendCStr_String(absolute, "/");
         }
-        appendRange_String(absolute, relPath);
+        appendRange_String(absolute, rel.path);
     }
-    return collect_String(absolute);
+    appendRange_String(absolute, rel.query);
+    cleanUrlPath_String(absolute);
+    return absolute;
 }
 
 void urlEncodeSpaces_String(iString *d) {
diff --git a/src/ui/documentwidget.c b/src/ui/documentwidget.c
index 200bda95..3b8c468b 100644
--- a/src/ui/documentwidget.c
+++ b/src/ui/documentwidget.c
@@ -144,14 +144,20 @@ void deinit_DocumentWidget(iDocumentWidget *d) {
     SDL_FreeCursor(d->handCursor);
 }
 
+#if 0
 static iString *cleanUrl_(const iString *url) {
     iString *clean = copy_String(url);
-    if (indexOfCStr_String(url, "://") == iInvalidPos && !startsWithCase_String(url, "gemini:")) {
+    if (startsWith_String(url, "//")) {
+        prependCStr_String(clean, "gemini:");
+    }
+    else if (indexOfCStr_String(url, "://") == iInvalidPos && !startsWithCase_String(url, "gemini:")
+             && !startsWithCase_String(url, "data:")) {
         /* Prepend default protocol. */
         prependCStr_String(clean, "gemini://");
     }
     return clean;
 }
+#endif
 
 static int documentWidth_DocumentWidget_(const iDocumentWidget *d) {
     const iWidget *w = constAs_Widget(d);
@@ -201,7 +207,7 @@ static iRangei visibleRange_DocumentWidget_(const iDocumentWidget *d) {
 
 static void addVisibleLink_DocumentWidget_(void *context, const iGmRun *run) {
     iDocumentWidget *d = context;
-    if (run->linkId) {
+    if (run->linkId && linkFlags_GmDocument(d->doc, run->linkId) & supportedProtocol_GmLinkFlag) {
         pushBack_PtrArray(&d->visibleLinks, run);
     }
 }
@@ -363,9 +369,8 @@ static void fetch_DocumentWidget_(iDocumentWidget *d) {
 }
 
 void setUrl_DocumentWidget(iDocumentWidget *d, const iString *url) {
-    iString *newUrl = collect_String(cleanUrl_(url));
-    if (cmpStringSc_String(d->url, newUrl, &iCaseInsensitive)) {
-        set_String(d->url, newUrl);
+    if (cmpStringSc_String(d->url, url, &iCaseInsensitive)) {
+        set_String(d->url, url);
         fetch_DocumentWidget_(d);
     }
     /* See if there a username in the URL. */ {
@@ -861,7 +866,7 @@ static void drawRun_DrawContext_(void *context, const iGmRun *run) {
     fillRange_DrawContext_(d, run, teal_ColorId, d->widget->foundMark, &d->inFoundMark);
     fillRange_DrawContext_(d, run, brown_ColorId, d->widget->selectMark, &d->inSelectMark);
     if (run->linkId && !isEmpty_Rect(run->bounds)) {
-        const int flags = linkFlags_GmDocument(doc, run->linkId);
+//        const int flags = linkFlags_GmDocument(doc, run->linkId);
         fg = /*flags & visited_GmLinkFlag ? gray88_ColorId :*/ white_ColorId;
         if (isHover || linkFlags_GmDocument(doc, run->linkId) & content_GmLinkFlag) {
             fg = linkColor_GmDocument(doc, run->linkId);
@@ -913,7 +918,7 @@ static void drawRun_DrawContext_(void *context, const iGmRun *run) {
             init_Url(&parts, url);
             const iString *host = collect_String(newRange_String(parts.host));
             fg = linkColor_GmDocument(doc, linkId);
-            const iBool showHost = (!isEmpty_String(host) && flags & userFriendly_GmLinkFlag);
+            const iBool showHost  = (!isEmpty_String(host) && flags & userFriendly_GmLinkFlag);
             const iBool showImage = (flags & imageFileExtension_GmLinkFlag) != 0;
             const iBool showAudio = (flags & audioFileExtension_GmLinkFlag) != 0;
             iString str;