summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/app.c3
-rw-r--r--src/feeds.c430
-rw-r--r--src/feeds.h40
3 files changed, 473 insertions, 0 deletions
diff --git a/src/app.c b/src/app.c
index 2e1d7be8..8db863c6 100644
--- a/src/app.c
+++ b/src/app.c
@@ -24,6 +24,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
24#include "bookmarks.h" 24#include "bookmarks.h"
25#include "defs.h" 25#include "defs.h"
26#include "embedded.h" 26#include "embedded.h"
27#include "feeds.h"
27#include "gmcerts.h" 28#include "gmcerts.h"
28#include "gmdocument.h" 29#include "gmdocument.h"
29#include "gmutil.h" 30#include "gmutil.h"
@@ -402,6 +403,7 @@ static void init_App_(iApp *d, int argc, char **argv) {
402 } 403 }
403#endif 404#endif
404 d->window = new_Window(d->initialWindowRect); 405 d->window = new_Window(d->initialWindowRect);
406 init_Feeds(dataDir_App_);
405 /* Widget state init. */ 407 /* Widget state init. */
406 processEvents_App(postedEventsOnly_AppEventMode); 408 processEvents_App(postedEventsOnly_AppEventMode);
407 if (!loadState_App_(d)) { 409 if (!loadState_App_(d)) {
@@ -435,6 +437,7 @@ static void init_App_(iApp *d, int argc, char **argv) {
435 437
436static void deinit_App(iApp *d) { 438static void deinit_App(iApp *d) {
437 saveState_App_(d); 439 saveState_App_(d);
440 deinit_Feeds();
438 save_Keys(dataDir_App_); 441 save_Keys(dataDir_App_);
439 deinit_Keys(); 442 deinit_Keys();
440 savePrefs_App_(d); 443 savePrefs_App_(d);
diff --git a/src/feeds.c b/src/feeds.c
new file mode 100644
index 00000000..cbfc36d0
--- /dev/null
+++ b/src/feeds.c
@@ -0,0 +1,430 @@
1/* Copyright 2020 Jaakko Keränen <jaakko.keranen@iki.fi>
2
3Redistribution and use in source and binary forms, with or without
4modification, are permitted provided that the following conditions are met:
5
61. Redistributions of source code must retain the above copyright notice, this
7 list of conditions and the following disclaimer.
82. Redistributions in binary form must reproduce the above copyright notice,
9 this list of conditions and the following disclaimer in the documentation
10 and/or other materials provided with the distribution.
11
12THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
13ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
16ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
22
23#include "feeds.h"
24#include "bookmarks.h"
25#include "gmrequest.h"
26#include "visited.h"
27#include "app.h"
28
29#include <the_Foundation/file.h>
30#include <the_Foundation/mutex.h>
31#include <the_Foundation/hash.h>
32#include <the_Foundation/queue.h>
33#include <the_Foundation/path.h>
34#include <the_Foundation/regexp.h>
35#include <the_Foundation/stringset.h>
36#include <the_Foundation/thread.h>
37#include <SDL_timer.h>
38
39iDeclareType(Feeds)
40iDeclareType(FeedJob)
41
42iDefineTypeConstruction(FeedEntry)
43
44void init_FeedEntry(iFeedEntry *d) {
45 iZap(d->timestamp);
46 init_String(&d->url);
47 init_String(&d->title);
48 d->bookmarkId = 0;
49}
50
51void deinit_FeedEntry(iFeedEntry *d) {
52 deinit_String(&d->title);
53 deinit_String(&d->url);
54}
55
56/*----------------------------------------------------------------------------------------------*/
57
58struct Impl_FeedJob {
59 iString url;
60 uint32_t bookmarkId;
61 iTime startTime;
62 iGmRequest *request;
63 iPtrArray results;
64};
65
66static void init_FeedJob(iFeedJob *d, const iBookmark *bookmark) {
67 initCopy_String(&d->url, &bookmark->url);
68 d->bookmarkId = id_Bookmark(bookmark);
69 d->request = NULL;
70 init_PtrArray(&d->results);
71 iZap(d->startTime);
72}
73
74static void deinit_FeedJob(iFeedJob *d) {
75 iRelease(d->request);
76 iForEach(PtrArray, i, &d->results) {
77 delete_FeedEntry(i.ptr);
78 }
79 deinit_PtrArray(&d->results);
80 deinit_String(&d->url);
81}
82
83iDefineTypeConstructionArgs(FeedJob, (const iBookmark *bm), bm)
84
85/*----------------------------------------------------------------------------------------------*/
86
87static const char *feedsFilename_Feeds_ = "feeds.txt";
88
89struct Impl_Feeds {
90 iMutex * mtx;
91 iString saveDir;
92 iTime lastRefreshedAt;
93 int refreshTimer;
94 iThread * worker;
95 iBool stopWorker;
96 iPtrArray jobs; /* pending */
97 iSortedArray entries; /* pointers to all discovered feed entries, sorted by entry ID (URL) */
98};
99
100static iFeeds feeds_;
101
102#define maxConcurrentRequests_Feeds 4
103
104static void submit_FeedJob_(iFeedJob *d) {
105 d->request = new_GmRequest(certs_App());
106 setUrl_GmRequest(d->request, &d->url);
107 initCurrent_Time(&d->startTime);
108 submit_GmRequest(d->request);
109}
110
111static iFeedJob *startNextJob_Feeds_(iFeeds *d) {
112 iFeedJob *job;
113 if (take_PtrArray(&d->jobs, 0, (void **) &job)) {
114 submit_FeedJob_(job);
115 return job;
116 }
117 return NULL;
118}
119
120static void parseResult_FeedJob_(iFeedJob *d) {
121 /* TODO: Should tell the user if the request failed. */
122 if (isSuccess_GmStatusCode(status_GmRequest(d->request))) {
123 iBeginCollect();
124 iRegExp *linkPattern =
125 new_RegExp("^=>\\s*([^\\s]+)\\s+"
126 "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])"
127 "([^0-9].*)",
128 0);
129 iString src;
130 initBlock_String(&src, body_GmRequest(d->request));
131 iRangecc srcLine = iNullRange;
132 while (nextSplit_Rangecc(range_String(&src), "\n", &srcLine)) {
133 iRangecc line = srcLine;
134 trimEnd_Rangecc(&line);
135 iRegExpMatch m;
136 init_RegExpMatch(&m);
137 if (matchRange_RegExp(linkPattern, line, &m)) {
138 const iRangecc url = capturedRange_RegExpMatch(&m, 1);
139 const iRangecc date = capturedRange_RegExpMatch(&m, 2);
140 const iRangecc title = capturedRange_RegExpMatch(&m, 3);
141 iFeedEntry *entry = new_FeedEntry();
142 entry->bookmarkId = d->bookmarkId;
143 setRange_String(&entry->url, url);
144 set_String(&entry->url, collect_String(lower_String(&entry->url)));
145 setRange_String(&entry->title, title);
146 int year, month, day;
147 sscanf(date.start, "%04d-%02d-%02d", &year, &month, &day);
148 init_Time(
149 &entry->timestamp,
150 &(iDate){
151 .year = year, .month = month, .day = day, .hour = 12 /* noon UTC */ });
152 pushBack_PtrArray(&d->results, entry);
153 }
154 }
155 deinit_String(&src);
156 iRelease(linkPattern);
157 iEndCollect();
158 }
159}
160
161static iBool updateEntries_Feeds_(iFeeds *d, iPtrArray *incoming) {
162 iBool gotNew = iFalse;
163 lock_Mutex(d->mtx);
164 iForEach(PtrArray, i, incoming) {
165 iFeedEntry *entry = i.ptr;
166 size_t pos;
167 if (locate_SortedArray(&d->entries, &entry, &pos)) {
168 iFeedEntry *existing = *(iFeedEntry **) at_SortedArray(&d->entries, pos);
169 /* Already known, but update it, maybe the time and label have changed. */
170 iBool changed = iFalse;
171 if (!equalCase_String(&existing->title, &entry->title) ||
172 cmp_Time(&existing->timestamp, &entry->timestamp)) {
173 changed = iTrue;
174 }
175 set_String(&existing->title, &entry->title);
176 existing->timestamp = entry->timestamp;
177 delete_FeedEntry(entry);
178 if (changed) {
179 /* TODO: better to use a new flag for read feed entries? */
180 removeUrl_Visited(visited_App(), &entry->url);
181 gotNew = iTrue;
182 }
183 }
184 else {
185 insert_SortedArray(&d->entries, &entry);
186 gotNew = iTrue;
187 }
188 remove_PtrArrayIterator(&i);
189 }
190 unlock_Mutex(d->mtx);
191 return gotNew;
192}
193
194static iThreadResult fetch_Feeds_(iThread *thread) {
195 iFeeds *d = &feeds_;
196 iUnused(thread);
197 iFeedJob *work[maxConcurrentRequests_Feeds]; /* We'll do a couple of concurrent requests. */
198 iZap(work);
199 iBool gotNew = iFalse;
200 postCommand_App("feeds.update.started");
201 while (!d->stopWorker) {
202 /* Start new jobs. */
203 iForIndices(i, work) {
204 if (!work[i]) {
205 work[i] = startNextJob_Feeds_(d);
206 }
207 }
208 sleep_Thread(0.5); /* TODO: wait on a Condition so we can exit quickly */
209 size_t ongoing = 0;
210 iForIndices(i, work) {
211 if (work[i]) {
212 if (isFinished_GmRequest(work[i]->request)) {
213 /* TODO: Handle redirects. Need to resubmit the job with new URL. */
214 parseResult_FeedJob_(work[i]);
215 gotNew |= updateEntries_Feeds_(d, &work[i]->results);
216 delete_FeedJob(work[i]);
217 work[i] = NULL;
218 }
219 else {
220 ongoing++;
221 }
222 /* TODO: abort job if it takes too long (> 15 seconds?) */
223 }
224 }
225 /* Stop if everything has finished. */
226 if (ongoing == 0 && isEmpty_PtrArray(&d->jobs)) {
227 break;
228 }
229 }
230 postCommandf_App("feeds.update.finished arg:%d", gotNew ? 1 : 0);
231 initCurrent_Time(&d->lastRefreshedAt);
232 return 0;
233}
234
235static iBool isSubscribed_(void *context, const iBookmark *bm) {
236 iUnused(context);
237 return indexOfCStr_String(&bm->tags, "subscribed") != iInvalidPos; /* TODO: RegExp with \b */
238}
239
240static iBool startWorker_Feeds_(iFeeds *d) {
241 if (d->worker) {
242 return iFalse; /* Oops? */
243 }
244 d->worker = new_Thread(fetch_Feeds_);
245 /* Queue up all the subscriptions for the worker. */
246 iConstForEach(PtrArray, i, list_Bookmarks(bookmarks_App(), NULL, isSubscribed_, NULL)) {
247 iFeedJob job;
248 init_FeedJob(&job, i.ptr);
249 pushBack_Array(&d->jobs, &job);
250 }
251 d->stopWorker = iFalse;
252 start_Thread(d->worker);
253 return iTrue;
254}
255
256static uint32_t refresh_Feeds_(uint32_t interval, void *data) {
257 /* Called in the SDL timer thread, so let's start a worker thread for running the update. */
258 startWorker_Feeds_(&feeds_);
259 return 1000 * 60 * 60;
260}
261
262static void stopWorker_Feeds_(iFeeds *d) {
263 if (d->worker) {
264 d->stopWorker = iTrue;
265 join_Thread(d->worker);
266 iReleasePtr(&d->worker);
267 }
268 /* TODO: Clear jobs */
269}
270
271static int cmp_FeedEntryPtr_(const void *a, const void *b) {
272 const iFeedEntry * const *elem[2] = { a, b };
273 return cmpStringCase_String(&(*elem[0])->url, &(*elem[1])->url);
274}
275
276static void save_Feeds_(iFeeds *d) {
277 iFile *f = new_File(collect_String(concatCStr_Path(&d->saveDir, feedsFilename_Feeds_)));
278 if (open_File(f, write_FileMode | text_FileMode)) {
279 lock_Mutex(d->mtx);
280 iString *str = new_String();
281 format_String(str, "%llu\n# Feeds\n", integralSeconds_Time(&d->lastRefreshedAt));
282 write_File(f, utf8_String(str));
283 /* Index of feeds for IDs. */ {
284 iConstForEach(PtrArray, i, list_Bookmarks(bookmarks_App(), NULL, isSubscribed_, NULL)) {
285 const iBookmark *bm = i.ptr;
286 format_String(str, "%08x %s\n", id_Bookmark(bm), cstr_String(&bm->url));
287 write_File(f, utf8_String(str));
288 }
289 }
290 writeData_File(f, "# Entries\n", 10);
291 iConstForEach(Array, i, &d->entries.values) {
292 const iFeedEntry *entry = *(const iFeedEntry **) i.value;
293 format_String(str, "%x\n%llu\n%s\n%s\n",
294 entry->bookmarkId,
295 integralSeconds_Time(&entry->timestamp),
296 cstr_String(&entry->url),
297 cstr_String(&entry->title));
298 write_File(f, utf8_String(str));
299 }
300 delete_String(str);
301 close_File(f);
302 unlock_Mutex(d->mtx);
303 }
304 iRelease(f);
305}
306
307iDeclareType(FeedHashNode)
308
309struct Impl_FeedHashNode {
310 iHashNode node;
311 uint32_t bookmarkId;
312};
313
314static void load_Feeds_(iFeeds *d) {
315 /* TODO: If there are lots of entries, it would make sense to load async. */
316 iFile *f = new_File(collect_String(concatCStr_Path(&d->saveDir, feedsFilename_Feeds_)));
317 if (open_File(f, read_FileMode | text_FileMode)) {
318 iBlock * src = readAll_File(f);
319 iRangecc line = iNullRange;
320 int section = 0;
321 iHash * feeds = new_Hash(); /* mapping from IDs to feed URLs */
322 while (nextSplit_Rangecc(range_Block(src), "\n", &line)) {
323 if (equal_Rangecc(line, "# Feeds")) {
324 section = 1;
325 continue;
326 }
327 else if (equal_Rangecc(line, "# Entries")) {
328 section = 2;
329 continue;
330 }
331 switch (section) {
332 case 0: {
333 unsigned long long ts = 0;
334 sscanf(line.start, "%llu", &ts);
335 d->lastRefreshedAt.ts.tv_sec = ts;
336 break;
337 }
338 case 1: {
339 if (size_Range(&line) > 8) {
340 uint32_t id = 0;
341 sscanf(line.start, "%08x", &id);
342 iString *feedUrl =
343 collect_String(newRange_String((iRangecc){ line.start + 9, line.end }));
344 const uint32_t bookmarkId = findUrl_Bookmarks(bookmarks_App(), feedUrl);
345 if (bookmarkId) {
346 iFeedHashNode *node = iMalloc(FeedHashNode);
347 node->node.key = id;
348 node->bookmarkId = bookmarkId;
349 insert_Hash(feeds, &node->node);
350 }
351 }
352 break;
353 }
354 case 2: {
355 const uint32_t feedId = strtoul(line.start, NULL, 16);
356 nextSplit_Rangecc(range_Block(src), "\n", &line);
357 const unsigned long long ts = strtoull(line.start, NULL, 10);
358 nextSplit_Rangecc(range_Block(src), "\n", &line);
359 iString url;
360 initRange_String(&url, line);
361 nextSplit_Rangecc(range_Block(src), "\n", &line);
362 iString title;
363 initRange_String(&title, line);
364 nextSplit_Rangecc(range_Block(src), "\n", &line);
365 /* Look it up in the hash. */
366 const iFeedHashNode *node = (iFeedHashNode *) value_Hash(feeds, feedId);
367 if (node) {
368 iFeedEntry *entry = new_FeedEntry();
369 entry->bookmarkId = node->bookmarkId;
370 entry->timestamp.ts.tv_sec = ts;
371 set_String(&entry->url, &url);
372 set_String(&entry->title, &title);
373 insert_SortedArray(&d->entries, &entry);
374 }
375 deinit_String(&title);
376 deinit_String(&url);
377 break;
378 }
379 }
380 }
381 /* Cleanup. */
382 delete_Block(src);
383 iForEach(Hash, i, feeds) {
384 free(i.value);
385 }
386 delete_Hash(feeds);
387 }
388 iRelease(f);
389}
390
391/*----------------------------------------------------------------------------------------------*/
392
393void init_Feeds(const char *saveDir) {
394 iFeeds *d = &feeds_;
395 d->mtx = new_Mutex();
396 initCStr_String(&d->saveDir, saveDir);
397 iZap(d->lastRefreshedAt);
398 d->worker = NULL;
399 init_PtrArray(&d->jobs);
400 init_SortedArray(&d->entries, sizeof(iFeedEntry *), cmp_FeedEntryPtr_);
401 load_Feeds_(d);
402 d->refreshTimer = SDL_AddTimer(5000, refresh_Feeds_, NULL);
403}
404
405void deinit_Feeds(void) {
406 iFeeds *d = &feeds_;
407 SDL_RemoveTimer(d->refreshTimer);
408 stopWorker_Feeds_(d);
409 iAssert(isEmpty_PtrArray(&d->jobs));
410 deinit_PtrArray(&d->jobs);
411 save_Feeds_(d);
412 deinit_String(&d->saveDir);
413 delete_Mutex(d->mtx);
414 iForEach(Array, i, &d->entries.values) {
415 iFeedEntry **entry = i.value;
416 delete_FeedEntry(*entry);
417 }
418 deinit_SortedArray(&d->entries);
419}
420
421const iPtrArray *listEntries_Feeds(void) {
422 iFeeds *d = &feeds_;
423 lock_Mutex(d->mtx);
424 /* The worker will never delete feed entries so we can use the same ones. Just make a copy
425 of the array in case the worker modifies it. */
426 iPtrArray *list = collect_PtrArray(copy_Array(&d->entries.values));
427 unlock_Mutex(d->mtx);
428 /* TODO: Sort the entries based on time. */
429 return list;
430}
diff --git a/src/feeds.h b/src/feeds.h
new file mode 100644
index 00000000..c5c3d447
--- /dev/null
+++ b/src/feeds.h
@@ -0,0 +1,40 @@
1/* Copyright 2020 Jaakko Keränen <jaakko.keranen@iki.fi>
2
3Redistribution and use in source and binary forms, with or without
4modification, are permitted provided that the following conditions are met:
5
61. Redistributions of source code must retain the above copyright notice, this
7 list of conditions and the following disclaimer.
82. Redistributions in binary form must reproduce the above copyright notice,
9 this list of conditions and the following disclaimer in the documentation
10 and/or other materials provided with the distribution.
11
12THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
13ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
14WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
15DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
16ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
17(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
18LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
19ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
21SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
22
23#include <the_Foundation/ptrarray.h>
24#include <the_Foundation/string.h>
25#include <the_Foundation/time.h>
26
27iDeclareType(FeedEntry)
28iDeclareTypeConstruction(FeedEntry)
29
30struct Impl_FeedEntry {
31 iTime timestamp;
32 iString url;
33 iString title;
34 uint32_t bookmarkId; /* note: runtime only, not a persistent ID */
35};
36
37void init_Feeds (const char *saveDir);
38void deinit_Feeds (void);
39
40const iPtrArray * listEntries_Feeds (void);