GCC Code Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 19.1% 42 / 0 / 220
Functions: 45.5% 10 / 0 / 22
Branches: 8.9% 11 / 0 / 124
shumate/shumate-tile-downloader.c
Line Branch Exec Source
1 /*
2 * Copyright (C) 2021 James Westman <james@jwestman.net>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, see <https://www.gnu.org/licenses/>.
16 */
17
18 #include <libsoup/soup.h>
19 #include "shumate-tile-downloader.h"
20 #include "shumate-file-cache.h"
21 #include "shumate-user-agent.h"
22 #include "shumate-profiling-private.h"
23 #include "shumate-data-source-request.h"
24
25 /**
26 * ShumateTileDownloader:
27 *
28 * A [class@DataSource] that asynchronously downloads tiles from an online
29 * service using a given template.
30 *
31 * It contains an internal [class@FileCache] to cache the tiles on the system.
32 */
33
34 struct _ShumateTileDownloader
35 {
36 ShumateDataSource parent_instance;
37
38 char *url_template;
39
40 SoupSession *soup_session;
41 ShumateFileCache *cache;
42 };
43
44
4/6
shumate_tile_downloader_class_intern_init:
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 7 times.
shumate_tile_downloader_get_type:
✓ Branch 2 → 3 taken 7 times.
✓ Branch 2 → 7 taken 18 times.
✓ Branch 4 → 5 taken 7 times.
✗ Branch 4 → 7 not taken.
 39 G_DEFINE_TYPE (ShumateTileDownloader, shumate_tile_downloader, SHUMATE_TYPE_DATA_SOURCE)
45
46 enum {
47 PROP_0,
48 PROP_URL_TEMPLATE,
49 N_PROPS
50 };
51 static GParamSpec *properties [N_PROPS];
52
53
54 /* The osm.org tile set require us to use no more than 2 simultaneous
55 * connections so let that be the default.
56 */
57 #define MAX_CONNS_DEFAULT 2
58
59
60 /**
61 * shumate_tile_downloader_new:
62 * @url_template: a URL template to fetch tiles from
63 *
64 * Creates a new [class@TileDownloader] that fetches tiles from an API and
65 * caches them on disk.
66 *
67 * See [property@TileDownloader:url-template] for the format of the URL template.
68 *
69 * Returns: (transfer full): a newly constructed [class@TileDownloader]
70 */
71 ShumateTileDownloader *
72 24 shumate_tile_downloader_new (const char *url_template)
73 {
74 24 return g_object_new (SHUMATE_TYPE_TILE_DOWNLOADER,
75 "url-template", url_template,
76 NULL);
77 }
78
79 static void
80 24 shumate_tile_downloader_constructed (GObject *object)
81 {
82 24 ShumateTileDownloader *self = (ShumateTileDownloader *)object;
83 24 g_autofree char *cache_key = NULL;
84
85
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 24 times.
 48 cache_key = g_strcanon (g_strdup (self->url_template), "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", '_');
86 24 self->cache = shumate_file_cache_new_full (100 * 1000 * 1000, cache_key, NULL);
87
88 24 G_OBJECT_CLASS (shumate_tile_downloader_parent_class)->constructed (object);
89 24 }
90
91 static void
92 24 shumate_tile_downloader_finalize (GObject *object)
93 {
94 24 ShumateTileDownloader *self = (ShumateTileDownloader *)object;
95
96
1/2
✓ Branch 2 → 3 taken 24 times.
✗ Branch 2 → 4 not taken.
 24 g_clear_pointer (&self->url_template, g_free);
97
1/2
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 24 times.
 24 g_clear_object (&self->soup_session);
98
1/2
✓ Branch 6 → 7 taken 24 times.
✗ Branch 6 → 8 not taken.
 24 g_clear_object (&self->cache);
99
100 24 G_OBJECT_CLASS (shumate_tile_downloader_parent_class)->finalize (object);
101 24 }
102
103 static void
104 ✗ shumate_tile_downloader_get_property (GObject *object,
105 guint prop_id,
106 GValue *value,
107 GParamSpec *pspec)
108 {
109 ✗ ShumateTileDownloader *self = SHUMATE_TILE_DOWNLOADER (object);
110
111 ✗ switch (prop_id)
112 {
113 ✗ case PROP_URL_TEMPLATE:
114 ✗ g_value_set_string (value, self->url_template);
115 ✗ break;
116 ✗ default:
117 ✗ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
118 }
119 ✗ }
120
121 static void
122 24 shumate_tile_downloader_set_property (GObject *object,
123 guint prop_id,
124 const GValue *value,
125 GParamSpec *pspec)
126 {
127 24 ShumateTileDownloader *self = SHUMATE_TILE_DOWNLOADER (object);
128
129
1/2
✓ Branch 2 → 3 taken 24 times.
✗ Branch 2 → 13 not taken.
 24 switch (prop_id)
130 {
131 24 case PROP_URL_TEMPLATE:
132
1/2
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 24 times.
 24 self->url_template = g_strdup (g_value_get_string (value));
133 24 break;
134 ✗ default:
135 ✗ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
136 }
137 24 }
138
139
140 static void
141 get_tile_data_async (ShumateDataSource *data_source,
142 int x,
143 int y,
144 int zoom_level,
145 GCancellable *cancellable,
146 GAsyncReadyCallback callback,
147 gpointer user_data);
148
149 static ShumateDataSourceRequest *
150 start_request (ShumateDataSource *data_source,
151 int x,
152 int y,
153 int zoom_level,
154 GCancellable *cancellable);
155
156 static void
157 7 shumate_tile_downloader_class_init (ShumateTileDownloaderClass *klass)
158 {
159 7 GObjectClass *object_class = G_OBJECT_CLASS (klass);
160 7 ShumateDataSourceClass *source_class = SHUMATE_DATA_SOURCE_CLASS (klass);
161
162 7 object_class->constructed = shumate_tile_downloader_constructed;
163 7 object_class->finalize = shumate_tile_downloader_finalize;
164 7 object_class->get_property = shumate_tile_downloader_get_property;
165 7 object_class->set_property = shumate_tile_downloader_set_property;
166
167 7 source_class->get_tile_data_async = get_tile_data_async;
168 7 source_class->start_request = start_request;
169
170 /**
171 * ShumateTileDownloader:url-template:
172 *
173 * A template for construting the URL to download a tile from.
174 *
175 * The template has the following replacements:
176 * - "{x}": The X coordinate of the tile
177 * - "{y}": The Y coordinate of the tile
178 * - "{z}": The zoom level of the tile
179 * - "{tmsy}": The inverted Y coordinate (i.e. tile numbering starts with 0 at
180 * the bottom, rather than top, of the map)
181 */
182 14 properties[PROP_URL_TEMPLATE] =
183 7 g_param_spec_string ("url-template", NULL, NULL,
184 NULL,
185 G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
186
187 7 g_object_class_install_properties (object_class, N_PROPS, properties);
188 7 }
189
190 static void
191 24 shumate_tile_downloader_init (ShumateTileDownloader *self)
192 {
193 24 }
194
195
196 typedef struct {
197 ShumateTileDownloader *self;
198 ShumateDataSourceRequest *req;
199 GCancellable *cancellable;
200 char *etag;
201 SoupMessage *msg;
202 GDateTime *modtime;
203 } FillTileData;
204
205 static void
206 ✗ fill_tile_data_free (FillTileData *data)
207 {
208 ✗ g_clear_object (&data->self);
209 ✗ g_clear_object (&data->req);
210 ✗ g_clear_object (&data->cancellable);
211 ✗ g_clear_pointer (&data->etag, g_free);
212 ✗ g_clear_object (&data->msg);
213 ✗ g_clear_pointer (&data->modtime, g_date_time_unref);
214 ✗ g_free (data);
215 ✗ }
216
217 ✗ G_DEFINE_AUTOPTR_CLEANUP_FUNC (FillTileData, fill_tile_data_free)
218
219 static void on_file_cache_get_tile (GObject *source_object, GAsyncResult *res, gpointer user_data);
220 static void fetch_from_network (FillTileData *data);
221 static void on_message_sent (GObject *source_object, GAsyncResult *res, gpointer user_data);
222 static void on_message_read (GObject *source_object, GAsyncResult *res, gpointer user_data);
223
224
225 static gboolean
226 ✗ tile_is_expired (GDateTime *modified_time)
227 {
228 ✗ g_autoptr(GDateTime) now = g_date_time_new_now_utc ();
229 ✗ GTimeSpan diff = g_date_time_difference (now, modified_time);
230
231 ✗ return diff > 7 * G_TIME_SPAN_DAY; /* Cache expires in 7 days */
232 }
233
234 static char *
235 ✗ get_modified_time_string (GDateTime *modified_time)
236 {
237 ✗ if (modified_time == NULL)
238 return NULL;
239
240 ✗ return g_date_time_format (modified_time, "%a, %d %b %Y %T %Z");
241 }
242
243
244 static char *
245 ✗ get_tile_uri (ShumateTileDownloader *self,
246 int x,
247 int y,
248 int z)
249 {
250 ✗ GString *string = g_string_new (self->url_template);
251 ✗ g_autofree char *x_str = g_strdup_printf("%d", x);
252 ✗ g_autofree char *y_str = g_strdup_printf("%d", y);
253 ✗ g_autofree char *z_str = g_strdup_printf("%d", z);
254 ✗ g_autofree char *tmsy_str = g_strdup_printf("%d", (1 << z) - y - 1);
255
256 ✗ g_string_replace (string, "{x}", x_str, 0);
257 ✗ g_string_replace (string, "{y}", y_str, 0);
258 ✗ g_string_replace (string, "{z}", z_str, 0);
259 ✗ g_string_replace (string, "{tmsy}", tmsy_str, 0);
260
261 ✗ return g_string_free (string, FALSE);
262 }
263
264 static void
265 ✗ on_request_notify_completed (GTask *task,
266 GParamSpec *pspec,
267 ShumateDataSourceRequest *req)
268 {
269 ✗ GBytes *bytes;
270
271 ✗ if ((bytes = shumate_data_source_request_get_data (req)))
272 ✗ g_task_return_pointer (task, g_bytes_ref (bytes), (GDestroyNotify)g_bytes_unref);
273 else
274 ✗ g_task_return_error (task, g_error_copy (shumate_data_source_request_get_error (req)));
275
276 ✗ g_clear_object (&task);
277 ✗ }
278
279 static void
280 ✗ on_request_notify_data (ShumateTileDownloader *self,
281 GParamSpec *pspec,
282 ShumateDataSourceRequest *req)
283 {
284 ✗ GBytes *bytes = shumate_data_source_request_get_data (req);
285
286 ✗ if (bytes != NULL)
287 {
288 ✗ int x = shumate_data_source_request_get_x (req);
289 ✗ int y = shumate_data_source_request_get_y (req);
290 ✗ int z = shumate_data_source_request_get_zoom_level (req);
291 ✗ g_autofree char *profiling_desc = g_strdup_printf ("(%d, %d) @ %d", x, y, z);
292
293 ✗ SHUMATE_PROFILE_START_NAMED (emit_received_data);
294 ✗ g_signal_emit_by_name (self, "received-data", x, y, z, bytes);
295 ✗ SHUMATE_PROFILE_END_NAMED (emit_received_data, profiling_desc);
296 }
297 ✗ }
298
299 static void
300 ✗ get_tile_data_async (ShumateDataSource *data_source,
301 int x,
302 int y,
303 int zoom_level,
304 GCancellable *cancellable,
305 GAsyncReadyCallback callback,
306 gpointer user_data)
307 {
308 ✗ g_autoptr(GTask) task = NULL;
309 ✗ ShumateTileDownloader *self = (ShumateTileDownloader *)data_source;
310 ✗ g_autoptr(ShumateDataSourceRequest) req = NULL;
311
312 ✗ g_return_if_fail (SHUMATE_IS_TILE_DOWNLOADER (self));
313 ✗ g_return_if_fail (!cancellable || G_IS_CANCELLABLE (cancellable));
314
315 ✗ task = g_task_new (self, cancellable, callback, user_data);
316 ✗ g_task_set_source_tag (task, get_tile_data_async);
317
318 ✗ req = start_request ((ShumateDataSource *)self, x, y, zoom_level, cancellable);
319
320 ✗ if (shumate_data_source_request_is_completed (req))
321 {
322 ✗ on_request_notify_data (self, NULL, req);
323 ✗ on_request_notify_completed (task, NULL, req);
324 }
325 else
326 {
327 ✗ g_signal_connect_object (req,
328 "notify::data",
329 (GCallback)on_request_notify_data,
330 self,
331 G_CONNECT_SWAPPED);
332 ✗ g_signal_connect_object (req,
333 "notify::completed",
334 (GCallback)on_request_notify_completed,
335 g_steal_pointer (&task),
336 G_CONNECT_SWAPPED);
337 }
338 }
339
340 static ShumateDataSourceRequest *
341 ✗ start_request (ShumateDataSource *data_source,
342 int x,
343 int y,
344 int zoom_level,
345 GCancellable *cancellable)
346 {
347 ✗ ShumateTileDownloader *self = (ShumateTileDownloader *)data_source;
348 ✗ g_autoptr(ShumateDataSourceRequest) req = NULL;
349 ✗ FillTileData *data;
350
351 ✗ req = shumate_data_source_request_new (x, y, zoom_level);
352 ✗ data = g_new0 (FillTileData, 1);
353 ✗ data->self = g_object_ref (self);
354 ✗ data->req = g_object_ref (req);
355 ✗ data->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
356
357 ✗ shumate_file_cache_get_tile_async (self->cache, x, y, zoom_level, cancellable, on_file_cache_get_tile, data);
358
359 ✗ return g_steal_pointer (&req);
360 }
361
362 static void
363 ✗ on_file_cache_get_tile (GObject *source_object, GAsyncResult *res, gpointer user_data)
364 {
365 ✗ g_autoptr(FillTileData) data = user_data;
366 ✗ g_autoptr(GBytes) bytes = NULL;
367
368 ✗ bytes = shumate_file_cache_get_tile_finish (SHUMATE_FILE_CACHE (source_object),
369 &data->etag, &data->modtime, res, NULL);
370
371 ✗ if (bytes != NULL)
372 {
373 ✗ gboolean complete = !tile_is_expired (data->modtime);
374
375 ✗ shumate_data_source_request_emit_data (data->req, bytes, complete);
376
377 ✗ if (complete)
378 ✗ return;
379 }
380
381 ✗ fetch_from_network (g_steal_pointer (&data));
382 }
383
384 static void
385 ✗ fetch_from_network (FillTileData *data_arg)
386 {
387 ✗ g_autoptr(FillTileData) data = data_arg;
388 ✗ g_autofree char *uri = NULL;
389 ✗ g_autofree char *modtime_string = NULL;
390 ✗ SoupMessageHeaders *headers;
391 ✗ int x, y, z;
392
393 ✗ if (g_cancellable_is_cancelled (data->cancellable))
394 {
395 ✗ g_autoptr(GError) error = g_error_new (G_IO_ERROR, G_IO_ERROR_CANCELLED, "Cancelled");
396 ✗ shumate_data_source_request_emit_error (data->req, error);
397 ✗ return;
398 }
399
400 ✗ x = shumate_data_source_request_get_x (data->req);
401 ✗ y = shumate_data_source_request_get_y (data->req);
402 ✗ z = shumate_data_source_request_get_zoom_level (data->req);
403 ✗ uri = get_tile_uri (data->self, x, y, z);
404
405 ✗ data->msg = soup_message_new (SOUP_METHOD_GET, uri);
406
407 ✗ if (data->msg == NULL)
408 {
409 ✗ g_autoptr(GError) error =
410 ✗ g_error_new (SHUMATE_TILE_DOWNLOADER_ERROR,
411 SHUMATE_TILE_DOWNLOADER_ERROR_MALFORMED_URL,
412 "The URL %s is not valid", uri);
413 ✗ shumate_data_source_request_emit_error (data->req, error);
414 ✗ return;
415 }
416
417 ✗ modtime_string = get_modified_time_string (data->modtime);
418 ✗ headers = soup_message_get_request_headers (data->msg);
419
420 /* If an etag is available, only use it.
421 * OSM servers seems to send now as the modified time for all tiles
422 * Omarender servers set the modified time correctly
423 */
424 ✗ if (data->etag)
425 {
426 ✗ g_debug ("If-None-Match: %s", data->etag);
427 ✗ soup_message_headers_append (headers, "If-None-Match", data->etag);
428 }
429 ✗ else if (modtime_string)
430 {
431 ✗ g_debug ("If-Modified-Since %s", modtime_string);
432 ✗ soup_message_headers_append (headers, "If-Modified-Since", modtime_string);
433 }
434
435 ✗ if (data->self->soup_session == NULL)
436 {
437 ✗ data->self->soup_session =
438 ✗ soup_session_new_with_options ("user-agent", "libshumate/" SHUMATE_VERSION,
439 "max-conns-per-host", MAX_CONNS_DEFAULT,
440 "max-conns", MAX_CONNS_DEFAULT,
441 NULL);
442 }
443
444 ✗ soup_session_send_async (data->self->soup_session,
445 data->msg,
446 G_PRIORITY_DEFAULT,
447 data->cancellable,
448 on_message_sent,
449 data);
450 ✗ data = NULL;
451 }
452
453 /* Receive the response from the network. If the tile hasn't been modified,
454 * return; otherwise, read the data into a GBytes. */
455 static void
456 ✗ on_message_sent (GObject *source_object, GAsyncResult *res, gpointer user_data)
457 {
458 ✗ g_autoptr(FillTileData) data = user_data;
459 ✗ g_autoptr(GInputStream) input_stream = NULL;
460 ✗ g_autoptr(GError) error = NULL;
461 ✗ g_autoptr(GOutputStream) output_stream = NULL;
462 ✗ SoupStatus status;
463 ✗ SoupMessageHeaders *headers;
464
465 ✗ input_stream = soup_session_send_finish (data->self->soup_session, res, &error);
466 ✗ if (error != NULL)
467 {
468 ✗ if (shumate_data_source_request_get_data (data->req))
469 {
470 /* The tile has already been filled from the cache, so the operation
471 * was overall successful even though the network request failed. */
472 ✗ g_debug ("Fetching tile failed, but there is a cached version (error: %s)", error->message);
473 ✗ shumate_data_source_request_complete (data->req);
474 }
475 else
476 ✗ shumate_data_source_request_emit_error (data->req, error);
477
478 return;
479 }
480
481 ✗ status = soup_message_get_status (data->msg);
482 ✗ headers = soup_message_get_response_headers (data->msg);
483
484 ✗ g_debug ("Got reply %d", status);
485
486 ✗ if (status == SOUP_STATUS_NOT_MODIFIED)
487 {
488 /* The tile has already been filled from the cache, and the server says
489 * it doesn't have a newer one. Just update the cache and return. */
490
491 ✗ int x, y, z;
492
493 ✗ x = shumate_data_source_request_get_x (data->req);
494 ✗ y = shumate_data_source_request_get_y (data->req);
495 ✗ z = shumate_data_source_request_get_zoom_level (data->req);
496
497 ✗ shumate_file_cache_mark_up_to_date (data->self->cache, x, y, z);
498 ✗ shumate_data_source_request_complete (data->req);
499 ✗ return;
500 }
501
502 ✗ if (!SOUP_STATUS_IS_SUCCESSFUL (status))
503 {
504 ✗ if (shumate_data_source_request_get_data (data->req))
505 {
506 ✗ g_debug ("Fetching tile failed, but there is a cached version (HTTP %s)",
507 soup_status_get_phrase (status));
508 ✗ shumate_data_source_request_complete (data->req);
509 }
510 else
511 {
512 ✗ g_autoptr(GError) error =
513 ✗ g_error_new (SHUMATE_TILE_DOWNLOADER_ERROR,
514 SHUMATE_TILE_DOWNLOADER_ERROR_BAD_RESPONSE,
515 "Unable to download tile: HTTP %s",
516 soup_status_get_phrase (status));
517 ✗ shumate_data_source_request_emit_error (data->req, error);
518 }
519
520 return;
521 }
522
523 /* Verify if the server sent an etag and save it */
524 ✗ g_clear_pointer (&data->etag, g_free);
525 ✗ data->etag = g_strdup (soup_message_headers_get_one (headers, "ETag"));
526 ✗ g_debug ("Received ETag %s", data->etag);
527
528 ✗ output_stream = g_memory_output_stream_new_resizable ();
529 ✗ g_output_stream_splice_async (output_stream,
530 input_stream,
531 G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
532 G_PRIORITY_DEFAULT,
533 data->cancellable,
534 on_message_read,
535 data);
536 ✗ data = NULL;
537 }
538
539 static void
540 ✗ on_message_read (GObject *source_object, GAsyncResult *res, gpointer user_data)
541 {
542 ✗ g_autoptr(FillTileData) data = user_data;
543 ✗ GOutputStream *output_stream = G_OUTPUT_STREAM (source_object);
544 ✗ g_autoptr(GError) error = NULL;
545 ✗ g_autoptr (GBytes) bytes = NULL;
546 ✗ int x = shumate_data_source_request_get_x (data->req);
547 ✗ int y = shumate_data_source_request_get_y (data->req);
548 ✗ int z = shumate_data_source_request_get_zoom_level (data->req);
549
550 ✗ g_output_stream_splice_finish (output_stream, res, &error);
551 ✗ if (error != NULL)
552 {
553 ✗ shumate_data_source_request_emit_error (data->req, error);
554 ✗ return;
555 }
556
557 ✗ bytes = g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (output_stream));
558 ✗ shumate_data_source_request_emit_data (data->req, bytes, TRUE);
559
560 ✗ shumate_file_cache_store_tile_async (data->self->cache,
561 x, y, z,
562 bytes,
563 ✗ data->etag,
564 NULL,
565 NULL,
566 NULL);
567 }
568
569
570 /**
571 * shumate_tile_downloader_error_quark:
572 *
573 * Gets the #ShumateTileDownloader error quark.
574 *
575 * Returns: a #GQuark
576 */
577
1/2
✓ Branch 2 → 3 taken 1 time.
✗ Branch 2 → 5 not taken.
 1 G_DEFINE_QUARK (shumate-tile-downloader-error-quark, shumate_tile_downloader_error);
578