[core] Incremental offline downloads

platform/default/mbgl/storage/offline_download.cpp

@@ -3,6 +3,7 @@
 #include <mbgl/storage/offline_download.hpp>
 #include <mbgl/storage/resource.hpp>
 #include <mbgl/storage/response.hpp>
+#include <mbgl/storage/http_file_source.hpp>
 #include <mbgl/style/parser.hpp>
 #include <mbgl/style/sources/geojson_source_impl.hpp>
 #include <mbgl/style/tile_source_impl.hpp>
@@ -49,44 +50,6 @@ void OfflineDownload::setState(OfflineRegionDownloadState state) {
     observer->statusChanged(status);
 }
 
-std::vector<Resource> OfflineDownload::spriteResources(const style::Parser& parser) const {
-    std::vector<Resource> result;
-
-    if (!parser.spriteURL.empty()) {
-        result.push_back(Resource::spriteImage(parser.spriteURL, definition.pixelRatio));
-        result.push_back(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio));
-    }
-
-    return result;
-}
-
-std::vector<Resource> OfflineDownload::glyphResources(const style::Parser& parser) const {
-    std::vector<Resource> result;
-
-    if (!parser.glyphURL.empty()) {
-        for (const auto& fontStack : parser.fontStacks()) {
-            for (uint32_t i = 0; i < 256; i++) {
-                result.push_back(
-                    Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * 256)));
-            }
-        }
-    }
-
-    return result;
-}
-
-std::vector<Resource>
-OfflineDownload::tileResources(SourceType type, uint16_t tileSize, const Tileset& tileset) const {
-    std::vector<Resource> result;
-
-    for (const auto& tile : definition.tileCover(type, tileSize, tileset.zoomRange)) {
-        result.push_back(
-            Resource::tile(tileset.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z, tileset.scheme));
-    }
-
-    return result;
-}
-
 OfflineRegionStatus OfflineDownload::getStatus() const {
     if (status.downloadState == OfflineRegionDownloadState::Active) {
         return status;
@@ -106,29 +69,27 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
     result.requiredResourceCountIsPrecise = true;
 
     for (const auto& source : parser.sources) {
-        switch (source->baseImpl->type) {
+        SourceType type = source->baseImpl->type;
+
+        switch (type) {
         case SourceType::Vector:
         case SourceType::Raster: {
             style::TileSourceImpl* tileSource =
                 static_cast<style::TileSourceImpl*>(source->baseImpl.get());
             const variant<std::string, Tileset>& urlOrTileset = tileSource->getURLOrTileset();
+            const uint16_t tileSize = tileSource->getTileSize();
 
             if (urlOrTileset.is<Tileset>()) {
                 result.requiredResourceCount +=
-                    tileResources(source->baseImpl->type, tileSource->getTileSize(),
-                                  urlOrTileset.get<Tileset>())
-                        .size();
+                    definition.tileCover(type, tileSize, urlOrTileset.get<Tileset>().zoomRange).size();
             } else {
                 result.requiredResourceCount += 1;
                 const std::string& url = urlOrTileset.get<std::string>();
                 optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
                 if (sourceResponse) {
                     result.requiredResourceCount +=
-                        tileResources(source->baseImpl->type, tileSource->getTileSize(),
-                                      style::TileSourceImpl::parseTileJSON(
-                                          *sourceResponse->data, url, source->baseImpl->type,
-                                          tileSource->getTileSize()))
-                            .size();
+                        definition.tileCover(type, tileSize, style::TileSourceImpl::parseTileJSON(
+                            *sourceResponse->data, url, type, tileSize).zoomRange).size();
                 } else {
                     result.requiredResourceCountIsPrecise = false;
                 }
@@ -140,7 +101,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
             style::GeoJSONSource::Impl* geojsonSource =
                 static_cast<style::GeoJSONSource::Impl*>(source->baseImpl.get());
 
-            if (!geojsonSource->loaded) {
+            if (geojsonSource->getURL()) {
                 result.requiredResourceCount += 1;
             }
             break;
@@ -152,18 +113,21 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
         }
     }
 
-    result.requiredResourceCount += spriteResources(parser).size();
-    result.requiredResourceCount += glyphResources(parser).size();
+    if (!parser.glyphURL.empty()) {
+        result.requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK;
+    }
+
+    if (!parser.spriteURL.empty()) {
+        result.requiredResourceCount += 2;
+    }
 
     return result;
 }
 
 void OfflineDownload::activateDownload() {
     status = OfflineRegionStatus();
     status.downloadState = OfflineRegionDownloadState::Active;
-
-    requiredSourceURLs.clear();
-
+    status.requiredResourceCount++;
     ensureResource(Resource::style(definition.styleURL), [&](Response styleResponse) {
         status.requiredResourceCountIsPrecise = true;
 
@@ -182,15 +146,16 @@ void OfflineDownload::activateDownload() {
                 const uint16_t tileSize = tileSource->getTileSize();
 
                 if (urlOrTileset.is<Tileset>()) {
-                    ensureTiles(type, tileSize, urlOrTileset.get<Tileset>());
+                    queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
                 } else {
                     const std::string& url = urlOrTileset.get<std::string>();
                     status.requiredResourceCountIsPrecise = false;
+                    status.requiredResourceCount++;
                     requiredSourceURLs.insert(url);
 
                     ensureResource(Resource::source(url), [=](Response sourceResponse) {
-                        ensureTiles(type, tileSize, style::TileSourceImpl::parseTileJSON(
-                                                        *sourceResponse.data, url, type, tileSize));
+                        queueTiles(type, tileSize, style::TileSourceImpl::parseTileJSON(
+                            *sourceResponse.data, url, type, tileSize));
 
                         requiredSourceURLs.erase(url);
                         if (requiredSourceURLs.empty()) {
@@ -206,7 +171,7 @@ void OfflineDownload::activateDownload() {
                     static_cast<style::GeoJSONSource::Impl*>(source->baseImpl.get());
 
                 if (geojsonSource->getURL()) {
-                    ensureResource(Resource::source(*geojsonSource->getURL()));
+                    queueResource(Resource::source(*geojsonSource->getURL()));
                 }
                 break;
             }
@@ -217,30 +182,73 @@ void OfflineDownload::activateDownload() {
             }
         }
 
-        for (const auto& resource : spriteResources(parser)) {
-            ensureResource(resource);
+        if (!parser.glyphURL.empty()) {
+            for (const auto& fontStack : parser.fontStacks()) {
+                for (uint32_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
+                    queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
+                }
+            }
         }
 
-        for (const auto& resource : glyphResources(parser)) {
-            ensureResource(resource);
+        if (!parser.spriteURL.empty()) {
+            queueResource(Resource::spriteImage(parser.spriteURL, definition.pixelRatio));
+            queueResource(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio));
         }
+
+        continueDownload();
     });
 }
 
+/*
+   Fill up our own request queue by requesting the next few resources. This is called
+   when activating the download, or when a request completes successfully.
+
+   Note "successfully"; it's not called when a requests receives an error. A request
+   that errors will be retried after some delay. So in that sense it's still "active"
+   and consuming resources, notably the request object, its timer, and network resources
+   when the timer fires.
+
+   We could try to squeeze in subsequent requests while we wait for the errored request
+   to retry. But that risks overloading the upstream request queue -- defeating our own
+   metering -- if there are a lot of errored requests that all come up for retry at the
+   same time. And many times, the cause of a request error will apply to many requests
+   of the same type. For instance if a server is unreachable, all the requests to that
+   host are going to error. In that case, continuing to try subsequent resources after
+   the first few errors is fruitless anyway.
+*/
+void OfflineDownload::continueDownload() {
+    if (resourcesRemaining.empty() && status.complete()) {
+        setState(OfflineRegionDownloadState::Inactive);
+        return;
+    }
+
+    while (!resourcesRemaining.empty() && requests.size() < HTTPFileSource::maximumConcurrentRequests()) {
+        ensureResource(resourcesRemaining.front());
+        resourcesRemaining.pop_front();
+    }
+}
+
 void OfflineDownload::deactivateDownload() {
+    requiredSourceURLs.clear();
+    resourcesRemaining.clear();
     requests.clear();
 }
 
-void OfflineDownload::ensureTiles(SourceType type, uint16_t tileSize, const Tileset& info) {
-    for (const auto& resource : tileResources(type, tileSize, info)) {
-        ensureResource(resource);
+void OfflineDownload::queueResource(Resource resource) {
+    status.requiredResourceCount++;
+    resourcesRemaining.push_front(std::move(resource));
+}
+
+void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
+    for (const auto& tile : definition.tileCover(type, tileSize, tileset.zoomRange)) {
+        status.requiredResourceCount++;
+        resourcesRemaining.push_back(
+            Resource::tile(tileset.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z, tileset.scheme));
     }
 }
 
 void OfflineDownload::ensureResource(const Resource& resource,
                                      std::function<void(Response)> callback) {
-    status.requiredResourceCount++;
-
     auto workRequestsIt = requests.insert(requests.begin(), nullptr);
     *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
         requests.erase(workRequestsIt);
@@ -260,11 +268,7 @@ void OfflineDownload::ensureResource(const Resource& resource,
             }
 
             observer->statusChanged(status);
-
-            if (status.complete()) {
-                setState(OfflineRegionDownloadState::Inactive);
-            }
-
+            continueDownload();
             return;
         }
 
@@ -299,9 +303,7 @@ void OfflineDownload::ensureResource(const Resource& resource,
                 return;
             }
 
-            if (status.complete()) {
-                setState(OfflineRegionDownloadState::Inactive);
-            }
+            continueDownload();
         });
     });
 }

platform/default/mbgl/storage/offline_download.hpp

@@ -1,17 +1,18 @@
 #pragma once
 
 #include <mbgl/storage/offline.hpp>
+#include <mbgl/storage/resource.hpp>
 
 #include <list>
 #include <unordered_set>
 #include <memory>
+#include <deque>
 
 namespace mbgl {
 
 class OfflineDatabase;
 class FileSource;
 class AsyncRequest;
-class Resource;
 class Response;
 class Tileset;
 
@@ -36,19 +37,15 @@ class OfflineDownload {
 
 private:
     void activateDownload();
+    void continueDownload();
     void deactivateDownload();
 
-    std::vector<Resource> spriteResources(const style::Parser&) const;
-    std::vector<Resource> glyphResources(const style::Parser&) const;
-    std::vector<Resource> tileResources(SourceType, uint16_t, const Tileset&) const;
-
     /*
      * Ensure that the resource is stored in the database, requesting it if necessary.
      * While the request is in progress, it is recorded in `requests`. If the download
      * is deactivated, all in progress requests are cancelled.
      */
     void ensureResource(const Resource&, std::function<void (Response)> = {});
-    void ensureTiles(SourceType, uint16_t, const Tileset&);
     bool checkTileCountLimit(const Resource& resource);
 
     int64_t id;
@@ -57,8 +54,13 @@ class OfflineDownload {
     FileSource& onlineFileSource;
     OfflineRegionStatus status;
     std::unique_ptr<OfflineRegionObserver> observer;
+
     std::list<std::unique_ptr<AsyncRequest>> requests;
     std::unordered_set<std::string> requiredSourceURLs;
+    std::deque<Resource> resourcesRemaining;
+
+    void queueResource(Resource);
+    void queueTiles(SourceType, uint16_t tileSize, const Tileset&);
 };
 
 } // namespace mbgl

platform/default/online_file_source.cpp

@@ -193,7 +193,7 @@ std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource
         break;
     }
 
-    return std::make_unique<OnlineFileRequest>(res, callback, *impl);
+    return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl);
 }
 
 OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_)

src/mbgl/text/glyph_range.hpp

@@ -17,4 +17,7 @@ struct GlyphRangeHash {
 
 typedef std::unordered_set<GlyphRange, GlyphRangeHash> GlyphRangeSet;
 
+constexpr uint32_t GLYPHS_PER_GLYPH_RANGE = 256;
+constexpr uint32_t GLYPH_RANGES_PER_FONT_STACK = 256;
+
 } // end namespace mbgl

test/src/mbgl/test/fake_file_source.hpp

@@ -2,6 +2,7 @@
 
 #include <mbgl/storage/file_source.hpp>
 
+#include <algorithm>
 #include <list>
 
 namespace mbgl {

test/storage/offline_download.cpp

@@ -1,8 +1,10 @@
 #include <mbgl/test/stub_file_source.hpp>
+#include <mbgl/test/fake_file_source.hpp>
 
 #include <mbgl/storage/offline.hpp>
 #include <mbgl/storage/offline_database.hpp>
 #include <mbgl/storage/offline_download.hpp>
+#include <mbgl/storage/http_file_source.hpp>
 #include <mbgl/util/run_loop.hpp>
 #include <mbgl/util/io.hpp>
 #include <mbgl/util/compression.hpp>
@@ -240,6 +242,29 @@ TEST(OfflineDownload, Activate) {
     test.loop.run();
 }
 
+TEST(OfflineDownload, DoesNotFloodTheFileSourceWithRequests) {
+    FakeFileSource fileSource;
+    OfflineTest test;
+    OfflineRegion region = test.createRegion();
+    OfflineDownload download(
+        region.getID(),
+        OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0),
+        test.db, fileSource);
+
+    auto observer = std::make_unique<MockObserver>();
+
+    download.setObserver(std::move(observer));
+    download.setState(OfflineRegionDownloadState::Active);
+    test.loop.runOnce();
+
+    EXPECT_EQ(1u, fileSource.requests.size());
+
+    fileSource.respond(Resource::Kind::Style, test.response("style.json"));
+    test.loop.runOnce();
+
+    EXPECT_EQ(HTTPFileSource::maximumConcurrentRequests(), fileSource.requests.size());
+}
+
 TEST(OfflineDownload, GetStatusNoResources) {
     OfflineTest test;
     OfflineRegion region = test.createRegion();