diff --git a/.gitmodules b/.gitmodules index 9876bf6711b..72537dd37ef 100644 --- a/.gitmodules +++ b/.gitmodules @@ -55,3 +55,6 @@ [submodule "vendor/glslang"] path = vendor/glslang url = https://github.com/KhronosGroup/glslang.git +[submodule "vendor/PMTiles"] + path = vendor/PMTiles + url = https://github.com/protomaps/PMTiles.git diff --git a/BUILD.bazel b/BUILD.bazel index 9e7d53e0373..80c7ccf3411 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -129,6 +129,7 @@ cc_library( "//vendor:eternal", "//vendor:mapbox-base", "//vendor:parsedate", + "//vendor:pmtiles", "//vendor:polylabel", "//vendor:protozero", "//vendor:unique_resource", diff --git a/CMakeLists.txt b/CMakeLists.txt index 36258a3b4b1..835aa2b7eec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ option(MLN_WITH_OPENGL "Build with OpenGL renderer" ON) option(MLN_WITH_EGL "Build with EGL renderer" OFF) option(MLN_WITH_VULKAN "Build with Vulkan renderer" OFF) option(MLN_WITH_OSMESA "Build with OSMesa (Software) renderer" OFF) +option(MLN_WITH_PMTILES "Build with PMTiles support" ON) option(MLN_WITH_WERROR "Make all compilation warnings errors" ON) option(MLN_LEGACY_RENDERER "Include the legacy rendering pathway" ON) option(MLN_DRAWABLE_RENDERER "Include the drawable rendering pathway" OFF) @@ -1447,6 +1448,7 @@ include(${PROJECT_SOURCE_DIR}/vendor/earcut.hpp.cmake) include(${PROJECT_SOURCE_DIR}/vendor/eternal.cmake) include(${PROJECT_SOURCE_DIR}/vendor/mapbox-base.cmake) include(${PROJECT_SOURCE_DIR}/vendor/parsedate.cmake) +include(${PROJECT_SOURCE_DIR}/vendor/pmtiles.cmake) include(${PROJECT_SOURCE_DIR}/vendor/polylabel.cmake) include(${PROJECT_SOURCE_DIR}/vendor/protozero.cmake) include(${PROJECT_SOURCE_DIR}/vendor/tracy.cmake) @@ -1470,6 +1472,7 @@ target_link_libraries( mbgl-vendor-earcut.hpp mbgl-vendor-eternal mbgl-vendor-parsedate + mbgl-vendor-pmtiles mbgl-vendor-polylabel mbgl-vendor-protozero mbgl-vendor-unique_resource @@ -1509,6 +1512,7 @@ export(TARGETS mbgl-vendor-earcut.hpp mbgl-vendor-eternal mbgl-vendor-parsedate + mbgl-vendor-pmtiles mbgl-vendor-polylabel mbgl-vendor-protozero mbgl-vendor-unique_resource diff --git a/bazel/core.bzl b/bazel/core.bzl index 7f4ca59f53d..6d7dbca391d 100644 --- a/bazel/core.bzl +++ b/bazel/core.bzl @@ -379,6 +379,7 @@ MLN_CORE_SOURCE = [ "src/mbgl/storage/local_file_source.hpp", "src/mbgl/storage/main_resource_loader.hpp", "src/mbgl/storage/network_status.cpp", + "src/mbgl/storage/pmtiles_file_source.hpp", "src/mbgl/storage/resource.cpp", "src/mbgl/storage/resource_options.cpp", "src/mbgl/storage/resource_transform.cpp", diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp index 4c896c4d6fc..6a70154df41 100644 --- a/include/mbgl/storage/file_source.hpp +++ b/include/mbgl/storage/file_source.hpp @@ -25,6 +25,7 @@ enum FileSourceType : uint8_t { FileSystem, Network, Mbtiles, + Pmtiles, ResourceLoader ///< %Resource loader acts as a proxy and has logic /// for request delegation to Asset, Cache, and other /// file sources. diff --git a/include/mbgl/storage/resource.hpp b/include/mbgl/storage/resource.hpp index 114bd2f7640..26fda2752ba 100644 --- a/include/mbgl/storage/resource.hpp +++ b/include/mbgl/storage/resource.hpp @@ -95,6 +95,7 @@ class Resource { // Includes auxiliary data if this is a tile request. std::optional tileData; + std::optional> dataRange = std::nullopt; std::optional priorModified = std::nullopt; std::optional priorExpires = std::nullopt; std::optional priorEtag = std::nullopt; diff --git a/include/mbgl/util/constants.hpp b/include/mbgl/util/constants.hpp index 51900f93ed1..847c72a2548 100644 --- a/include/mbgl/util/constants.hpp +++ b/include/mbgl/util/constants.hpp @@ -60,6 +60,7 @@ constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5; constexpr const char* ASSET_PROTOCOL = "asset://"; constexpr const char* FILE_PROTOCOL = "file://"; constexpr const char* MBTILES_PROTOCOL = "mbtiles://"; +constexpr const char* PMTILES_PROTOCOL = "pmtiles://"; constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15; diff --git a/platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp b/platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp index 92420ca7d8e..4bd80f1e78f 100644 --- a/platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp +++ b/platform/android/MapLibreAndroid/src/cpp/http_file_source.cpp @@ -91,9 +91,15 @@ void RegisterNativeHTTPRequest(jni::JNIEnv& env) { HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource::Callback callback_) : resource(resource_), callback(callback_) { + std::string dataRangeStr; std::string etagStr; std::string modifiedStr; + if (resource.dataRange) { + dataRangeStr = std::string("bytes=") + std::to_string(resource.dataRange->first) + std::string("-") + + std::to_string(resource.dataRange->second); + } + if (resource.priorEtag) { etagStr = *resource.priorEtag; } else if (resource.priorModified) { @@ -104,13 +110,14 @@ HTTPRequest::HTTPRequest(jni::JNIEnv& env, const Resource& resource_, FileSource static auto& javaClass = jni::Class::Singleton(env); static auto constructor = - javaClass.GetConstructor(env); + javaClass.GetConstructor(env); javaRequest = jni::NewGlobal(env, javaClass.New(env, constructor, reinterpret_cast(this), jni::Make(env, resource.url), + jni::Make(env, dataRangeStr), jni::Make(env, etagStr), jni::Make(env, modifiedStr), (jboolean)(resource_.usage == Resource::Usage::Offline))); diff --git a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java index ffe8e21a8a2..05a889192e1 100644 --- a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java +++ b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/HttpRequest.java @@ -18,12 +18,13 @@ public interface HttpRequest { * @param httpRequest callback to be invoked when we receive a response * @param nativePtr the pointer associated to the request * @param resourceUrl the resource url to download + * @param dataRange http header, used to indicate the part of a resource that the server should return * @param etag http header, identifier for a specific version of a resource * @param modified http header, used to determine if a resource hasn't been modified since * @param offlineUsage flag to indicate a resource will be used for offline, appends offline=true as a query parameter */ void executeRequest(HttpResponder httpRequest, long nativePtr, String resourceUrl, - String etag, String modified, boolean offlineUsage); + String dataRange, String etag, String modified, boolean offlineUsage); /** * Cancels the request. diff --git a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java index e41429cd3f5..0c4c43aab68 100644 --- a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java +++ b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/http/NativeHttpRequest.java @@ -19,7 +19,8 @@ public class NativeHttpRequest implements HttpResponder { private long nativePtr; @Keep - private NativeHttpRequest(long nativePtr, String resourceUrl, String etag, String modified, boolean offlineUsage) { + private NativeHttpRequest(long nativePtr, String resourceUrl, String dataRange, String etag, String modified, + boolean offlineUsage) { this.nativePtr = nativePtr; if (resourceUrl.startsWith("local://")) { @@ -27,7 +28,7 @@ private NativeHttpRequest(long nativePtr, String resourceUrl, String etag, Strin executeLocalRequest(resourceUrl); return; } - httpRequest.executeRequest(this, nativePtr, resourceUrl, etag, modified, offlineUsage); + httpRequest.executeRequest(this, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage); } public void cancel() { diff --git a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java index 275e363cd21..7b5ca63c76a 100644 --- a/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java +++ b/platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/module/http/HttpRequestImpl.java @@ -58,7 +58,8 @@ public class HttpRequestImpl implements HttpRequest { @Override public void executeRequest(HttpResponder httpRequest, long nativePtr, @NonNull String resourceUrl, - @NonNull String etag, @NonNull String modified, boolean offlineUsage) { + @NonNull String dataRange, @NonNull String etag, @NonNull String modified, + boolean offlineUsage) { OkHttpCallback callback = new OkHttpCallback(httpRequest); try { HttpUrl httpUrl = HttpUrl.parse(resourceUrl); @@ -74,6 +75,11 @@ public void executeRequest(HttpResponder httpRequest, long nativePtr, @NonNull S .url(resourceUrl) .tag(resourceUrl.toLowerCase(MapLibreConstants.MAPLIBRE_LOCALE)) .addHeader("User-Agent", userAgentString); + + if (dataRange.length() > 0) { + builder.addHeader("Range", dataRange); + } + if (etag.length() > 0) { builder.addHeader("If-None-Match", etag); } else if (modified.length() > 0) { diff --git a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt index a9d2840696f..e49f567f411 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt +++ b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ExampleHttpRequestImpl.kt @@ -11,11 +11,11 @@ import org.maplibre.android.module.http.HttpRequestImpl */ class ExampleHttpRequestImpl : HttpRequest { override fun executeRequest(httpRequest: HttpResponder, nativePtr: Long, resourceUrl: String, - etag: String, modified: String, offlineUsage: Boolean) + dataRange: String, etag: String, modified: String, offlineUsage: Boolean) { // Load all json documents and any pbf ending with a 0. if (resourceUrl.endsWith(".json") || resourceUrl.endsWith("0.pbf")) { - impl.executeRequest(httpRequest, nativePtr, resourceUrl, etag, modified, offlineUsage) + impl.executeRequest(httpRequest, nativePtr, resourceUrl, dataRange, etag, modified, offlineUsage) } else { // All other requests get an instant 404! httpRequest.onResponse( diff --git a/platform/android/android.cmake b/platform/android/android.cmake index dd01b8f11e1..fbddbb2a4f0 100644 --- a/platform/android/android.cmake +++ b/platform/android/android.cmake @@ -61,6 +61,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp> ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp diff --git a/platform/darwin/src/http_file_source.mm b/platform/darwin/src/http_file_source.mm index 007a0b902d4..8c2fe83c4c3 100644 --- a/platform/darwin/src/http_file_source.mm +++ b/platform/darwin/src/http_file_source.mm @@ -264,6 +264,13 @@ BOOL isValidMapboxEndpoint(NSURL *url) { [req addValue:@(util::rfc1123(*resource.priorModified).c_str()) forHTTPHeaderField:@"If-Modified-Since"]; } + + if (resource.dataRange) { + NSString *rangeHeader = [NSString stringWithFormat:@"bytes=%lld-%lld", + static_cast(resource.dataRange->first), + static_cast(resource.dataRange->second)]; + [req setValue:rangeHeader forHTTPHeaderField:@"Range"]; + } [req addValue:impl->userAgent forHTTPHeaderField:@"User-Agent"]; @@ -360,7 +367,7 @@ BOOL isValidMapboxEndpoint(NSURL *url) { response.etag = std::string([etag UTF8String]); } - if (responseCode == 200) { + if (responseCode == 200 || responseCode == 206) { response.data = std::make_shared((const char *)[data bytes], [data length]); } else if (responseCode == 204 || (responseCode == 404 && isTile)) { response.noContent = true; diff --git a/platform/default/BUILD.bazel b/platform/default/BUILD.bazel index bb1dfd3576f..95f94e32ce6 100644 --- a/platform/default/BUILD.bazel +++ b/platform/default/BUILD.bazel @@ -57,6 +57,7 @@ cc_library( "src/mbgl/storage/offline_database.cpp", "src/mbgl/storage/offline_download.cpp", "src/mbgl/storage/online_file_source.cpp", + "src/mbgl/storage/pmtiles_file_source.cpp", "src/mbgl/storage/sqlite3.cpp", "src/mbgl/text/bidi.cpp", "src/mbgl/util/compression.cpp", diff --git a/platform/default/include/mbgl/storage/local_file_request.hpp b/platform/default/include/mbgl/storage/local_file_request.hpp index 15354d36a4e..0d342695422 100644 --- a/platform/default/include/mbgl/storage/local_file_request.hpp +++ b/platform/default/include/mbgl/storage/local_file_request.hpp @@ -8,6 +8,8 @@ template class ActorRef; class FileSourceRequest; -void requestLocalFile(const std::string&, const ActorRef&); +void requestLocalFile(const std::string& path, + const ActorRef& req, + const std::optional>& dataRange = std::nullopt); } // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source_manager.cpp b/platform/default/src/mbgl/storage/file_source_manager.cpp index d3068404407..b78a94e6dc2 100644 --- a/platform/default/src/mbgl/storage/file_source_manager.cpp +++ b/platform/default/src/mbgl/storage/file_source_manager.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace mbgl { @@ -37,6 +38,11 @@ class DefaultFileSourceManagerImpl final : public FileSourceManager { return std::make_unique(resourceOptions, clientOptions); }); + registerFileSourceFactory(FileSourceType::Pmtiles, + [](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) { + return std::make_unique(resourceOptions, clientOptions); + }); + registerFileSourceFactory(FileSourceType::Network, [](const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) { return std::make_unique(resourceOptions, clientOptions); diff --git a/platform/default/src/mbgl/storage/http_file_source.cpp b/platform/default/src/mbgl/storage/http_file_source.cpp index 0b0d191c2d5..6bb4ff9220a 100644 --- a/platform/default/src/mbgl/storage/http_file_source.cpp +++ b/platform/default/src/mbgl/storage/http_file_source.cpp @@ -269,6 +269,12 @@ HTTPRequest::HTTPRequest(HTTPFileSource::Impl *context_, Resource resource_, Fil resource(std::move(resource_)), callback(std::move(callback_)), handle(context->getHandle()) { + if (resource.dataRange) { + const std::string header = std::string("Range: bytes=") + std::to_string(resource.dataRange->first) + + std::string("-") + std::to_string(resource.dataRange->second); + headers = curl_slist_append(headers, header.c_str()); + } + // If there's already a response, set the correct etags/modified headers to // make sure we are getting a 304 response if possible. This avoids // redownloading unchanged data. @@ -417,7 +423,7 @@ void HTTPRequest::handleResult(CURLcode code) { long responseCode = 0; curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode); - if (responseCode == 200) { + if (responseCode == 200 || responseCode == 206) { if (data) { response->data = std::move(data); } else { diff --git a/platform/default/src/mbgl/storage/local_file_request.cpp b/platform/default/src/mbgl/storage/local_file_request.cpp index 6204f552ae4..8c6cf43d808 100644 --- a/platform/default/src/mbgl/storage/local_file_request.cpp +++ b/platform/default/src/mbgl/storage/local_file_request.cpp @@ -11,7 +11,9 @@ namespace mbgl { -void requestLocalFile(const std::string& path, const ActorRef& req) { +void requestLocalFile(const std::string& path, + const ActorRef& req, + const std::optional>& dataRange) { Response response; struct stat buf; int result = stat(path.c_str(), &buf); @@ -21,7 +23,7 @@ void requestLocalFile(const std::string& path, const ActorRef } else if (result == -1 && errno == ENOENT) { response.error = std::make_unique(Response::Error::Reason::NotFound); } else { - auto data = util::readFile(path); + auto data = util::readFile(path, dataRange); if (!data) { response.error = std::make_unique(Response::Error::Reason::Other, std::string("Cannot read file ") + path); diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp index c42ad520c1a..3fef8ee7862 100644 --- a/platform/default/src/mbgl/storage/local_file_source.cpp +++ b/platform/default/src/mbgl/storage/local_file_source.cpp @@ -25,8 +25,8 @@ class LocalFileSource::Impl { : resourceOptions(resourceOptions_.clone()), clientOptions(clientOptions_.clone()) {} - void request(const std::string& url, const ActorRef& req) { - if (!acceptsURL(url)) { + void request(const Resource& resource, const ActorRef& req) { + if (!acceptsURL(resource.url)) { Response response; response.error = std::make_unique(Response::Error::Reason::Other, "Invalid file URL"); req.invoke(&FileSourceRequest::setResponse, response); @@ -34,8 +34,9 @@ class LocalFileSource::Impl { } // Cut off the protocol and prefix with path. - const auto path = mbgl::util::percentDecode(url.substr(std::char_traits::length(util::FILE_PROTOCOL))); - requestLocalFile(path, req); + const auto path = mbgl::util::percentDecode( + resource.url.substr(std::char_traits::length(util::FILE_PROTOCOL))); + requestLocalFile(path, req, resource.dataRange); } void setResourceOptions(ResourceOptions options) { @@ -77,7 +78,7 @@ LocalFileSource::~LocalFileSource() = default; std::unique_ptr LocalFileSource::request(const Resource& resource, Callback callback) { auto req = std::make_unique(std::move(callback)); - impl->actor().invoke(&Impl::request, resource.url, req->actor()); + impl->actor().invoke(&Impl::request, resource, req->actor()); return req; } diff --git a/platform/default/src/mbgl/storage/main_resource_loader.cpp b/platform/default/src/mbgl/storage/main_resource_loader.cpp index 66699128fe9..2f523f1e8d2 100644 --- a/platform/default/src/mbgl/storage/main_resource_loader.cpp +++ b/platform/default/src/mbgl/storage/main_resource_loader.cpp @@ -21,12 +21,14 @@ class MainResourceLoaderThread { std::shared_ptr databaseFileSource_, std::shared_ptr localFileSource_, std::shared_ptr onlineFileSource_, - std::shared_ptr mbtilesFileSource_) + std::shared_ptr mbtilesFileSource_, + std::shared_ptr pmtilesFileSource_) : assetFileSource(std::move(assetFileSource_)), databaseFileSource(std::move(databaseFileSource_)), localFileSource(std::move(localFileSource_)), onlineFileSource(std::move(onlineFileSource_)), - mbtilesFileSource(std::move(mbtilesFileSource_)) {} + mbtilesFileSource(std::move(mbtilesFileSource_)), + pmtilesFileSource(std::move(pmtilesFileSource_)) {} void request(AsyncRequest* req, const Resource& resource, const ActorRef& ref) { auto callback = [ref](const Response& res) { @@ -70,6 +72,9 @@ class MainResourceLoaderThread { } else if (mbtilesFileSource && mbtilesFileSource->canRequest(resource)) { // Local file request tasks[req] = mbtilesFileSource->request(resource, callback); + } else if (pmtilesFileSource && pmtilesFileSource->canRequest(resource)) { + // Local file request + tasks[req] = pmtilesFileSource->request(resource, callback); } else if (localFileSource && localFileSource->canRequest(resource)) { // Local file request tasks[req] = localFileSource->request(resource, callback); @@ -131,6 +136,7 @@ class MainResourceLoaderThread { const std::shared_ptr localFileSource; const std::shared_ptr onlineFileSource; const std::shared_ptr mbtilesFileSource; + const std::shared_ptr pmtilesFileSource; std::map> tasks; }; @@ -142,12 +148,14 @@ class MainResourceLoader::Impl { std::shared_ptr databaseFileSource_, std::shared_ptr localFileSource_, std::shared_ptr onlineFileSource_, - std::shared_ptr mbtilesFileSource_) + std::shared_ptr mbtilesFileSource_, + std::shared_ptr pmtilesFileSource_) : assetFileSource(std::move(assetFileSource_)), databaseFileSource(std::move(databaseFileSource_)), localFileSource(std::move(localFileSource_)), onlineFileSource(std::move(onlineFileSource_)), mbtilesFileSource(std::move(mbtilesFileSource_)), + pmtilesFileSource(std::move(pmtilesFileSource_)), supportsCacheOnlyRequests_(bool(databaseFileSource)), thread(std::make_unique>( util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_WORKER), @@ -156,7 +164,8 @@ class MainResourceLoader::Impl { databaseFileSource, localFileSource, onlineFileSource, - mbtilesFileSource)), + mbtilesFileSource, + pmtilesFileSource)), resourceOptions(resourceOptions_.clone()), clientOptions(clientOptions_.clone()) {} @@ -175,7 +184,8 @@ class MainResourceLoader::Impl { (localFileSource && localFileSource->canRequest(resource)) || (databaseFileSource && databaseFileSource->canRequest(resource)) || (onlineFileSource && onlineFileSource->canRequest(resource)) || - (mbtilesFileSource && mbtilesFileSource->canRequest(resource)); + (mbtilesFileSource && mbtilesFileSource->canRequest(resource)) || + (pmtilesFileSource && pmtilesFileSource->canRequest(resource)); } bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; } @@ -192,6 +202,7 @@ class MainResourceLoader::Impl { localFileSource->setResourceOptions(options.clone()); onlineFileSource->setResourceOptions(options.clone()); mbtilesFileSource->setResourceOptions(options.clone()); + pmtilesFileSource->setResourceOptions(options.clone()); } ResourceOptions getResourceOptions() { @@ -207,6 +218,7 @@ class MainResourceLoader::Impl { localFileSource->setClientOptions(options.clone()); onlineFileSource->setClientOptions(options.clone()); mbtilesFileSource->setClientOptions(options.clone()); + pmtilesFileSource->setClientOptions(options.clone()); } ClientOptions getClientOptions() { @@ -220,6 +232,7 @@ class MainResourceLoader::Impl { const std::shared_ptr localFileSource; const std::shared_ptr onlineFileSource; const std::shared_ptr mbtilesFileSource; + const std::shared_ptr pmtilesFileSource; const bool supportsCacheOnlyRequests_; const std::unique_ptr> thread; mutable std::mutex resourceOptionsMutex; @@ -236,7 +249,8 @@ MainResourceLoader::MainResourceLoader(const ResourceOptions& resourceOptions, c FileSourceManager::get()->getFileSource(FileSourceType::Database, resourceOptions, clientOptions), FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, resourceOptions, clientOptions), FileSourceManager::get()->getFileSource(FileSourceType::Network, resourceOptions, clientOptions), - FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, resourceOptions, clientOptions))) {} + FileSourceManager::get()->getFileSource(FileSourceType::Mbtiles, resourceOptions, clientOptions), + FileSourceManager::get()->getFileSource(FileSourceType::Pmtiles, resourceOptions, clientOptions))) {} MainResourceLoader::~MainResourceLoader() = default; diff --git a/platform/default/src/mbgl/storage/pmtiles_file_source.cpp b/platform/default/src/mbgl/storage/pmtiles_file_source.cpp new file mode 100644 index 00000000000..9089f65ea8f --- /dev/null +++ b/platform/default/src/mbgl/storage/pmtiles_file_source.cpp @@ -0,0 +1,586 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#if defined(__QT__) && (defined(_WIN32) || defined(__EMSCRIPTEN__)) +#include +#else +#include +#endif + +namespace { +// https://github.com/protomaps/PMTiles/blob/main/spec/v3/spec.md#3-header +constexpr int pmtilesHeaderOffset = 0; +constexpr int pmtilesHeaderLength = 127; + +// To avoid allocating lots of memory with PMTiles directory caching, +// set a limit so it doesn't grow unlimited +constexpr int MAX_DIRECTORY_CACHE_ENTRIES = 100; + +bool acceptsURL(const std::string& url) { + return url.starts_with(mbgl::util::PMTILES_PROTOCOL); +} + +std::string extract_url(const std::string& url) { + return url.substr(std::char_traits::length(mbgl::util::PMTILES_PROTOCOL)); +} +} // namespace + +namespace mbgl { +using namespace rapidjson; + +using AsyncCallback = std::function)>; +using AsyncTileCallback = std::function, std::unique_ptr)>; + +class PMTilesFileSource::Impl { +public: + explicit Impl(const ActorRef&, const ResourceOptions& resourceOptions_, const ClientOptions& clientOptions_) + : resourceOptions(resourceOptions_.clone()), + clientOptions(clientOptions_.clone()) {} + + // Generate a tilejson resource from .pmtiles file + void request_tilejson(AsyncRequest* req, const Resource& resource, const ActorRef& ref) { + auto url = extract_url(resource.url); + + getMetadata(url, req, [=, this](std::unique_ptr error) { + Response response; + + if (error) { + response.error = std::move(error); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + response.data = std::make_shared(metadata_cache.at(url)); + ref.invoke(&FileSourceRequest::setResponse, response); + }); + } + + // Load data for specific tile + void request_tile(AsyncRequest* req, const Resource& resource, ActorRef ref) { + auto url = extract_url(resource.url); + + getHeader(url, req, [=, this](std::unique_ptr error) { + if (error) { + Response response; + response.noContent = true; + response.error = std::move(error); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + pmtiles::headerv3 header = header_cache.at(url); + + if (resource.tileData->z < header.min_zoom || resource.tileData->z > header.max_zoom) { + Response response; + response.noContent = true; + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + uint64_t tileID; + + try { + tileID = pmtiles::zxy_to_tileid(static_cast(resource.tileData->z), + static_cast(resource.tileData->x), + static_cast(resource.tileData->y)); + } catch (const std::exception& e) { + Response response; + response.noContent = true; + response.error = std::make_unique(Response::Error::Reason::Other, + std::string("Invalid tile: ") + e.what()); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + getTileAddress( + url, + req, + tileID, + header.root_dir_offset, + header.root_dir_bytes, + 0, + [=, this](std::pair tileAddress, std::unique_ptr tileError) { + if (tileError) { + Response response; + response.noContent = true; + response.error = std::move(tileError); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + if (tileAddress.first == 0 && tileAddress.second == 0) { + Response response; + response.noContent = true; + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + Resource tileResource(Resource::Kind::Source, url); + tileResource.loadingMethod = Resource::LoadingMethod::Network; + tileResource.dataRange = std::make_pair(tileAddress.first, + tileAddress.first + tileAddress.second - 1); + + tasks[req] = getFileSource()->request(tileResource, [=](const Response& tileResponse) { + Response response; + response.noContent = true; + + if (tileResponse.error) { + response.error = std::make_unique( + tileResponse.error->reason, + std::string("Error fetching PMTiles tile: ") + tileResponse.error->message); + ref.invoke(&FileSourceRequest::setResponse, response); + return; + } + + response.data = tileResponse.data; + response.noContent = false; + response.modified = tileResponse.modified; + response.expires = tileResponse.expires; + response.etag = tileResponse.etag; + + if (header.tile_compression == pmtiles::COMPRESSION_GZIP) { + response.data = std::make_shared(util::decompress(*tileResponse.data)); + } + + ref.invoke(&FileSourceRequest::setResponse, response); + return; + }); + }); + }); + } + + void setResourceOptions(ResourceOptions options) { + std::lock_guard lock(resourceOptionsMutex); + resourceOptions = options; + } + + ResourceOptions getResourceOptions() { + std::lock_guard lock(resourceOptionsMutex); + return resourceOptions.clone(); + } + + void setClientOptions(ClientOptions options) { + std::lock_guard lock(clientOptionsMutex); + clientOptions = options; + } + + ClientOptions getClientOptions() { + std::lock_guard lock(clientOptionsMutex); + return clientOptions.clone(); + } + +private: + mutable std::mutex resourceOptionsMutex; + mutable std::mutex clientOptionsMutex; + ResourceOptions resourceOptions; + ClientOptions clientOptions; + + std::shared_ptr fileSource; + std::map header_cache; + std::map metadata_cache; + std::map>> directory_cache; + std::map> directory_cache_control; + std::map> tasks; + + std::shared_ptr getFileSource() { + if (!fileSource) { + fileSource = FileSourceManager::get()->getFileSource( + FileSourceType::ResourceLoader, resourceOptions, clientOptions); + } + + return fileSource; + } + + void getHeader(const std::string& url, AsyncRequest* req, AsyncCallback callback) { + if (header_cache.find(url) != header_cache.end()) { + callback(std::unique_ptr()); + } + + Resource resource(Resource::Kind::Source, url); + resource.loadingMethod = Resource::LoadingMethod::Network; + + resource.dataRange = std::make_pair(pmtilesHeaderOffset, + pmtilesHeaderOffset + pmtilesHeaderLength - 1); + + tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) { + if (response.error) { + std::string message = std::string("Error fetching PMTiles header: ") + response.error->message; + + if (response.error->message.empty() && response.error->reason == Response::Error::Reason::NotFound) { + if (url.starts_with(mbgl::util::FILE_PROTOCOL)) { + message += "path not found: " + + url.substr(std::char_traits::length(mbgl::util::FILE_PROTOCOL)); + } else { + message += "url not found: " + url; + } + } + + callback(std::make_unique(response.error->reason, message)); + + return; + } + + try { + pmtiles::headerv3 header = pmtiles::deserialize_header(response.data->substr(0, 127)); + + if ((header.internal_compression != pmtiles::COMPRESSION_NONE && + header.internal_compression != pmtiles::COMPRESSION_GZIP) || + (header.tile_compression != pmtiles::COMPRESSION_NONE && + header.tile_compression != pmtiles::COMPRESSION_GZIP)) { + throw std::runtime_error("Compression method not supported"); + } + + header_cache.emplace(url, header); + + callback(std::unique_ptr()); + } catch (const std::exception& e) { + callback(std::make_unique(Response::Error::Reason::Other, + std::string("Error parsing PMTiles header: ") + e.what())); + } + }); + } + + void getMetadata(std::string& url, AsyncRequest* req, AsyncCallback callback) { + if (metadata_cache.find(url) != metadata_cache.end()) { + callback(std::unique_ptr()); + } + + getHeader(url, req, [=, this](std::unique_ptr error) { + if (error) { + callback(std::move(error)); + return; + } + + pmtiles::headerv3 header = header_cache.at(url); + + auto parse_callback = [=, this](const std::string& data) { + Document doc; + + auto& allocator = doc.GetAllocator(); + + if (!data.empty()) { + doc.Parse(data); + } + + if (!doc.IsObject()) { + doc.SetObject(); + } + + doc.AddMember("tilejson", "3.0.0", allocator); + + if (!doc.HasMember("scheme")) { + doc.AddMember("scheme", rapidjson::Value().SetString("xyz"), allocator); + } + + if (!doc.HasMember("tiles")) { + doc.AddMember("tiles", rapidjson::Value(), allocator); + } + + if (!doc["tiles"].IsArray()) { + doc["tiles"] = rapidjson::Value().SetArray().PushBack( + rapidjson::Value().SetString(std::string(util::PMTILES_PROTOCOL + url), allocator), allocator); + } + + if (!doc.HasMember("bounds")) { + doc.AddMember("bounds", rapidjson::Value(), allocator); + } + + if (!doc["bounds"].IsArray()) { + doc["bounds"] = rapidjson::Value() + .SetArray() + .PushBack(static_cast(header.min_lon_e7) / 1e7, allocator) + .PushBack(static_cast(header.min_lat_e7) / 1e7, allocator) + .PushBack(static_cast(header.max_lon_e7) / 1e7, allocator) + .PushBack(static_cast(header.max_lat_e7) / 1e7, allocator); + } + + if (!doc.HasMember("center")) { + doc.AddMember("center", rapidjson::Value(), allocator); + } + + if (!doc["center"].IsArray()) { + doc["center"] = rapidjson::Value() + .SetArray() + .PushBack(static_cast(header.center_lon_e7) / 1e7, allocator) + .PushBack(static_cast(header.center_lat_e7) / 1e7, allocator) + .PushBack(header.center_zoom, allocator); + } + + if (!doc.HasMember("minzoom")) { + doc.AddMember("minzoom", rapidjson::Value(), allocator); + } + + auto& minzoom = doc["minzoom"]; + + if (minzoom.IsString()) { + minzoom.SetInt(std::atoi(minzoom.GetString())); + } + + if (!minzoom.IsNumber()) { + minzoom = rapidjson::Value().SetUint(header.min_zoom); + } + + doc["minzoom"] = minzoom; + + if (!doc.HasMember("maxzoom")) { + doc.AddMember("maxzoom", rapidjson::Value(), allocator); + } + + auto& maxzoom = doc["maxzoom"]; + + if (maxzoom.IsString()) { + maxzoom.SetInt(std::atoi(maxzoom.GetString())); + } + + if (!maxzoom.IsNumber()) { + maxzoom = rapidjson::Value().SetUint(header.max_zoom); + } + + doc["maxzoom"] = maxzoom; + + std::string metadata = serialize(doc); + metadata_cache.emplace(url, metadata); + + callback(std::unique_ptr()); + }; + + if (header.json_metadata_bytes > 0) { + Resource resource(Resource::Kind::Source, url); + resource.loadingMethod = Resource::LoadingMethod::Network; + resource.dataRange = std::make_pair(header.json_metadata_offset, + header.json_metadata_offset + header.json_metadata_bytes - 1); + + tasks[req] = getFileSource()->request(resource, [=](const Response& responseMetadata) { + if (responseMetadata.error) { + callback(std::make_unique( + responseMetadata.error->reason, + std::string("Error fetching PMTiles metadata: ") + responseMetadata.error->message)); + + return; + } + + std::string data = *responseMetadata.data; + + if (header.internal_compression == pmtiles::COMPRESSION_GZIP) { + data = util::decompress(data); + } + + parse_callback(data); + }); + + return; + } + + parse_callback(std::string()); + }); + } + + void storeDirectory(const std::string& url, + uint64_t directoryOffset, + uint64_t directoryLength, + const std::string& directoryData) { + if (directory_cache.find(url) == directory_cache.end()) { + directory_cache.emplace(url, std::map>()); + directory_cache_control.emplace(url, std::vector()); + } + + std::string directory_cache_key = url + "|" + std::to_string(directoryOffset) + "|" + + std::to_string(directoryLength); + directory_cache.at(url).emplace(directory_cache_key, pmtiles::deserialize_directory(directoryData)); + directory_cache_control.at(url).emplace_back(directory_cache_key); + + if (directory_cache_control.at(url).size() > MAX_DIRECTORY_CACHE_ENTRIES) { + directory_cache.at(url).erase(directory_cache_control.at(url).front()); + directory_cache_control.at(url).erase(directory_cache_control.at(url).begin()); + } + } + + void getDirectory(const std::string& url, + AsyncRequest* req, + uint64_t directoryOffset, + uint32_t directoryLength, + AsyncCallback callback) { + std::string directory_cache_key = url + "|" + std::to_string(directoryOffset) + "|" + + std::to_string(directoryLength); + + if (directory_cache.find(url) != directory_cache.end() && + directory_cache.at(url).find(directory_cache_key) != directory_cache.at(url).end()) { + if (directory_cache_control.at(url).back() != directory_cache_key) { + directory_cache_control.at(url).emplace_back(directory_cache_key); + + for (auto it = directory_cache_control.at(url).begin(); it != directory_cache_control.at(url).end(); + ++it) { + if (*it == directory_cache_key) { + directory_cache_control.at(url).erase(it); + break; + } + } + } + + callback(std::unique_ptr()); + return; + } + + getHeader(url, req, [=, this](std::unique_ptr error) { + if (error) { + callback(std::move(error)); + return; + } + + pmtiles::headerv3 header = header_cache.at(url); + + Resource resource(Resource::Kind::Source, url); + resource.loadingMethod = Resource::LoadingMethod::Network; + resource.dataRange = std::make_pair(directoryOffset, directoryOffset + directoryLength - 1); + + tasks[req] = getFileSource()->request(resource, [=, this](const Response& response) { + if (response.error) { + callback(std::make_unique( + response.error->reason, + std::string("Error fetching PMTiles directory: ") + response.error->message)); + + return; + } + + try { + std::string directoryData = *response.data; + + if (header.internal_compression == pmtiles::COMPRESSION_GZIP) { + directoryData = util::decompress(directoryData); + } + + storeDirectory(url, directoryOffset, directoryLength, directoryData); + + callback(std::unique_ptr()); + } catch (const std::exception& e) { + callback(std::make_unique( + Response::Error::Reason::Other, + std::string(std::string("Error parsing PMTiles directory: ") + e.what()))); + } + }); + }); + } + + void getTileAddress(const std::string& url, + AsyncRequest* req, + uint64_t tileID, + uint64_t directoryOffset, + uint32_t directoryLength, + uint32_t directoryDepth, + AsyncTileCallback callback) { + if (directoryDepth > 3) { + callback(std::make_pair(0, 0), + std::make_unique( + Response::Error::Reason::Other, + std::string("Error fetching PMTiles tile address: Maximum directory depth exceeded"))); + + return; + } + + getDirectory(url, req, directoryOffset, directoryLength, [=, this](std::unique_ptr error) { + if (error) { + callback(std::make_pair(0, 0), std::move(error)); + return; + } + + pmtiles::headerv3 header = header_cache.at(url); + std::vector directory = directory_cache.at(url).at( + url + "|" + std::to_string(directoryOffset) + "|" + std::to_string(directoryLength)); + + pmtiles::entryv3 entry = pmtiles::find_tile(directory, tileID); + + if (entry.length > 0) { + if (entry.run_length > 0) { + callback(std::make_pair(header.tile_data_offset + entry.offset, entry.length), {}); + return; + } + + getTileAddress(url, + req, + tileID, + header.leaf_dirs_offset + entry.offset, + entry.length, + directoryDepth + 1, + std::move(callback)); + return; + } + + callback(std::make_pair(0, 0), {}); + }); + } + + std::string serialize(Document& doc) { + StringBuffer buffer; + Writer writer(buffer); + doc.Accept(writer); + + return std::string(buffer.GetString(), buffer.GetSize()); + } +}; + +PMTilesFileSource::PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) + : thread(std::make_unique>( + util::makeThreadPrioritySetter(platform::EXPERIMENTAL_THREAD_PRIORITY_FILE), + "PMTilesFileSource", + resourceOptions.clone(), + clientOptions.clone())) {} + +std::unique_ptr PMTilesFileSource::request(const Resource& resource, FileSource::Callback callback) { + auto req = std::make_unique(std::move(callback)); + + // assume if there is a tile request, that the pmtiles file has been validated + if (resource.kind == Resource::Tile) { + thread->actor().invoke(&Impl::request_tile, req.get(), resource, req->actor()); + return req; + } + + // return TileJSON + thread->actor().invoke(&Impl::request_tilejson, req.get(), resource, req->actor()); + return req; +} + +bool PMTilesFileSource::canRequest(const Resource& resource) const { + return acceptsURL(resource.url); +} + +PMTilesFileSource::~PMTilesFileSource() = default; + +void PMTilesFileSource::setResourceOptions(ResourceOptions options) { + thread->actor().invoke(&Impl::setResourceOptions, options.clone()); +} + +ResourceOptions PMTilesFileSource::getResourceOptions() { + return thread->actor().ask(&Impl::getResourceOptions).get(); +} + +void PMTilesFileSource::setClientOptions(ClientOptions options) { + thread->actor().invoke(&Impl::setClientOptions, options.clone()); +} + +ClientOptions PMTilesFileSource::getClientOptions() { + return thread->actor().ask(&Impl::getClientOptions).get(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/pmtiles_file_source_stub.cpp b/platform/default/src/mbgl/storage/pmtiles_file_source_stub.cpp new file mode 100644 index 00000000000..48ff404756a --- /dev/null +++ b/platform/default/src/mbgl/storage/pmtiles_file_source_stub.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include + +namespace mbgl { + +class PMTilesFileSource::Impl { +public: + Impl() = default; + ~Impl() = default; +}; + +PMTilesFileSource::PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions) {} + +std::unique_ptr PMTilesFileSource::request(const Resource& resource, FileSource::Callback callback) { + return nullptr; +} + +bool PMTilesFileSource::canRequest(const Resource& resource) const { + return false; +} + +PMTilesFileSource::~PMTilesFileSource() = default; + +void PMTilesFileSource::setResourceOptions(ResourceOptions options) {} + +ResourceOptions PMTilesFileSource::getResourceOptions() { + return {}; +} + +void PMTilesFileSource::setClientOptions(ClientOptions options) {} + +ClientOptions PMTilesFileSource::getClientOptions() { + return {}; +} + +} // namespace mbgl diff --git a/platform/linux/linux.cmake b/platform/linux/linux.cmake index e1374d067a5..4289fd57a63 100644 --- a/platform/linux/linux.cmake +++ b/platform/linux/linux.cmake @@ -50,6 +50,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp> ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp diff --git a/platform/macos/macos.cmake b/platform/macos/macos.cmake index 4ca02db212c..48953eb5b74 100644 --- a/platform/macos/macos.cmake +++ b/platform/macos/macos.cmake @@ -92,6 +92,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp> ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp diff --git a/platform/qt/qt.cmake b/platform/qt/qt.cmake index 0f2bf061b35..04920ca8b38 100644 --- a/platform/qt/qt.cmake +++ b/platform/qt/qt.cmake @@ -84,6 +84,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp> ${PROJECT_SOURCE_DIR}/platform/$,default/src/mbgl/storage/sqlite3.cpp,qt/src/mbgl/sqlite3.cpp> ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/compression.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/util/filesystem.cpp diff --git a/platform/qt/src/mbgl/http_request.cpp b/platform/qt/src/mbgl/http_request.cpp index 0fb1993146a..973760a419a 100644 --- a/platform/qt/src/mbgl/http_request.cpp +++ b/platform/qt/src/mbgl/http_request.cpp @@ -49,6 +49,12 @@ QNetworkRequest HTTPRequest::networkRequest() const { req.setRawHeader("User-Agent", agent); #endif + if (m_resource.dataRange) { + std::string range = std::string("bytes=") + std::to_string(m_resource.dataRange->first) + std::string("-") + + std::to_string(m_resource.dataRange->second); + req.setRawHeader("Range", QByteArray(range.data(), static_cast(range.size()))); + } + if (m_resource.priorEtag) { const auto etag = m_resource.priorEtag; req.setRawHeader("If-None-Match", QByteArray(etag->data(), static_cast(etag->size()))); @@ -112,7 +118,8 @@ void HTTPRequest::handleNetworkReply(QNetworkReply* reply, const QByteArray& dat int responseCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); switch (responseCode) { - case 200: { + case 200: + case 206: { if (data.isEmpty()) { response.data = std::make_shared(); } else { diff --git a/platform/windows/windows.cmake b/platform/windows/windows.cmake index 8665a6f0be0..8e5a6493eb4 100644 --- a/platform/windows/windows.cmake +++ b/platform/windows/windows.cmake @@ -51,6 +51,7 @@ target_sources( ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_database.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/offline_download.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/online_file_source.cpp + ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/$,pmtiles_file_source.cpp,pmtiles_file_source_stub.cpp> ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/storage/sqlite3.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/bidi.cpp ${PROJECT_SOURCE_DIR}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp diff --git a/src/mbgl/storage/pmtiles_file_source.hpp b/src/mbgl/storage/pmtiles_file_source.hpp new file mode 100644 index 00000000000..47d34139efe --- /dev/null +++ b/src/mbgl/storage/pmtiles_file_source.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +namespace mbgl { +// File source for supporting .pmtiles maps +class PMTilesFileSource : public FileSource { +public: + PMTilesFileSource(const ResourceOptions& resourceOptions, const ClientOptions& clientOptions); + ~PMTilesFileSource() override; + + std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override; + + void setResourceOptions(ResourceOptions) override; + ResourceOptions getResourceOptions() override; + + void setClientOptions(ClientOptions) override; + ClientOptions getClientOptions() override; + +private: + class Impl; + std::unique_ptr> thread; // impl +}; + +} // namespace mbgl diff --git a/src/mbgl/util/io.cpp b/src/mbgl/util/io.cpp index e66389e3319..4603c020935 100644 --- a/src/mbgl/util/io.cpp +++ b/src/mbgl/util/io.cpp @@ -46,14 +46,23 @@ std::string read_file(const std::string &filename) { } } -std::optional readFile(const std::string &filename) { +std::optional readFile(const std::string &filename, + const std::optional> &dataRange) { MLN_TRACE_FUNC(); std::ifstream file(filename, std::ios::binary); if (file.good()) { - std::stringstream data; - data << file.rdbuf(); - return data.str(); + if (dataRange) { + size_t size = static_cast(dataRange->second - dataRange->first + 1); + std::string data(size, '\0'); + file.seekg(static_cast(dataRange->first)); + file.read(&data[0], static_cast(size)); + return data; + } else { + std::stringstream data; + data << file.rdbuf(); + return data.str(); + } } return {}; } diff --git a/src/mbgl/util/io.hpp b/src/mbgl/util/io.hpp index 7e57b312798..bc08a7b925a 100644 --- a/src/mbgl/util/io.hpp +++ b/src/mbgl/util/io.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -15,7 +16,8 @@ struct IOException : std::runtime_error { void write_file(const std::string& filename, const std::string& data); std::string read_file(const std::string& filename); -std::optional readFile(const std::string& filename); +std::optional readFile(const std::string& filename, + const std::optional>& dataRange = std::nullopt); void deleteFile(const std::string& filename); void copyFile(const std::string& destination, const std::string& source); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 816037eea13..2955aa3a51b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,6 +41,7 @@ add_library( ${PROJECT_SOURCE_DIR}/test/storage/offline_database.test.cpp ${PROJECT_SOURCE_DIR}/test/storage/offline_download.test.cpp ${PROJECT_SOURCE_DIR}/test/storage/online_file_source.test.cpp + ${PROJECT_SOURCE_DIR}/test/storage/pmtiles_file_source.test.cpp ${PROJECT_SOURCE_DIR}/test/storage/resource.test.cpp ${PROJECT_SOURCE_DIR}/test/storage/sqlite.test.cpp ${PROJECT_SOURCE_DIR}/test/style/conversion/conversion_impl.test.cpp diff --git a/test/fixtures/storage/pmtiles/geography-class-png.pmtiles b/test/fixtures/storage/pmtiles/geography-class-png.pmtiles new file mode 100644 index 00000000000..aec0ca2a629 Binary files /dev/null and b/test/fixtures/storage/pmtiles/geography-class-png.pmtiles differ diff --git a/test/src/mbgl/test/http_server.cpp b/test/src/mbgl/test/http_server.cpp index 92c651b3751..25bb6cee3a1 100644 --- a/test/src/mbgl/test/http_server.cpp +++ b/test/src/mbgl/test/http_server.cpp @@ -9,6 +9,8 @@ #include #include +#include +#include using namespace std::literals::string_literals; @@ -19,6 +21,7 @@ void runServer(std::unique_ptr& server) { using namespace httplib; server->Get("/test", [](const Request& req, Response& res) { + std::string content = "Hello World!"; if (req.has_param("modified")) { std::string str = util::rfc1123(util::parseTimestamp(std::stoi(req.get_param_value("modified")))); res.set_header("Last-Modified", str); @@ -33,7 +36,15 @@ void runServer(std::unique_ptr& server) { if (req.has_param("cachecontrol")) { res.set_header("Cache-Control", "max-age=" + req.get_param_value("cachecontrol")); } - res.set_content("Hello World!", "text/plain"); + if (req.has_param("range")) { + std::string str = req.get_param_value("range"); + str = str.substr(std::char_traits::length("bytes=")); + uint64_t start = std::strtoull(str.substr(0, str.find("-")).c_str(), nullptr, 10); + uint64_t end = std::strtoull(str.substr(str.find("-") + 1).c_str(), nullptr, 10); + content = content.substr(start, end - start + 1); + res.status = 206; + } + res.set_content(content, "text/plain"); }); server->Get("/stale", [](const Request&, Response&) { diff --git a/test/storage/http_file_source.test.cpp b/test/storage/http_file_source.test.cpp index 68c5e890bdb..e17a9489a31 100644 --- a/test/storage/http_file_source.test.cpp +++ b/test/storage/http_file_source.test.cpp @@ -37,6 +37,27 @@ TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP200)) { loop.run(); } +TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP206)) { + util::RunLoop loop; + HTTPFileSource fs(ResourceOptions::Default(), ClientOptions()); + + Resource resource(Resource::Unknown, "http://127.0.0.1:3000/test"); + resource.dataRange = std::make_pair(3, 8); + + auto req = fs.request(resource, [&](Response res) { + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("lo Wor", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + + loop.run(); +} + TEST(HTTPFileSource, TEST_REQUIRES_SERVER(HTTP404)) { util::RunLoop loop; HTTPFileSource fs(ResourceOptions::Default(), ClientOptions()); diff --git a/test/storage/local_file_source.test.cpp b/test/storage/local_file_source.test.cpp index c845d5bf8be..e0e6c211b4b 100644 --- a/test/storage/local_file_source.test.cpp +++ b/test/storage/local_file_source.test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #if defined(WIN32) @@ -25,7 +26,7 @@ std::string toAbsoluteURL(const std::string& fileName) { #else char* cwd = getcwd(buff, PATH_MAX + 1); #endif - std::string url = {"file://" + std::string(cwd) + "/test/fixtures/storage/assets/" + fileName}; + std::string url = {mbgl::util::FILE_PROTOCOL + std::string(cwd) + "/test/fixtures/storage/assets/" + fileName}; assert(url.size() <= PATH_MAX); return url; } @@ -76,6 +77,25 @@ TEST(LocalFileSource, NonEmptyFile) { loop.run(); } +TEST(LocalFileSource, PartialFile) { + util::RunLoop loop; + + LocalFileSource fs(ResourceOptions::Default(), ClientOptions()); + + Resource resource(Resource::Unknown, toAbsoluteURL("nonempty")); + resource.dataRange = std::make_pair(4, 12); + + std::unique_ptr req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("ent is he", *res.data); + loop.stop(); + }); + + loop.run(); +} + TEST(LocalFileSource, NonExistentFile) { util::RunLoop loop; diff --git a/test/storage/pmtiles_file_source.test.cpp b/test/storage/pmtiles_file_source.test.cpp new file mode 100644 index 00000000000..abbe34fb0c2 --- /dev/null +++ b/test/storage/pmtiles_file_source.test.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace { + +std::string toAbsoluteURL(const std::string &fileName) { + auto path = std::filesystem::current_path() / "test/fixtures/storage/pmtiles" / fileName; + return std::string(mbgl::util::PMTILES_PROTOCOL) + std::string(mbgl::util::FILE_PROTOCOL) + path.string(); +} + +} // namespace + +using namespace mbgl; + +TEST(PMTilesFileSource, AcceptsURL) { + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + EXPECT_TRUE(pmtiles.canRequest(Resource::style("pmtiles:///test"))); + EXPECT_FALSE(pmtiles.canRequest(Resource::style("pmtile://test"))); + EXPECT_FALSE(pmtiles.canRequest(Resource::style("pmtiles:"))); + EXPECT_FALSE(pmtiles.canRequest(Resource::style(""))); +} + +// Nonexistent pmtiles file raises error +TEST(PMTilesFileSource, NonExistentFile) { + util::RunLoop loop; + + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + + std::unique_ptr req = pmtiles.request( + {Resource::Unknown, toAbsoluteURL("does_not_exist")}, [&](Response res) { + req.reset(); + ASSERT_NE(nullptr, res.error); + EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); + EXPECT_NE((res.error->message).find("path not found"), std::string::npos); + ASSERT_FALSE(res.data.get()); + loop.stop(); + }); + + loop.run(); +} + +// Existing pmtiles file default request returns TileJSON +TEST(PMTilesFileSource, TileJSON) { + util::RunLoop loop; + + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + + std::unique_ptr req = pmtiles.request( + {Resource::Unknown, toAbsoluteURL("geography-class-png.pmtiles")}, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + // basic test that TileJSON included a tile URL + EXPECT_NE((*res.data).find("geography-class-png.pmtiles"), std::string::npos); + loop.stop(); + }); + + loop.run(); +} + +// Existing tiles return tile data +TEST(PMTilesFileSource, Tile) { + util::RunLoop loop; + + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + + std::unique_ptr req = pmtiles.request( + Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ), + [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + ASSERT_EQ(res.noContent, false); + loop.stop(); + }); + + loop.run(); +} + +// Nonexistent tiles do not raise errors, they simply return no content +TEST(PMTilesFileSource, NonExistentTile) { + util::RunLoop loop; + + PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions()); + + std::unique_ptr req = pmtiles.request( + Resource::tile(toAbsoluteURL("geography-class-png.pmtiles"), 1.0, 0, 0, 4, Tileset::Scheme::XYZ), + [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_FALSE(res.data.get()); + ASSERT_EQ(res.noContent, true); + loop.stop(); + }); + + loop.run(); +} diff --git a/test/storage/server.js b/test/storage/server.js index a7de0944d4a..04ccc430bfc 100755 --- a/test/storage/server.js +++ b/test/storage/server.js @@ -15,6 +15,7 @@ var app = express(); app.disable('etag'); app.get('/test', function (req, res) { + var content = 'Hello World!'; if (req.query.modified) { res.setHeader('Last-Modified', (new Date(req.query.modified * 1000)).toUTCString()); } @@ -27,7 +28,12 @@ app.get('/test', function (req, res) { if (req.query.cachecontrol) { res.setHeader('Cache-Control', req.query.cachecontrol); } - res.send('Hello World!'); + if (req.range()) { + const [ range ] = req.range(); + content = content.substring(range.start, range.end + 1); + res.status(206); + } + res.send(content); }); app.get('/stale/*', function() { diff --git a/vendor/BUILD.bazel b/vendor/BUILD.bazel index 66815d83b44..78fbf69c6c8 100644 --- a/vendor/BUILD.bazel +++ b/vendor/BUILD.bazel @@ -154,6 +154,15 @@ cc_library( visibility = ["//visibility:public"], ) +# vendor/pmtiles-files.json +cc_library( + name = "pmtiles", + hdrs = ["PMTiles/cpp/pmtiles.hpp"], + copts = CPP_FLAGS, + includes = ["PMTiles/cpp"], + visibility = ["//visibility:public"], +) + # vendor/polylabel-files.json cc_library( name = "polylabel", diff --git a/vendor/PMTiles b/vendor/PMTiles new file mode 160000 index 00000000000..60f3331ca9d --- /dev/null +++ b/vendor/PMTiles @@ -0,0 +1 @@ +Subproject commit 60f3331ca9d5cfa4fcdf1e076a3d3059aebb17bb diff --git a/vendor/pmtiles.cmake b/vendor/pmtiles.cmake new file mode 100644 index 00000000000..cb31addee70 --- /dev/null +++ b/vendor/pmtiles.cmake @@ -0,0 +1,21 @@ +if(TARGET mbgl-vendor-pmtiles) + return() +endif() + +add_library( + mbgl-vendor-pmtiles INTERFACE +) + +target_include_directories( + mbgl-vendor-pmtiles SYSTEM + INTERFACE ${CMAKE_CURRENT_LIST_DIR}/PMTiles/cpp +) + +set_target_properties( + mbgl-vendor-pmtiles + PROPERTIES + INTERFACE_MAPLIBRE_NAME "pmtiles" + INTERFACE_MAPLIBRE_URL "https://github.com/protomaps/PMTiles" + INTERFACE_MAPLIBRE_AUTHOR "Protomaps LLC" + INTERFACE_MAPLIBRE_LICENSE ${CMAKE_CURRENT_LIST_DIR}/PMTiles/LICENSE +)