From 002f478231ad9a0134cf646b5e268453f6e07cf6 Mon Sep 17 00:00:00 2001 From: Jakub <106778416+MarekZegare4@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:54:25 +0200 Subject: [PATCH] feat(ui-lvgl): vector region packs, PBF input, region list One .vpk file per region in /vmap (header, sorted index of tiles and points, then the data), read in place with the index in PSRAM; loose {dz}/{x}/{y}.vt/.vp files still work. osm_vector.py writes packs (--pack/--name), reads OSM PBF extracts for a box (--pbf/--bbox, via pyosmium) and picks label languages (--lang). Map tools > Vector regions lists the packs on the card: show one on the map, delete it. No on-device download: packs are made with the tool. Co-Authored-By: Claude Opus 5.5 --- docs/development/l2-roadmap.md | 14 +- examples/companion_radio/ui-lvgl/MapRegions.h | 120 +++++++++++++ examples/companion_radio/ui-lvgl/MapScreen.h | 1 + examples/companion_radio/ui-lvgl/NavMap.h | 11 +- examples/companion_radio/ui-lvgl/UITask.cpp | 3 + examples/companion_radio/ui-lvgl/UITask.h | 3 + .../ui-lvgl/map/VectorLabels.h | 45 +++-- .../companion_radio/ui-lvgl/map/VectorPacks.h | 167 ++++++++++++++++++ .../ui-lvgl/map/VectorTileProvider.h | 26 ++- tools/maps/osm_vector.py | 115 +++++++++--- tools/maps/pbf.py | 65 +++++++ 11 files changed, 509 insertions(+), 61 deletions(-) create mode 100644 examples/companion_radio/ui-lvgl/MapRegions.h create mode 100644 examples/companion_radio/ui-lvgl/map/VectorPacks.h create mode 100644 tools/maps/pbf.py diff --git a/docs/development/l2-roadmap.md b/docs/development/l2-roadmap.md index 69bb0026..5f830de7 100644 --- a/docs/development/l2-roadmap.md +++ b/docs/development/l2-roadmap.md @@ -267,8 +267,18 @@ Ideas to come back to (2026-09-26): - [x] Contours on the vector map: osm_vector.py --dem (tools/maps/dem.py, standard library: Terrain Tiles from AWS Open Data, SRTM ~25 m, smoothed, marching squares); 100 m lines from z12, 20 m from z15. -- [ ] Vector maps next: a PBF pipeline for regions + download on the - device, styling (anti-aliasing, dark theme). +- [x] Vector regions: a region is one pack file (/vmap/*.vpk, index + + tiles + points, read in place; osm_vector.py --pack); input from + Overpass or Geofabrik PBF extracts (--pbf, pyosmium; several across a + border) for a --bbox; --lang for names. Map tools > Vector regions + lists the packs (show, delete). No download on the device: packs are + made by the user. +- [ ] A Tools page on the solo site (meshcore-solo-site, beside the sim): + vector pack maker in the browser (pick a box on a map, Overpass + + DEM, a JS port of osm_vector.py in a worker, .vpk to save), the GPX + downloader (tools/gpx-downloader) and screenshots (tools/screenshot.py + over Web Serial). +- [ ] Vector map styling: anti-aliasing, dark theme. User list, second batch (2026-09-26): diff --git a/examples/companion_radio/ui-lvgl/MapRegions.h b/examples/companion_radio/ui-lvgl/MapRegions.h new file mode 100644 index 00000000..04100007 --- /dev/null +++ b/examples/companion_radio/ui-lvgl/MapRegions.h @@ -0,0 +1,120 @@ +#pragma once +// Map tools > Vector regions: the region packs on the card (/vmap/*.vpk, +// map/VectorPacks.h, made with tools/maps/osm_vector.py --pack): show one on +// the map, delete it. +// +// Single-TU fragment: included by ui-lvgl/UITask.cpp after MapAreas.h. + +namespace regions { + +static int s_sel = -1; // the pack a popup is about +static uint32_t s_del_armed_ms = 0; +static lv_obj_t* s_del_lbl = nullptr; + +enum : uint8_t { A_SHOW, A_DELETE }; + +static void onDeleted(lv_event_t* e) { if (lv_event_get_target(e) == s_del_lbl) s_del_lbl = nullptr; } +static void onPack(lv_event_t* e) { s_ui->mapRegionPopup((int)(intptr_t)lv_event_get_user_data(e)); } +static void onAct(lv_event_t* e) { s_ui->mapRegionAction((uint8_t)(uintptr_t)lv_event_get_user_data(e)); } + +static lv_obj_t* actButton(lv_obj_t* row, const char* text, uint8_t act, bool accent) { + lv_obj_t* b = lv_button_create(row); + lv_obj_set_height(b, 36); + lv_obj_set_flex_grow(b, 1); + lv_obj_set_style_pad_hor(b, 4, 0); + lv_obj_set_style_radius(b, theme::RADIUS, 0); + lv_obj_set_style_shadow_width(b, 0, 0); + lv_obj_set_style_bg_color(b, lv_color_hex(accent ? theme::ACCENT_DIM : theme::SURFACE), 0); + lv_obj_set_style_bg_color(b, lv_color_hex(theme::SURFACE_2), LV_STATE_PRESSED); + lv_obj_add_event_cb(b, onAct, LV_EVENT_CLICKED, (void*)(uintptr_t)act); + lv_obj_t* l = label(b, text, THEME_FONT_SMALL, theme::TEXT); + lv_obj_center(l); + return l; +} + +static void fmtMB(char* out, size_t n, uint32_t bytes) { + if (bytes >= 1024 * 1024) snprintf(out, n, "%.1f MB", bytes / 1048576.0); + else snprintf(out, n, "%lu KB", (unsigned long)((bytes + 1023) / 1024)); +} + +} // namespace regions + +void UITask::mapRegionsPopup() { + using namespace regions; + mapview::s_vpacks.scan(); + lv_obj_t* panel = navPopupPanel("Vector regions", true); + lv_obj_t* list = scrollList(panel); + int np = mapview::s_vpacks.count(); + if (np) { + lv_obj_t* g = group(list, nullptr); + for (int i = 0; i < np; i++) { + const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(i); + char sub[48], mb[16]; + fmtMB(mb, sizeof(mb), p.size); + snprintf(sub, sizeof(sub), "%s%s", mb, (p.flags & mapview::VectorPacks::F_CONTOURS) ? " - contours" : ""); + listRow(g, p.name[0] ? p.name : p.file, sub, onPack, (void*)(intptr_t)i); + } + } + lv_obj_t* l = label(list, np ? "Packs in /vmap on the SD card, made with tools/maps/osm_vector.py --pack." + : "No regions yet. Make a pack with tools/maps/osm_vector.py --pack and copy it to /vmap on the SD card.", + THEME_FONT_SMALL, theme::TEXT_MUTED); + lv_label_set_long_mode(l, LV_LABEL_LONG_WRAP); + lv_obj_set_width(l, LV_PCT(100)); +} + +void UITask::mapRegionPopup(int idx) { + using namespace regions; + if (idx < 0 || idx >= mapview::s_vpacks.count()) return; + s_sel = idx; + const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(idx); + lv_obj_t* panel = navPopupPanel(p.name[0] ? p.name : p.file, false); + char info[128], mb[16]; + fmtMB(mb, sizeof(mb), p.size); + snprintf(info, sizeof(info), "%s, zoom %d-18%s\n%s", mb, p.zmin, (p.flags & mapview::VectorPacks::F_CONTOURS) ? ", contours" : "", p.file); + lv_obj_t* l = label(panel, info, THEME_FONT_SMALL, theme::TEXT); + lv_label_set_long_mode(l, LV_LABEL_LONG_WRAP); + lv_obj_set_width(l, LV_PCT(100)); + lv_obj_t* r = toolRow(panel); + actButton(r, UI_SYMBOL_MAP " Show", A_SHOW, true); + s_del_lbl = actButton(r, LV_SYMBOL_TRASH " Delete", A_DELETE, false); + lv_obj_add_event_cb(s_del_lbl, onDeleted, LV_EVENT_DELETE, NULL); + s_del_armed_ms = 0; +} + +void UITask::mapRegionAction(uint8_t act) { + using namespace regions; + if (s_sel < 0 || s_sel >= mapview::s_vpacks.count()) return; + const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(s_sel); + switch (act) { + case A_SHOW: { + navClosePopup(); + double x0 = mapview::lonToTileX(p.box[0] / 1e6, 0), x1 = mapview::lonToTileX(p.box[2] / 1e6, 0); + double y0 = mapview::latToTileY(p.box[3] / 1e6, 0), y1 = mapview::latToTileY(p.box[1] / 1e6, 0); + int w = _map_area ? lv_obj_get_width(_map_area) : 320, h = _map_area ? lv_obj_get_height(_map_area) : 218; + _map_z = areas::fitZoom(x1 - x0, y1 - y0, w - 40, h - 60); + double s = (double)(1 << _map_z); + _map_cx = (x0 + x1) / 2 * s; + _map_cy = (y0 + y1) / 2 * s; + _map_follow = false; + layoutMap(); + return; + } + case A_DELETE: { + if (!s_del_armed_ms || millis() - s_del_armed_ms > 3000) { // a second tap within 3 s confirms + s_del_armed_ms = millis() | 1; + if (s_del_lbl) lv_label_set_text(s_del_lbl, "Delete?"); + return; + } + char path[80]; + snprintf(path, sizeof(path), "%s/%s", mapview::VECTOR_ROOT, p.file); + mapview::s_vpacks.closeOpen(); + remove(path); + mapview::s_vector.available(); // lists the packs again, forgets the data read + mapview::labels::flush(); + mapview::s_cache.invalidate(); + mapRegionsPopup(); + showToast("Region deleted"); + return; + } + } +} diff --git a/examples/companion_radio/ui-lvgl/MapScreen.h b/examples/companion_radio/ui-lvgl/MapScreen.h index 43e64915..8612e5dd 100644 --- a/examples/companion_radio/ui-lvgl/MapScreen.h +++ b/examples/companion_radio/ui-lvgl/MapScreen.h @@ -16,6 +16,7 @@ #include #include "map/TileProvider.h" +#include "map/VectorPacks.h" #include "map/VectorTileProvider.h" #include "map/VectorLabels.h" #include "map/TileCache.h" diff --git a/examples/companion_radio/ui-lvgl/NavMap.h b/examples/companion_radio/ui-lvgl/NavMap.h index fc040617..39ecb27f 100644 --- a/examples/companion_radio/ui-lvgl/NavMap.h +++ b/examples/companion_radio/ui-lvgl/NavMap.h @@ -779,7 +779,7 @@ void UITask::shareToMessage(const char* text) { namespace navmap { enum : uint8_t { TL_TRAIL_TOGGLE, TL_TRAIL_SAVE, TL_TRAIL_LOAD, TL_TRAIL_RESET, TL_TRAIL_GPX, TL_TRACKBACK, - TL_SHARE_TOGGLE, TL_SHARE_ONCE, TL_DOWNLOAD, TL_AREAS, TL_OPTIONS, + TL_SHARE_TOGGLE, TL_SHARE_ONCE, TL_DOWNLOAD, TL_AREAS, TL_REGIONS, TL_OPTIONS, TL_WP_HERE, TL_WP_COORDS, TL_SPOT_ADD, TL_SPOT_GO, TL_ST_LOAD, TL_ST_GPX, TL_ST_DELETE }; @@ -976,7 +976,7 @@ void UITask::navToolsPopup() { lv_obj_t* tsw = switchRow(g, "Hiking trails", "Marked routes in their colours", nullptr); if (lvport::trailsOn()) lv_obj_add_state(tsw, LV_STATE_CHECKED); lv_obj_add_event_cb(tsw, onTrails, LV_EVENT_VALUE_CHANGED, NULL); - lv_obj_t* vsw = switchRow(g, "Vector map (test)", "Drawn on the device from /vmap", nullptr); + lv_obj_t* vsw = switchRow(g, "Vector map", "Drawn on the device: trails, contours", nullptr); if (lvport::vectorOn()) lv_obj_add_state(vsw, LV_STATE_CHECKED); lv_obj_add_event_cb(vsw, onVectorMap, LV_EVENT_VALUE_CHANGED, NULL); listRow(g, "Download an area", "Pick it with a frame, for use offline", onNavTool, (void*)(uintptr_t)navmap::TL_DOWNLOAD); @@ -984,6 +984,10 @@ void UITask::navToolsPopup() { int na = mapview::s_areas.count(); snprintf(ar, sizeof(ar), na ? "%d on the card - rename, refresh, delete" : "None yet", na); listRow(g, "Map areas", ar, onNavTool, (void*)(uintptr_t)navmap::TL_AREAS); + char vr[48]; + int nv = mapview::s_vpacks.count(); + snprintf(vr, sizeof(vr), nv ? "%d on the card" : "None - packs go in /vmap", nv); + listRow(g, "Vector regions", vr, onNavTool, (void*)(uintptr_t)navmap::TL_REGIONS); listRow(g, "Map options", "Trail, live sharing, arrival alert", onNavTool, (void*)(uintptr_t)navmap::TL_OPTIONS); _nav_reset_armed_ms = 0; @@ -1278,6 +1282,9 @@ void UITask::navToolAction(uint8_t act) { case navmap::TL_AREAS: mapAreasPopup(); return; + case navmap::TL_REGIONS: + mapRegionsPopup(); + return; case navmap::TL_OPTIONS: // back returns to the map showSchemaSettings(settings::PG_NAV); return; diff --git a/examples/companion_radio/ui-lvgl/UITask.cpp b/examples/companion_radio/ui-lvgl/UITask.cpp index b77d0129..91eef312 100644 --- a/examples/companion_radio/ui-lvgl/UITask.cpp +++ b/examples/companion_radio/ui-lvgl/UITask.cpp @@ -2183,6 +2183,7 @@ static lv_obj_t* switchRow(lv_obj_t* parent, const char* text, const char* sub, #include "MapScreen.h" #include "NavMap.h" #include "MapAreas.h" +#include "MapRegions.h" #include "HomeScreen.h" #include "ClockScreen.h" #include "RadioScreen.h" @@ -3105,6 +3106,8 @@ extern "C" EMSCRIPTEN_KEEPALIVE void sim_open(const char* name) { if (!strcmp(name, "maptools")) { s_ui->navToolsPopup(); return; } if (!strcmp(name, "areasel")) { s_ui->areaSelectBegin(); return; } if (!strcmp(name, "areas")) { s_ui->mapAreasPopup(); return; } + if (!strcmp(name, "regions")) { s_ui->mapRegionsPopup(); return; } + if (!strcmp(name, "region0")) { s_ui->mapRegionPopup(0); return; } if (!strcmp(name, "advert")) { s_ui->advertPopup(); return; } } // Scrolls the screen's (or a popup's) main list by dy; returns what was left to scroll. diff --git a/examples/companion_radio/ui-lvgl/UITask.h b/examples/companion_radio/ui-lvgl/UITask.h index facc7ee0..58d3b47e 100644 --- a/examples/companion_radio/ui-lvgl/UITask.h +++ b/examples/companion_radio/ui-lvgl/UITask.h @@ -81,6 +81,9 @@ public: void mapAreasPopup(); void mapAreaPopup(int idx); void mapAreaAction(uint8_t act); + void mapRegionsPopup(); // Map tools > Vector regions (MapRegions.h) + void mapRegionPopup(int idx); + void mapRegionAction(uint8_t act); void mapAreaRenameDone(bool ok); int mapZoomLevel() const; void setTapWake(bool on); diff --git a/examples/companion_radio/ui-lvgl/map/VectorLabels.h b/examples/companion_radio/ui-lvgl/map/VectorLabels.h index 4b4974c9..c6c14d72 100644 --- a/examples/companion_radio/ui-lvgl/map/VectorLabels.h +++ b/examples/companion_radio/ui-lvgl/map/VectorLabels.h @@ -37,33 +37,36 @@ static void load(int dz, int x, int y) { PoiTile* t = &s_tiles[0]; for (PoiTile& s : s_tiles) if (s.used < t->used) t = &s; t->dz = dz; t->x = x; t->y = y; t->n = 0; t->used = ++s_tick; - char path[64]; - snprintf(path, sizeof(path), "%s/%d/%d/%d.vp", VECTOR_ROOT, dz, x, y); - FILE* f = fopen(path, "rb"); - if (!f) return; - uint8_t h[6]; - if (fread(h, 1, 6, f) != 6 || memcmp(h, "VP1", 3) != 0) { fclose(f); return; } - int count = h[4] | (h[5] << 8); + VectorPacks::Src src; + if (!s_vpacks.find(VectorPacks::K_POINTS, dz, x, y, src)) return; + uint8_t* buf = src.len >= 6 && src.len < 256 * 1024 ? psramBuf(src.len) : nullptr; + bool ok = buf && readFast(src.f, buf, src.len); + s_vpacks.done(src); + if (!ok || memcmp(buf, "VP1", 3) != 0) { free(buf); return; } + const uint8_t* q = buf + 6; + const uint8_t* end = buf + src.len; + int count = buf[4] | (buf[5] << 8); if (count > t->cap) { free(t->p); t->p = psramBuf(count); t->cap = t->p ? count : 0; } - while (t->n < t->cap && t->n < count) { - uint8_t r[8]; - if (fread(r, 1, 8, f) != 8) break; + while (t->n < t->cap && t->n < count && q + 8 <= end) { Poi& p = t->p[t->n]; - p.cls = r[0]; - p.u = r[1] | (r[2] << 8); - p.v = r[3] | (r[4] << 8); - p.ele = (int16_t)(r[5] | (r[6] << 8)); - int len = r[7] > LABEL_MAX ? LABEL_MAX : r[7]; - if (fread(p.name, 1, len, f) != (size_t)len) break; - if (r[7] > len) fseek(f, r[7] - len, SEEK_CUR); + p.cls = q[0]; + p.u = q[1] | (q[2] << 8); + p.v = q[3] | (q[4] << 8); + p.ele = (int16_t)(q[5] | (q[6] << 8)); + int nl = q[7]; + q += 8; + if (q + nl > end) break; + int len = nl > LABEL_MAX ? LABEL_MAX : nl; + memcpy(p.name, q, len); p.name[len] = '\0'; + q += nl; t->n++; } - fclose(f); + free(buf); } // What layout() placed, for the draw callback (names copied: a later read @@ -205,6 +208,12 @@ static void layout(double left, double top, int w, int h, int z, bool on, int bo lv_obj_invalidate(s_layer); } +// The packs changed: what was read (or found missing) goes. +static void flush() { + for (PoiTile& t : s_tiles) t.dz = -1; + clear(); +} + // A tile layout() found missing, read now; true if one was. static bool loadOne() { if (s_want_dz < 0) return false; diff --git a/examples/companion_radio/ui-lvgl/map/VectorPacks.h b/examples/companion_radio/ui-lvgl/map/VectorPacks.h new file mode 100644 index 00000000..0223190e --- /dev/null +++ b/examples/companion_radio/ui-lvgl/map/VectorPacks.h @@ -0,0 +1,167 @@ +#pragma once +// Vector map regions: packs (/sdcard/vmap/*.vpk from tools/maps/osm_vector.py +// --pack) read in place -- one file per region, its index of tiles and points +// in PSRAM. Loose files ({dz}/{x}/{y}.vt / .vp in the same folder, the tool +// without --pack) still work; a pack is looked in first. +// +// Pack: a 64-byte header ('VPK1', count, index and data offsets, box in +// degrees x 1e6, zooms, flags, name), `count` sorted 16-byte index entries +// (kind, dz, x, y, offset, length), then the files. +// +// Single-TU fragment: included by ui-lvgl/MapScreen.h before VectorTileProvider.h. + +#include + +namespace mapview { + +static const char* const VECTOR_ROOT = "/sdcard/vmap"; + +class VectorPacks { +public: + enum : uint8_t { K_TILE = 0, K_POINTS = 1 }; + enum : uint8_t { F_CONTOURS = 1 }; + static const int MAX = 16; + + struct Pack { + char file[48]; // name in VECTOR_ROOT + char name[29]; // the region's + int32_t box[4]; // lon0 lat0 lon1 lat1, degrees x 1e6 + uint8_t zmin, zmax, flags; + uint32_t count, index_off, data_off, size; + uint8_t* index; // count x 16 bytes (PSRAM); null: too big, searched in the file + }; + + // Where one file of the data is: read `len` bytes from `f` (already at its + // start), then done() -- a pack's file stays open for the next read. + struct Src { FILE* f = nullptr; uint32_t len = 0; bool owned = false; }; + + // Lists the packs again (the map opening, a download finished). + void scan() { + closeOpen(); + for (int i = 0; i < _n; i++) free(_p[i].index); + _n = 0; + DIR* d = opendir(VECTOR_ROOT); + if (!d) return; + while (struct dirent* e = readdir(d)) { + size_t l = strlen(e->d_name); + if (_n >= MAX || l < 5 || l >= sizeof(_p[0].file) || strcasecmp(e->d_name + l - 4, ".vpk") != 0) continue; + if (open(_p[_n], e->d_name)) _n++; + } + closedir(d); + } + + int count() const { return _n; } + const Pack& at(int i) const { return _p[i]; } + + bool find(uint8_t kind, int dz, int x, int y, Src& s) { + uint64_t key = keyOf(kind, dz, x, y); + for (int i = 0; i < _n; i++) { + Pack& p = _p[i]; + uint32_t off, len; + if (!lookup(p, key, off, len)) continue; + FILE* f = handle(i); + if (!f || fseek(f, p.data_off + off, SEEK_SET) != 0) return false; + s.f = f; s.len = len; s.owned = false; + return true; + } + char path[64]; // a loose file + snprintf(path, sizeof(path), "%s/%d/%d/%d.%s", VECTOR_ROOT, dz, x, y, kind == K_TILE ? "vt" : "vp"); + FILE* f = fopen(path, "rb"); + if (!f) return false; + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, 0, SEEK_SET); + s.f = f; s.len = sz > 0 ? (uint32_t)sz : 0; s.owned = true; + return true; + } + void done(Src& s) { + if (s.owned && s.f) fclose(s.f); + s.f = nullptr; + } + + // Before a pack file is replaced or deleted. + void closeOpen() { + if (_open_f) fclose(_open_f); + _open_f = nullptr; + _open_i = -1; + } + +private: + Pack _p[MAX]; + int _n = 0; + FILE* _open_f = nullptr; // the pack read last + int _open_i = -1; + static const uint32_t INDEX_IN_RAM_MAX = 2u * 1024 * 1024; + + static uint64_t keyOf(uint8_t kind, int dz, int x, int y) { + return ((uint64_t)kind << 40) | ((uint64_t)(dz & 0xFF) << 32) | ((uint64_t)(x & 0xFFFF) << 16) | (uint64_t)(y & 0xFFFF); + } + static uint64_t keyAt(const uint8_t* e) { + return keyOf(e[0], e[1], e[2] | (e[3] << 8), e[4] | (e[5] << 8)); + } + static uint32_t u32(const uint8_t* b) { return b[0] | (b[1] << 8) | (b[2] << 16) | ((uint32_t)b[3] << 24); } + + FILE* handle(int i) { + if (_open_i == i && _open_f) return _open_f; + closeOpen(); + char path[64]; + snprintf(path, sizeof(path), "%s/%s", VECTOR_ROOT, _p[i].file); + _open_f = fopen(path, "rb"); + _open_i = _open_f ? i : -1; + return _open_f; + } + + bool open(Pack& p, const char* file) { + char path[64]; + snprintf(path, sizeof(path), "%s/%s", VECTOR_ROOT, file); + FILE* f = fopen(path, "rb"); + if (!f) return false; + uint8_t h[64]; + bool ok = fread(h, 1, 64, f) == 64 && memcmp(h, "VPK1", 4) == 0; + if (ok) { + snprintf(p.file, sizeof(p.file), "%s", file); + p.count = u32(h + 4); p.index_off = u32(h + 8); p.data_off = u32(h + 12); + for (int k = 0; k < 4; k++) p.box[k] = (int32_t)u32(h + 16 + 4 * k); + p.zmin = h[32]; p.zmax = h[33]; p.flags = h[34]; + memcpy(p.name, h + 36, 28); + p.name[28] = '\0'; + fseek(f, 0, SEEK_END); + p.size = (uint32_t)ftell(f); + p.index = nullptr; + uint32_t isz = p.count * 16; + ok = p.index_off + isz <= p.size && p.data_off <= p.size; + if (ok && isz <= INDEX_IN_RAM_MAX) { + p.index = psramBuf(isz ? isz : 1); + if (p.index && (fseek(f, p.index_off, SEEK_SET) != 0 || !readFast(f, p.index, isz))) { free(p.index); p.index = nullptr; } + } + } + fclose(f); + return ok; + } + + // Binary search of the index (in RAM, or read entry by entry). + bool lookup(Pack& p, uint64_t key, uint32_t& off, uint32_t& len) { + int lo = 0, hi = (int)p.count - 1; + uint8_t e[16]; + FILE* f = nullptr; + while (lo <= hi) { + int mid = (lo + hi) / 2; + const uint8_t* ent; + if (p.index) { + ent = p.index + 16 * mid; + } else { + if (!f) f = handle((int)(&p - _p)); + if (!f || fseek(f, p.index_off + 16 * mid, SEEK_SET) != 0 || fread(e, 1, 16, f) != 16) return false; + ent = e; + } + uint64_t k = keyAt(ent); + if (k == key) { off = u32(ent + 8); len = u32(ent + 12); return true; } + if (k < key) lo = mid + 1; else hi = mid - 1; + } + return false; + } +}; + +static VectorPacks s_vpacks; + +} // namespace mapview diff --git a/examples/companion_radio/ui-lvgl/map/VectorTileProvider.h b/examples/companion_radio/ui-lvgl/map/VectorTileProvider.h index c9cccdd2..02202ae8 100644 --- a/examples/companion_radio/ui-lvgl/map/VectorTileProvider.h +++ b/examples/companion_radio/ui-lvgl/map/VectorTileProvider.h @@ -1,6 +1,6 @@ #pragma once -// Vector map (spike): tiles from tools/maps/osm_vector.py under VECTOR_ROOT, -// {dz}/{x}/{y}.vt (format VT3) at data zooms 10, 12 and 14, drawn here into the 256x256 RGB565 +// Vector map: tiles from tools/maps/osm_vector.py (region packs or loose +// files, map/VectorPacks.h), format VT3, at data zooms 10, 12 and 14, drawn here into the 256x256 RGB565 // buffer the map asks for -- polygons by scanline (even-odd), lines as quads // with round joins. No anti-aliasing; names of points in VectorLabels.h. Where there is no vector // data (or below zoom 10) the raster provider draws instead, so the two mix. @@ -15,14 +15,14 @@ namespace mapview { static inline int32_t vmin(int32_t a, int32_t b) { return a < b ? a : b; } static inline int32_t vmax(int32_t a, int32_t b) { return a > b ? a : b; } -static const char* const VECTOR_ROOT = "/sdcard/vmap"; - class VectorTileProvider : public TileProvider { public: explicit VectorTileProvider(TileProvider& fallback) : _fb(fallback) {} bool available() override { struct stat st; + s_vpacks.scan(); + for (Data& d : _slot) d.dz = -1; // a pack may have changed _have = stat(VECTOR_ROOT, &st) == 0 && S_ISDIR(st.st_mode); bool fb = _fb.available(); return _have || fb; @@ -104,14 +104,10 @@ private: d = &_slot[0]; for (Data& s : _slot) if (s.used < d->used) d = &s; d->dz = dz; d->x = x; d->y = y; d->ok = false; d->used = ++_tick; - char path[64]; - snprintf(path, sizeof(path), "%s/%d/%d/%d.vt", VECTOR_ROOT, dz, x, y); - FILE* f = fopen(path, "rb"); - if (!f) return false; - fseek(f, 0, SEEK_END); - long sz = ftell(f); - fseek(f, 0, SEEK_SET); - if (sz < 6 || sz > 2 * 1024 * 1024) { fclose(f); return false; } + VectorPacks::Src src; + if (!s_vpacks.find(VectorPacks::K_TILE, dz, x, y, src)) return false; + long sz = src.len; + if (sz < 6 || sz > 2 * 1024 * 1024) { s_vpacks.done(src); return false; } if ((size_t)sz > d->cap) { if (d->buf) free(d->buf); #if defined(ESP32) @@ -120,10 +116,10 @@ private: d->buf = (uint8_t*)malloc(sz); #endif d->cap = d->buf ? sz : 0; - if (!d->buf) { fclose(f); return false; } + if (!d->buf) { s_vpacks.done(src); return false; } } - d->len = readFast(f, d->buf, sz) ? sz : 0; - fclose(f); + d->len = readFast(src.f, d->buf, sz) ? sz : 0; + s_vpacks.done(src); d->ok = d->len == (size_t)sz && memcmp(d->buf, "VT3", 3) == 0; _data = d->buf; _data_len = d->len; diff --git a/tools/maps/osm_vector.py b/tools/maps/osm_vector.py index 9e2d0348..e48b2b6a 100644 --- a/tools/maps/osm_vector.py +++ b/tools/maps/osm_vector.py @@ -8,6 +8,13 @@ buildings; and named points for the labels (places, peaks, huts, springs...). No street names (the raster map has them). tools/maps/osm_vector.py area.json [points.json] --out vmap/ [--dem dem-cache/] + tools/maps/osm_vector.py area.json ... --pack tatry.vpk --name "Tatry" + tools/maps/osm_vector.py --pbf malopolskie-latest.osm.pbf --bbox 49.15,19.75,49.35,20.25 \ + --dem dem-cache/ --pack tatry.vpk --name "Tatry" + +--pbf reads a Geofabrik extract (tools/maps/pbf.py, needs pyosmium: pip +install osmium) for the --bbox (south,west,north,east) -- no Overpass limits; +give it more than once for a box across a border. With --dem, contour lines too (tools/maps/dem.py: Terrain Tiles fetched into that folder): every 20 m (from zoom 15) with a darker one every 100 m. @@ -16,6 +23,17 @@ Three data zooms: 10 (drawn at z10-11), 12 (z12-13), 14 (z14-18); the device pic data tile covering the tile it draws and scales it. Copy the output folder to the card as /sdcard/vmap. +A pack (--pack) holds a whole region in one file, which the device reads in +place from /sdcard/vmap/*.vpk (a card writes one big file much faster than +thousands of small ones, and a download is one file): + header, 64 bytes: 'VPK1' count:u32 index_off:u32 data_off:u32 + lon0 lat0 lon1 lat1: i32 (degrees x 1e6) zmin zmax flags(1 contours) 0: u8 + name: 28 bytes UTF-8, 0-padded + count x index entry, 16 bytes, sorted: kind:u8 (0 tile, 1 points) dz:u8 + x:u16 y:u16 0:u16 off:u32 (from data_off) len:u32 + the files, one after another. +Without --pack the files go to --out as {dz}/{x}/{y}.vt / .vp. + Tile format 'VT3' (little-endian): 'V' 'T' '3' dz:u8 count:u16 count x feature, in drawing order: @@ -395,15 +413,36 @@ def clip_ring(pts, lo, hi): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument('json', nargs='+') + ap.add_argument('json', nargs='*', help='Overpass JSON file(s)') + ap.add_argument('--pbf', action='append', default=[], help='an OSM PBF extract (with --bbox); again for more') + ap.add_argument('--bbox', help='south,west,north,east: only this box (and the pack\'s box)') ap.add_argument('--out', default='vmap') + ap.add_argument('--pack', metavar='FILE.vpk', help='write one pack file instead of a folder') + ap.add_argument('--name', default='', help="the region's name in the pack") + ap.add_argument('--lang', help='names in this language where OSM has them (name:pl ...)') ap.add_argument('--dem', metavar='CACHE', help='add contour lines; DEM tiles are kept in this folder') ap.add_argument('--contour-step', type=int, default=20) ap.add_argument('--index-step', type=int, default=100) a = ap.parse_args() + box = None + if a.bbox: + s_, w_, n_, e_ = (float(v) for v in a.bbox.split(',')) + box = (w_, s_, e_, n_) elements = [] for fn in a.json: elements += json.load(open(fn))['elements'] + if a.pbf: + if not box: + ap.error('--pbf needs --bbox') + import pbf + for fn in a.pbf: # neighbouring extracts: what both have is drawn twice, harmless + elements += pbf.read_pbf(fn, box) + if not elements: + ap.error('no input') + if not box: + lons = [p['lon'] for e in elements for p in (e.get('geometry') or []) if p] or [0] + lats = [p['lat'] for e in elements for p in (e.get('geometry') or []) if p] or [0] + box = (min(lons), min(lats), max(lons), max(lats)) feats = [] # (cls, colour, 'poly'|'line', [parts in world coords]) way_routes = defaultdict(set) # way id -> route colours along it @@ -414,7 +453,12 @@ def main(): pt = point_class(t) if pt is not None: pos = (e['lon'], e['lat']) if 'lat' in e else (e['center']['lon'], e['center']['lat']) if 'center' in e else None - name = t.get('name', '') + if not pos and e.get('geometry'): # a hut drawn as a building: its middle + g = [p for p in e['geometry'] if p] + pos = (sum(p['lon'] for p in g) / len(g), sum(p['lat'] for p in g) / len(g)) + if pos and not (box[0] <= pos[0] <= box[2] and box[1] <= pos[1] <= box[3]): + pos = None + name = (a.lang and t.get('name:' + a.lang)) or t.get('name', '') ele = elevation(t) if pos and (name or (pt == T_PEAK and ele is not None)): # unnamed: just peaks, by their height x, y = world(*pos) @@ -457,19 +501,19 @@ def main(): packed = sum(c << (4 * i) for i, c in enumerate(cols)) feats.append((L_ROUTES, packed, 'line', join_lines(ways))) if a.dem: - lons = [p['lon'] for e in elements for p in (e.get('geometry') or []) if p] - lats = [p['lat'] for e in elements for p in (e.get('geometry') or []) if p] nc = 0 - for v, line in dem.contour_lines(a.dem, min(lons), min(lats), max(lons), max(lats), a.contour_step): + for v, line in dem.contour_lines(a.dem, *box, a.contour_step): feats.append((L_CONTOUR_IDX if round(v) % a.index_step == 0 else L_CONTOUR, 0, 'line', [line])) nc += 1 print(f'{nc} contour lines', file=sys.stderr) print(f'{len(feats)} features, {len(way_routes)} route ways', file=sys.stderr) - total_bytes = total_tiles = 0 + files = {} # (kind 0 tile / 1 points, dz, x, y) -> bytes for dz in DATA_ZOOMS: n = 1 << dz base_tol = 1.0 if dz == DATA_ZOOMS[-1] else 4.0 # half a pixel at the deepest zoom drawn from it + (wx0, wy0), (wx1, wy1) = world(box[0], box[3]), world(box[2], box[1]) + tx_lo, ty_lo, tx_hi, ty_hi = int(wx0 * n), int(wy0 * n), int(wx1 * n), int(wy1 * n) # tiles of the box only tiles = defaultdict(list) for cls, col, kind, parts in feats: if MIN_DZ.get(cls, 0) > dz: @@ -480,8 +524,8 @@ def main(): allp = [p for part in parts for p in part] if not allp: continue - x0 = int(min(p[0] for p in allp) * n); x1 = int(max(p[0] for p in allp) * n) - y0 = int(min(p[1] for p in allp) * n); y1 = int(max(p[1] for p in allp) * n) + x0 = max(int(min(p[0] for p in allp) * n), tx_lo); x1 = min(int(max(p[0] for p in allp) * n), tx_hi) + y0 = max(int(min(p[1] for p in allp) * n), ty_lo); y1 = min(int(max(p[1] for p in allp) * n), ty_hi) for tx in range(x0, x1 + 1): for ty in range(y0, y1 + 1): out = [] @@ -529,12 +573,7 @@ def main(): buf += varint(len(body)) + body count += 1 struct.pack_into(' the elements osm_vector.py works on (as Overpass gives +them with `out geom;`), for a box. Needs pyosmium (pip install osmium). + +Two passes over the file: relations first (hiking routes, multipolygons: which +ways they need), then nodes and ways with their node locations. A way is kept +if it touches the box; a relation if one of its ways was kept. +""" +import sys + +WAY_KEYS = ('highway', 'waterway', 'natural', 'landuse', 'building', 'tourism') +AREA_KEYS = ('natural', 'landuse', 'building', 'water') +NODE_KEYS = ('natural', 'tourism', 'amenity', 'place') + + +def read_pbf(path, box): + """box: (lon0, lat0, lon1, lat1). Returns a list of element dicts.""" + import osmium + lon0, lat0, lon1, lat1 = box + + rels, want = [], {} # relations kept; way id -> [(relation index, role)] + for r in osmium.FileProcessor(path, osmium.osm.RELATION): + t = dict(r.tags) + if not (t.get('route') == 'hiking' or (t.get('type') == 'multipolygon' and any(k in t for k in AREA_KEYS))): + continue + i = len(rels) + rels.append({'type': 'relation', 'id': r.id, 'tags': t, 'members': []}) + for m in r.members: + if m.type == 'w': + want.setdefault(m.ref, []).append((i, m.role)) + print(f'PBF: {len(rels)} relations', file=sys.stderr) + + elements, geoms = [], {} + # Untagged nodes (most of them) never reach Python; ways do, untagged ones + # can be a multipolygon's outline. + fp = osmium.FileProcessor(path, osmium.osm.NODE | osmium.osm.WAY).with_locations() \ + .with_filter(osmium.filter.EmptyTagFilter().enable_for(osmium.osm.NODE)) + for o in fp: + if o.is_node(): + if not any(k in o.tags for k in NODE_KEYS): + continue + loc = o.location + if loc.valid() and lon0 <= loc.lon <= lon1 and lat0 <= loc.lat <= lat1: + elements.append({'type': 'node', 'id': o.id, 'lat': loc.lat, 'lon': loc.lon, 'tags': dict(o.tags)}) + continue + in_rel = o.id in want + tagged = any(k in o.tags for k in WAY_KEYS) + if not in_rel and not tagged: + continue + g = [{'lat': n.lat, 'lon': n.lon} for n in o.nodes if n.location.valid()] + if len(g) < 2: + continue + if (max(p['lon'] for p in g) < lon0 or min(p['lon'] for p in g) > lon1 or + max(p['lat'] for p in g) < lat0 or min(p['lat'] for p in g) > lat1): + continue + if in_rel: + geoms[o.id] = g + if tagged: + elements.append({'type': 'way', 'id': o.id, 'tags': dict(o.tags), 'geometry': g}) + for wid, uses in want.items(): + if wid in geoms: + for i, role in uses: + rels[i]['members'].append({'type': 'way', 'ref': wid, 'role': role, 'geometry': geoms[wid]}) + kept = [r for r in rels if r['members']] + print(f'PBF: {len(elements)} nodes / ways, {len(kept)} relations in the box', file=sys.stderr) + return elements + kept