feat(ui-lvgl): vector region packs, PBF input, region list

One .vpk file per region in /vmap (header, sorted index of tiles and
points, then the data), read in place with the index in PSRAM; loose
{dz}/{x}/{y}.vt/.vp files still work. osm_vector.py writes packs
(--pack/--name), reads OSM PBF extracts for a box (--pbf/--bbox, via
pyosmium) and picks label languages (--lang). Map tools > Vector regions
lists the packs on the card: show one on the map, delete it. No
on-device download: packs are made with the tool.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/development/l2-roadmap.md

@@ -267,8 +267,18 @@ Ideas to come back to (2026-09-26):
 - [x] Contours on the vector map: osm_vector.py --dem (tools/maps/dem.py,
       standard library: Terrain Tiles from AWS Open Data, SRTM ~25 m,
       smoothed, marching squares); 100 m lines from z12, 20 m from z15.
-- [ ] Vector maps next: a PBF pipeline for regions + download on the
-      device, styling (anti-aliasing, dark theme).
+- [x] Vector regions: a region is one pack file (/vmap/*.vpk, index +
+      tiles + points, read in place; osm_vector.py --pack); input from
+      Overpass or Geofabrik PBF extracts (--pbf, pyosmium; several across a
+      border) for a --bbox; --lang for names. Map tools > Vector regions
+      lists the packs (show, delete). No download on the device: packs are
+      made by the user.
+- [ ] A Tools page on the solo site (meshcore-solo-site, beside the sim):
+      vector pack maker in the browser (pick a box on a map, Overpass +
+      DEM, a JS port of osm_vector.py in a worker, .vpk to save), the GPX
+      downloader (tools/gpx-downloader) and screenshots (tools/screenshot.py
+      over Web Serial).
+- [ ] Vector map styling: anti-aliasing, dark theme.
 
 User list, second batch (2026-09-26):
 

examples/companion_radio/ui-lvgl/MapRegions.h

@@ -0,0 +1,120 @@
+#pragma once
+// Map tools > Vector regions: the region packs on the card (/vmap/*.vpk,
+// map/VectorPacks.h, made with tools/maps/osm_vector.py --pack): show one on
+// the map, delete it.
+//
+// Single-TU fragment: included by ui-lvgl/UITask.cpp after MapAreas.h.
+
+namespace regions {
+
+static int s_sel = -1;                // the pack a popup is about
+static uint32_t s_del_armed_ms = 0;
+static lv_obj_t* s_del_lbl = nullptr;
+
+enum : uint8_t { A_SHOW, A_DELETE };
+
+static void onDeleted(lv_event_t* e) { if (lv_event_get_target(e) == s_del_lbl) s_del_lbl = nullptr; }
+static void onPack(lv_event_t* e) { s_ui->mapRegionPopup((int)(intptr_t)lv_event_get_user_data(e)); }
+static void onAct(lv_event_t* e)  { s_ui->mapRegionAction((uint8_t)(uintptr_t)lv_event_get_user_data(e)); }
+
+static lv_obj_t* actButton(lv_obj_t* row, const char* text, uint8_t act, bool accent) {
+  lv_obj_t* b = lv_button_create(row);
+  lv_obj_set_height(b, 36);
+  lv_obj_set_flex_grow(b, 1);
+  lv_obj_set_style_pad_hor(b, 4, 0);
+  lv_obj_set_style_radius(b, theme::RADIUS, 0);
+  lv_obj_set_style_shadow_width(b, 0, 0);
+  lv_obj_set_style_bg_color(b, lv_color_hex(accent ? theme::ACCENT_DIM : theme::SURFACE), 0);
+  lv_obj_set_style_bg_color(b, lv_color_hex(theme::SURFACE_2), LV_STATE_PRESSED);
+  lv_obj_add_event_cb(b, onAct, LV_EVENT_CLICKED, (void*)(uintptr_t)act);
+  lv_obj_t* l = label(b, text, THEME_FONT_SMALL, theme::TEXT);
+  lv_obj_center(l);
+  return l;
+}
+
+static void fmtMB(char* out, size_t n, uint32_t bytes) {
+  if (bytes >= 1024 * 1024) snprintf(out, n, "%.1f MB", bytes / 1048576.0);
+  else snprintf(out, n, "%lu KB", (unsigned long)((bytes + 1023) / 1024));
+}
+
+}  // namespace regions
+
+void UITask::mapRegionsPopup() {
+  using namespace regions;
+  mapview::s_vpacks.scan();
+  lv_obj_t* panel = navPopupPanel("Vector regions", true);
+  lv_obj_t* list = scrollList(panel);
+  int np = mapview::s_vpacks.count();
+  if (np) {
+    lv_obj_t* g = group(list, nullptr);
+    for (int i = 0; i < np; i++) {
+      const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(i);
+      char sub[48], mb[16];
+      fmtMB(mb, sizeof(mb), p.size);
+      snprintf(sub, sizeof(sub), "%s%s", mb, (p.flags & mapview::VectorPacks::F_CONTOURS) ? "  -  contours" : "");
+      listRow(g, p.name[0] ? p.name : p.file, sub, onPack, (void*)(intptr_t)i);
+    }
+  }
+  lv_obj_t* l = label(list, np ? "Packs in /vmap on the SD card, made with tools/maps/osm_vector.py --pack."
+                               : "No regions yet. Make a pack with tools/maps/osm_vector.py --pack and copy it to /vmap on the SD card.",
+                      THEME_FONT_SMALL, theme::TEXT_MUTED);
+  lv_label_set_long_mode(l, LV_LABEL_LONG_WRAP);
+  lv_obj_set_width(l, LV_PCT(100));
+}
+
+void UITask::mapRegionPopup(int idx) {
+  using namespace regions;
+  if (idx < 0 || idx >= mapview::s_vpacks.count()) return;
+  s_sel = idx;
+  const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(idx);
+  lv_obj_t* panel = navPopupPanel(p.name[0] ? p.name : p.file, false);
+  char info[128], mb[16];
+  fmtMB(mb, sizeof(mb), p.size);
+  snprintf(info, sizeof(info), "%s, zoom %d-18%s\n%s", mb, p.zmin, (p.flags & mapview::VectorPacks::F_CONTOURS) ? ", contours" : "", p.file);
+  lv_obj_t* l = label(panel, info, THEME_FONT_SMALL, theme::TEXT);
+  lv_label_set_long_mode(l, LV_LABEL_LONG_WRAP);
+  lv_obj_set_width(l, LV_PCT(100));
+  lv_obj_t* r = toolRow(panel);
+  actButton(r, UI_SYMBOL_MAP " Show", A_SHOW, true);
+  s_del_lbl = actButton(r, LV_SYMBOL_TRASH " Delete", A_DELETE, false);
+  lv_obj_add_event_cb(s_del_lbl, onDeleted, LV_EVENT_DELETE, NULL);
+  s_del_armed_ms = 0;
+}
+
+void UITask::mapRegionAction(uint8_t act) {
+  using namespace regions;
+  if (s_sel < 0 || s_sel >= mapview::s_vpacks.count()) return;
+  const mapview::VectorPacks::Pack& p = mapview::s_vpacks.at(s_sel);
+  switch (act) {
+    case A_SHOW: {
+      navClosePopup();
+      double x0 = mapview::lonToTileX(p.box[0] / 1e6, 0), x1 = mapview::lonToTileX(p.box[2] / 1e6, 0);
+      double y0 = mapview::latToTileY(p.box[3] / 1e6, 0), y1 = mapview::latToTileY(p.box[1] / 1e6, 0);
+      int w = _map_area ? lv_obj_get_width(_map_area) : 320, h = _map_area ? lv_obj_get_height(_map_area) : 218;
+      _map_z = areas::fitZoom(x1 - x0, y1 - y0, w - 40, h - 60);
+      double s = (double)(1 << _map_z);
+      _map_cx = (x0 + x1) / 2 * s;
+      _map_cy = (y0 + y1) / 2 * s;
+      _map_follow = false;
+      layoutMap();
+      return;
+    }
+    case A_DELETE: {
+      if (!s_del_armed_ms || millis() - s_del_armed_ms > 3000) {   // a second tap within 3 s confirms
+        s_del_armed_ms = millis() | 1;
+        if (s_del_lbl) lv_label_set_text(s_del_lbl, "Delete?");
+        return;
+      }
+      char path[80];
+      snprintf(path, sizeof(path), "%s/%s", mapview::VECTOR_ROOT, p.file);
+      mapview::s_vpacks.closeOpen();
+      remove(path);
+      mapview::s_vector.available();   // lists the packs again, forgets the data read
+      mapview::labels::flush();
+      mapview::s_cache.invalidate();
+      mapRegionsPopup();
+      showToast("Region deleted");
+      return;
+    }
+  }
+}

examples/companion_radio/ui-lvgl/MapScreen.h

@@ -16,6 +16,7 @@
 
 #include <math.h>
 #include "map/TileProvider.h"
+#include "map/VectorPacks.h"
 #include "map/VectorTileProvider.h"
 #include "map/VectorLabels.h"
 #include "map/TileCache.h"

examples/companion_radio/ui-lvgl/NavMap.h

@@ -779,7 +779,7 @@ void UITask::shareToMessage(const char* text) {
 
 namespace navmap {
 enum : uint8_t { TL_TRAIL_TOGGLE, TL_TRAIL_SAVE, TL_TRAIL_LOAD, TL_TRAIL_RESET, TL_TRAIL_GPX, TL_TRACKBACK,
-                 TL_SHARE_TOGGLE, TL_SHARE_ONCE, TL_DOWNLOAD, TL_AREAS, TL_OPTIONS,
+                 TL_SHARE_TOGGLE, TL_SHARE_ONCE, TL_DOWNLOAD, TL_AREAS, TL_REGIONS, TL_OPTIONS,
                  TL_WP_HERE, TL_WP_COORDS, TL_SPOT_ADD, TL_SPOT_GO,
                  TL_ST_LOAD, TL_ST_GPX, TL_ST_DELETE };
 
@@ -976,7 +976,7 @@ void UITask::navToolsPopup() {
   lv_obj_t* tsw = switchRow(g, "Hiking trails", "Marked routes in their colours", nullptr);
   if (lvport::trailsOn()) lv_obj_add_state(tsw, LV_STATE_CHECKED);
   lv_obj_add_event_cb(tsw, onTrails, LV_EVENT_VALUE_CHANGED, NULL);
-  lv_obj_t* vsw = switchRow(g, "Vector map (test)", "Drawn on the device from /vmap", nullptr);
+  lv_obj_t* vsw = switchRow(g, "Vector map", "Drawn on the device: trails, contours", nullptr);
   if (lvport::vectorOn()) lv_obj_add_state(vsw, LV_STATE_CHECKED);
   lv_obj_add_event_cb(vsw, onVectorMap, LV_EVENT_VALUE_CHANGED, NULL);
   listRow(g, "Download an area", "Pick it with a frame, for use offline", onNavTool, (void*)(uintptr_t)navmap::TL_DOWNLOAD);
@@ -984,6 +984,10 @@ void UITask::navToolsPopup() {
   int na = mapview::s_areas.count();
   snprintf(ar, sizeof(ar), na ? "%d on the card - rename, refresh, delete" : "None yet", na);
   listRow(g, "Map areas", ar, onNavTool, (void*)(uintptr_t)navmap::TL_AREAS);
+  char vr[48];
+  int nv = mapview::s_vpacks.count();
+  snprintf(vr, sizeof(vr), nv ? "%d on the card" : "None - packs go in /vmap", nv);
+  listRow(g, "Vector regions", vr, onNavTool, (void*)(uintptr_t)navmap::TL_REGIONS);
   listRow(g, "Map options", "Trail, live sharing, arrival alert", onNavTool, (void*)(uintptr_t)navmap::TL_OPTIONS);
 
   _nav_reset_armed_ms = 0;
@@ -1278,6 +1282,9 @@ void UITask::navToolAction(uint8_t act) {
     case navmap::TL_AREAS:
       mapAreasPopup();
       return;
+    case navmap::TL_REGIONS:
+      mapRegionsPopup();
+      return;
     case navmap::TL_OPTIONS:   // back returns to the map
       showSchemaSettings(settings::PG_NAV);
       return;

examples/companion_radio/ui-lvgl/UITask.cpp

@@ -2183,6 +2183,7 @@ static lv_obj_t* switchRow(lv_obj_t* parent, const char* text, const char* sub,
 #include "MapScreen.h"
 #include "NavMap.h"
 #include "MapAreas.h"
+#include "MapRegions.h"
 #include "HomeScreen.h"
 #include "ClockScreen.h"
 #include "RadioScreen.h"
@@ -3105,6 +3106,8 @@ extern "C" EMSCRIPTEN_KEEPALIVE void sim_open(const char* name) {
   if (!strcmp(name, "maptools")) { s_ui->navToolsPopup(); return; }
   if (!strcmp(name, "areasel")) { s_ui->areaSelectBegin(); return; }
   if (!strcmp(name, "areas")) { s_ui->mapAreasPopup(); return; }
+  if (!strcmp(name, "regions")) { s_ui->mapRegionsPopup(); return; }
+  if (!strcmp(name, "region0")) { s_ui->mapRegionPopup(0); return; }
   if (!strcmp(name, "advert")) { s_ui->advertPopup(); return; }
 }
 // Scrolls the screen's (or a popup's) main list by dy; returns what was left to scroll.

examples/companion_radio/ui-lvgl/UITask.h

@@ -81,6 +81,9 @@ public:
   void mapAreasPopup();
   void mapAreaPopup(int idx);
   void mapAreaAction(uint8_t act);
+  void mapRegionsPopup();          // Map tools > Vector regions (MapRegions.h)
+  void mapRegionPopup(int idx);
+  void mapRegionAction(uint8_t act);
   void mapAreaRenameDone(bool ok);
   int  mapZoomLevel() const;
   void setTapWake(bool on);

examples/companion_radio/ui-lvgl/map/VectorLabels.h

@@ -37,33 +37,36 @@ static void load(int dz, int x, int y) {
   PoiTile* t = &s_tiles[0];
   for (PoiTile& s : s_tiles) if (s.used < t->used) t = &s;
   t->dz = dz; t->x = x; t->y = y; t->n = 0; t->used = ++s_tick;
-  char path[64];
-  snprintf(path, sizeof(path), "%s/%d/%d/%d.vp", VECTOR_ROOT, dz, x, y);
-  FILE* f = fopen(path, "rb");
-  if (!f) return;
-  uint8_t h[6];
-  if (fread(h, 1, 6, f) != 6 || memcmp(h, "VP1", 3) != 0) { fclose(f); return; }
-  int count = h[4] | (h[5] << 8);
+  VectorPacks::Src src;
+  if (!s_vpacks.find(VectorPacks::K_POINTS, dz, x, y, src)) return;
+  uint8_t* buf = src.len >= 6 && src.len < 256 * 1024 ? psramBuf<uint8_t>(src.len) : nullptr;
+  bool ok = buf && readFast(src.f, buf, src.len);
+  s_vpacks.done(src);
+  if (!ok || memcmp(buf, "VP1", 3) != 0) { free(buf); return; }
+  const uint8_t* q = buf + 6;
+  const uint8_t* end = buf + src.len;
+  int count = buf[4] | (buf[5] << 8);
   if (count > t->cap) {
     free(t->p);
     t->p = psramBuf<Poi>(count);
     t->cap = t->p ? count : 0;
   }
-  while (t->n < t->cap && t->n < count) {
-    uint8_t r[8];
-    if (fread(r, 1, 8, f) != 8) break;
+  while (t->n < t->cap && t->n < count && q + 8 <= end) {
     Poi& p = t->p[t->n];
-    p.cls = r[0];
-    p.u = r[1] | (r[2] << 8);
-    p.v = r[3] | (r[4] << 8);
-    p.ele = (int16_t)(r[5] | (r[6] << 8));
-    int len = r[7] > LABEL_MAX ? LABEL_MAX : r[7];
-    if (fread(p.name, 1, len, f) != (size_t)len) break;
-    if (r[7] > len) fseek(f, r[7] - len, SEEK_CUR);
+    p.cls = q[0];
+    p.u = q[1] | (q[2] << 8);
+    p.v = q[3] | (q[4] << 8);
+    p.ele = (int16_t)(q[5] | (q[6] << 8));
+    int nl = q[7];
+    q += 8;
+    if (q + nl > end) break;
+    int len = nl > LABEL_MAX ? LABEL_MAX : nl;
+    memcpy(p.name, q, len);
     p.name[len] = '\0';
+    q += nl;
     t->n++;
   }
-  fclose(f);
+  free(buf);
 }
 
 // What layout() placed, for the draw callback (names copied: a later read
@@ -205,6 +208,12 @@ static void layout(double left, double top, int w, int h, int z, bool on, int bo
   lv_obj_invalidate(s_layer);
 }
 
+// The packs changed: what was read (or found missing) goes.
+static void flush() {
+  for (PoiTile& t : s_tiles) t.dz = -1;
+  clear();
+}
+
 // A tile layout() found missing, read now; true if one was.
 static bool loadOne() {
   if (s_want_dz < 0) return false;

examples/companion_radio/ui-lvgl/map/VectorPacks.h

@@ -0,0 +1,167 @@
+#pragma once
+// Vector map regions: packs (/sdcard/vmap/*.vpk from tools/maps/osm_vector.py
+// --pack) read in place -- one file per region, its index of tiles and points
+// in PSRAM. Loose files ({dz}/{x}/{y}.vt / .vp in the same folder, the tool
+// without --pack) still work; a pack is looked in first.
+//
+// Pack: a 64-byte header ('VPK1', count, index and data offsets, box in
+// degrees x 1e6, zooms, flags, name), `count` sorted 16-byte index entries
+// (kind, dz, x, y, offset, length), then the files.
+//
+// Single-TU fragment: included by ui-lvgl/MapScreen.h before VectorTileProvider.h.
+
+#include <dirent.h>
+
+namespace mapview {
+
+static const char* const VECTOR_ROOT = "/sdcard/vmap";
+
+class VectorPacks {
+public:
+  enum : uint8_t { K_TILE = 0, K_POINTS = 1 };
+  enum : uint8_t { F_CONTOURS = 1 };
+  static const int MAX = 16;
+
+  struct Pack {
+    char file[48];     // name in VECTOR_ROOT
+    char name[29];     // the region's
+    int32_t box[4];    // lon0 lat0 lon1 lat1, degrees x 1e6
+    uint8_t zmin, zmax, flags;
+    uint32_t count, index_off, data_off, size;
+    uint8_t* index;    // count x 16 bytes (PSRAM); null: too big, searched in the file
+  };
+
+  // Where one file of the data is: read `len` bytes from `f` (already at its
+  // start), then done() -- a pack's file stays open for the next read.
+  struct Src { FILE* f = nullptr; uint32_t len = 0; bool owned = false; };
+
+  // Lists the packs again (the map opening, a download finished).
+  void scan() {
+    closeOpen();
+    for (int i = 0; i < _n; i++) free(_p[i].index);
+    _n = 0;
+    DIR* d = opendir(VECTOR_ROOT);
+    if (!d) return;
+    while (struct dirent* e = readdir(d)) {
+      size_t l = strlen(e->d_name);
+      if (_n >= MAX || l < 5 || l >= sizeof(_p[0].file) || strcasecmp(e->d_name + l - 4, ".vpk") != 0) continue;
+      if (open(_p[_n], e->d_name)) _n++;
+    }
+    closedir(d);
+  }
+
+  int count() const { return _n; }
+  const Pack& at(int i) const { return _p[i]; }
+
+  bool find(uint8_t kind, int dz, int x, int y, Src& s) {
+    uint64_t key = keyOf(kind, dz, x, y);
+    for (int i = 0; i < _n; i++) {
+      Pack& p = _p[i];
+      uint32_t off, len;
+      if (!lookup(p, key, off, len)) continue;
+      FILE* f = handle(i);
+      if (!f || fseek(f, p.data_off + off, SEEK_SET) != 0) return false;
+      s.f = f; s.len = len; s.owned = false;
+      return true;
+    }
+    char path[64];   // a loose file
+    snprintf(path, sizeof(path), "%s/%d/%d/%d.%s", VECTOR_ROOT, dz, x, y, kind == K_TILE ? "vt" : "vp");
+    FILE* f = fopen(path, "rb");
+    if (!f) return false;
+    fseek(f, 0, SEEK_END);
+    long sz = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    s.f = f; s.len = sz > 0 ? (uint32_t)sz : 0; s.owned = true;
+    return true;
+  }
+  void done(Src& s) {
+    if (s.owned && s.f) fclose(s.f);
+    s.f = nullptr;
+  }
+
+  // Before a pack file is replaced or deleted.
+  void closeOpen() {
+    if (_open_f) fclose(_open_f);
+    _open_f = nullptr;
+    _open_i = -1;
+  }
+
+private:
+  Pack _p[MAX];
+  int _n = 0;
+  FILE* _open_f = nullptr;   // the pack read last
+  int _open_i = -1;
+  static const uint32_t INDEX_IN_RAM_MAX = 2u * 1024 * 1024;
+
+  static uint64_t keyOf(uint8_t kind, int dz, int x, int y) {
+    return ((uint64_t)kind << 40) | ((uint64_t)(dz & 0xFF) << 32) | ((uint64_t)(x & 0xFFFF) << 16) | (uint64_t)(y & 0xFFFF);
+  }
+  static uint64_t keyAt(const uint8_t* e) {
+    return keyOf(e[0], e[1], e[2] | (e[3] << 8), e[4] | (e[5] << 8));
+  }
+  static uint32_t u32(const uint8_t* b) { return b[0] | (b[1] << 8) | (b[2] << 16) | ((uint32_t)b[3] << 24); }
+
+  FILE* handle(int i) {
+    if (_open_i == i && _open_f) return _open_f;
+    closeOpen();
+    char path[64];
+    snprintf(path, sizeof(path), "%s/%s", VECTOR_ROOT, _p[i].file);
+    _open_f = fopen(path, "rb");
+    _open_i = _open_f ? i : -1;
+    return _open_f;
+  }
+
+  bool open(Pack& p, const char* file) {
+    char path[64];
+    snprintf(path, sizeof(path), "%s/%s", VECTOR_ROOT, file);
+    FILE* f = fopen(path, "rb");
+    if (!f) return false;
+    uint8_t h[64];
+    bool ok = fread(h, 1, 64, f) == 64 && memcmp(h, "VPK1", 4) == 0;
+    if (ok) {
+      snprintf(p.file, sizeof(p.file), "%s", file);
+      p.count = u32(h + 4); p.index_off = u32(h + 8); p.data_off = u32(h + 12);
+      for (int k = 0; k < 4; k++) p.box[k] = (int32_t)u32(h + 16 + 4 * k);
+      p.zmin = h[32]; p.zmax = h[33]; p.flags = h[34];
+      memcpy(p.name, h + 36, 28);
+      p.name[28] = '\0';
+      fseek(f, 0, SEEK_END);
+      p.size = (uint32_t)ftell(f);
+      p.index = nullptr;
+      uint32_t isz = p.count * 16;
+      ok = p.index_off + isz <= p.size && p.data_off <= p.size;
+      if (ok && isz <= INDEX_IN_RAM_MAX) {
+        p.index = psramBuf<uint8_t>(isz ? isz : 1);
+        if (p.index && (fseek(f, p.index_off, SEEK_SET) != 0 || !readFast(f, p.index, isz))) { free(p.index); p.index = nullptr; }
+      }
+    }
+    fclose(f);
+    return ok;
+  }
+
+  // Binary search of the index (in RAM, or read entry by entry).
+  bool lookup(Pack& p, uint64_t key, uint32_t& off, uint32_t& len) {
+    int lo = 0, hi = (int)p.count - 1;
+    uint8_t e[16];
+    FILE* f = nullptr;
+    while (lo <= hi) {
+      int mid = (lo + hi) / 2;
+      const uint8_t* ent;
+      if (p.index) {
+        ent = p.index + 16 * mid;
+      } else {
+        if (!f) f = handle((int)(&p - _p));
+        if (!f || fseek(f, p.index_off + 16 * mid, SEEK_SET) != 0 || fread(e, 1, 16, f) != 16) return false;
+        ent = e;
+      }
+      uint64_t k = keyAt(ent);
+      if (k == key) { off = u32(ent + 8); len = u32(ent + 12); return true; }
+      if (k < key) lo = mid + 1; else hi = mid - 1;
+    }
+    return false;
+  }
+};
+
+static VectorPacks s_vpacks;
+
+}  // namespace mapview

examples/companion_radio/ui-lvgl/map/VectorTileProvider.h

@@ -1,6 +1,6 @@
 #pragma once
-// Vector map (spike): tiles from tools/maps/osm_vector.py under VECTOR_ROOT,
-// {dz}/{x}/{y}.vt (format VT3) at data zooms 10, 12 and 14, drawn here into the 256x256 RGB565
+// Vector map: tiles from tools/maps/osm_vector.py (region packs or loose
+// files, map/VectorPacks.h), format VT3, at data zooms 10, 12 and 14, drawn here into the 256x256 RGB565
 // buffer the map asks for -- polygons by scanline (even-odd), lines as quads
 // with round joins. No anti-aliasing; names of points in VectorLabels.h. Where there is no vector
 // data (or below zoom 10) the raster provider draws instead, so the two mix.
@@ -15,14 +15,14 @@ namespace mapview {
 static inline int32_t vmin(int32_t a, int32_t b) { return a < b ? a : b; }
 static inline int32_t vmax(int32_t a, int32_t b) { return a > b ? a : b; }
 
-static const char* const VECTOR_ROOT = "/sdcard/vmap";
-
 class VectorTileProvider : public TileProvider {
 public:
   explicit VectorTileProvider(TileProvider& fallback) : _fb(fallback) {}
 
   bool available() override {
     struct stat st;
+    s_vpacks.scan();
+    for (Data& d : _slot) d.dz = -1;   // a pack may have changed
     _have = stat(VECTOR_ROOT, &st) == 0 && S_ISDIR(st.st_mode);
     bool fb = _fb.available();
     return _have || fb;
@@ -104,14 +104,10 @@ private:
     d = &_slot[0];
     for (Data& s : _slot) if (s.used < d->used) d = &s;
     d->dz = dz; d->x = x; d->y = y; d->ok = false; d->used = ++_tick;
-    char path[64];
-    snprintf(path, sizeof(path), "%s/%d/%d/%d.vt", VECTOR_ROOT, dz, x, y);
-    FILE* f = fopen(path, "rb");
-    if (!f) return false;
-    fseek(f, 0, SEEK_END);
-    long sz = ftell(f);
-    fseek(f, 0, SEEK_SET);
-    if (sz < 6 || sz > 2 * 1024 * 1024) { fclose(f); return false; }
+    VectorPacks::Src src;
+    if (!s_vpacks.find(VectorPacks::K_TILE, dz, x, y, src)) return false;
+    long sz = src.len;
+    if (sz < 6 || sz > 2 * 1024 * 1024) { s_vpacks.done(src); return false; }
     if ((size_t)sz > d->cap) {
       if (d->buf) free(d->buf);
 #if defined(ESP32)
@@ -120,10 +116,10 @@ private:
       d->buf = (uint8_t*)malloc(sz);
 #endif
       d->cap = d->buf ? sz : 0;
-      if (!d->buf) { fclose(f); return false; }
+      if (!d->buf) { s_vpacks.done(src); return false; }
     }
-    d->len = readFast(f, d->buf, sz) ? sz : 0;
-    fclose(f);
+    d->len = readFast(src.f, d->buf, sz) ? sz : 0;
+    s_vpacks.done(src);
     d->ok = d->len == (size_t)sz && memcmp(d->buf, "VT3", 3) == 0;
     _data = d->buf;
     _data_len = d->len;

tools/maps/osm_vector.py

@@ -8,6 +8,13 @@ buildings; and named points for the labels (places, peaks, huts, springs...).
 No street names (the raster map has them).
 
   tools/maps/osm_vector.py area.json [points.json] --out vmap/ [--dem dem-cache/]
+  tools/maps/osm_vector.py area.json ... --pack tatry.vpk --name "Tatry"
+  tools/maps/osm_vector.py --pbf malopolskie-latest.osm.pbf --bbox 49.15,19.75,49.35,20.25 \
+      --dem dem-cache/ --pack tatry.vpk --name "Tatry"
+
+--pbf reads a Geofabrik extract (tools/maps/pbf.py, needs pyosmium: pip
+install osmium) for the --bbox (south,west,north,east) -- no Overpass limits;
+give it more than once for a box across a border.
 
 With --dem, contour lines too (tools/maps/dem.py: Terrain Tiles fetched into
 that folder): every 20 m (from zoom 15) with a darker one every 100 m.
@@ -16,6 +23,17 @@ Three data zooms: 10 (drawn at z10-11), 12 (z12-13), 14 (z14-18); the device pic
 data tile covering the tile it draws and scales it. Copy the output folder to
 the card as /sdcard/vmap.
 
+A pack (--pack) holds a whole region in one file, which the device reads in
+place from /sdcard/vmap/*.vpk (a card writes one big file much faster than
+thousands of small ones, and a download is one file):
+  header, 64 bytes: 'VPK1' count:u32 index_off:u32 data_off:u32
+    lon0 lat0 lon1 lat1: i32 (degrees x 1e6)  zmin zmax flags(1 contours) 0: u8
+    name: 28 bytes UTF-8, 0-padded
+  count x index entry, 16 bytes, sorted: kind:u8 (0 tile, 1 points) dz:u8
+    x:u16 y:u16 0:u16 off:u32 (from data_off) len:u32
+  the files, one after another.
+Without --pack the files go to --out as {dz}/{x}/{y}.vt / .vp.
+
 Tile format 'VT3' (little-endian):
   'V' 'T' '3' dz:u8  count:u16
   count x feature, in drawing order:
@@ -395,15 +413,36 @@ def clip_ring(pts, lo, hi):
 
 def main():
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument('json', nargs='+')
+    ap.add_argument('json', nargs='*', help='Overpass JSON file(s)')
+    ap.add_argument('--pbf', action='append', default=[], help='an OSM PBF extract (with --bbox); again for more')
+    ap.add_argument('--bbox', help='south,west,north,east: only this box (and the pack\'s box)')
     ap.add_argument('--out', default='vmap')
+    ap.add_argument('--pack', metavar='FILE.vpk', help='write one pack file instead of a folder')
+    ap.add_argument('--name', default='', help="the region's name in the pack")
+    ap.add_argument('--lang', help='names in this language where OSM has them (name:pl ...)')
     ap.add_argument('--dem', metavar='CACHE', help='add contour lines; DEM tiles are kept in this folder')
     ap.add_argument('--contour-step', type=int, default=20)
     ap.add_argument('--index-step', type=int, default=100)
     a = ap.parse_args()
+    box = None
+    if a.bbox:
+        s_, w_, n_, e_ = (float(v) for v in a.bbox.split(','))
+        box = (w_, s_, e_, n_)
     elements = []
     for fn in a.json:
         elements += json.load(open(fn))['elements']
+    if a.pbf:
+        if not box:
+            ap.error('--pbf needs --bbox')
+        import pbf
+        for fn in a.pbf:   # neighbouring extracts: what both have is drawn twice, harmless
+            elements += pbf.read_pbf(fn, box)
+    if not elements:
+        ap.error('no input')
+    if not box:
+        lons = [p['lon'] for e in elements for p in (e.get('geometry') or []) if p] or [0]
+        lats = [p['lat'] for e in elements for p in (e.get('geometry') or []) if p] or [0]
+        box = (min(lons), min(lats), max(lons), max(lats))
 
     feats = []   # (cls, colour, 'poly'|'line', [parts in world coords])
     way_routes = defaultdict(set)   # way id -> route colours along it
@@ -414,7 +453,12 @@ def main():
         pt = point_class(t)
         if pt is not None:
             pos = (e['lon'], e['lat']) if 'lat' in e else (e['center']['lon'], e['center']['lat']) if 'center' in e else None
-            name = t.get('name', '')
+            if not pos and e.get('geometry'):   # a hut drawn as a building: its middle
+                g = [p for p in e['geometry'] if p]
+                pos = (sum(p['lon'] for p in g) / len(g), sum(p['lat'] for p in g) / len(g))
+            if pos and not (box[0] <= pos[0] <= box[2] and box[1] <= pos[1] <= box[3]):
+                pos = None
+            name = (a.lang and t.get('name:' + a.lang)) or t.get('name', '')
             ele = elevation(t)
             if pos and (name or (pt == T_PEAK and ele is not None)):   # unnamed: just peaks, by their height
                 x, y = world(*pos)
@@ -457,19 +501,19 @@ def main():
         packed = sum(c << (4 * i) for i, c in enumerate(cols))
         feats.append((L_ROUTES, packed, 'line', join_lines(ways)))
     if a.dem:
-        lons = [p['lon'] for e in elements for p in (e.get('geometry') or []) if p]
-        lats = [p['lat'] for e in elements for p in (e.get('geometry') or []) if p]
         nc = 0
-        for v, line in dem.contour_lines(a.dem, min(lons), min(lats), max(lons), max(lats), a.contour_step):
+        for v, line in dem.contour_lines(a.dem, *box, a.contour_step):
             feats.append((L_CONTOUR_IDX if round(v) % a.index_step == 0 else L_CONTOUR, 0, 'line', [line]))
             nc += 1
         print(f'{nc} contour lines', file=sys.stderr)
     print(f'{len(feats)} features, {len(way_routes)} route ways', file=sys.stderr)
 
-    total_bytes = total_tiles = 0
+    files = {}   # (kind 0 tile / 1 points, dz, x, y) -> bytes
     for dz in DATA_ZOOMS:
         n = 1 << dz
         base_tol = 1.0 if dz == DATA_ZOOMS[-1] else 4.0   # half a pixel at the deepest zoom drawn from it
+        (wx0, wy0), (wx1, wy1) = world(box[0], box[3]), world(box[2], box[1])
+        tx_lo, ty_lo, tx_hi, ty_hi = int(wx0 * n), int(wy0 * n), int(wx1 * n), int(wy1 * n)   # tiles of the box only
         tiles = defaultdict(list)
         for cls, col, kind, parts in feats:
             if MIN_DZ.get(cls, 0) > dz:
@@ -480,8 +524,8 @@ def main():
             allp = [p for part in parts for p in part]
             if not allp:
                 continue
-            x0 = int(min(p[0] for p in allp) * n); x1 = int(max(p[0] for p in allp) * n)
-            y0 = int(min(p[1] for p in allp) * n); y1 = int(max(p[1] for p in allp) * n)
+            x0 = max(int(min(p[0] for p in allp) * n), tx_lo); x1 = min(int(max(p[0] for p in allp) * n), tx_hi)
+            y0 = max(int(min(p[1] for p in allp) * n), ty_lo); y1 = min(int(max(p[1] for p in allp) * n), ty_hi)
             for tx in range(x0, x1 + 1):
                 for ty in range(y0, y1 + 1):
                     out = []
@@ -529,12 +573,7 @@ def main():
                     buf += varint(len(body)) + body
                     count += 1
             struct.pack_into('<H', buf, 4, min(count, 65535))
-            d = os.path.join(a.out, str(dz), str(tx))
-            os.makedirs(d, exist_ok=True)
-            with open(os.path.join(d, f'{ty}.vt'), 'wb') as f:
-                f.write(buf)
-            total_bytes += len(buf)
-            total_tiles += 1
+            files[(0, dz, tx, ty)] = bytes(buf)
     for dz in DATA_ZOOMS:
         n = 1 << dz
         tiles = defaultdict(list)
@@ -549,17 +588,45 @@ def main():
             buf = bytearray(b'VP1' + bytes([dz]) + struct.pack('<H', len(pl)))
             for pt, _, px, py, ele, name in pl:
                 buf += struct.pack('<BHHhB', pt, px, py, -32768 if ele is None else ele, len(name)) + name
-            d = os.path.join(a.out, str(dz), str(tx))
-            os.makedirs(d, exist_ok=True)
-            with open(os.path.join(d, f'{ty}.vp'), 'wb') as f:
-                f.write(buf)
-            total_bytes += len(buf)
+            files[(1, dz, tx, ty)] = bytes(buf)
     print(f'{len(points)} points', file=sys.stderr)
-    with open(os.path.join(a.out, 'attribution.txt'), 'w') as f:
-        f.write('© OpenStreetMap contributors (ODbL)\n')
-        if a.dem:
-            f.write(dem.ATTRIBUTION + '\n')
-    print(f'{total_tiles} tiles, {total_bytes / 1024:.0f} KB', file=sys.stderr)
+    total = sum(len(b) for b in files.values())
+    if a.pack:
+        name = a.name or os.path.splitext(os.path.basename(a.pack))[0]
+        write_pack(a.pack, files, box, name, bool(a.dem))
+    else:
+        for (kind, dz, x, y), b in files.items():
+            d = os.path.join(a.out, str(dz), str(x))
+            os.makedirs(d, exist_ok=True)
+            with open(os.path.join(d, f'{y}.{"vp" if kind else "vt"}'), 'wb') as f:
+                f.write(b)
+        with open(os.path.join(a.out, 'attribution.txt'), 'w') as f:
+            f.write('© OpenStreetMap contributors (ODbL)\n')
+            if a.dem:
+                f.write(dem.ATTRIBUTION + '\n')
+    print(f'{sum(1 for k in files if k[0] == 0)} tiles, {total / 1024:.0f} KB', file=sys.stderr)
+
+
+def write_pack(path, files, box, name, contours):
+    keys = sorted(files)
+    index = bytearray()
+    off = 0
+    for k in keys:
+        index += struct.pack('<BBHHHII', k[0], k[1], k[2], k[3], 0, off, len(files[k]))
+        off += len(files[k])
+    nb = name.encode('utf-8')[:27]
+    while nb and (nb[-1] & 0xC0) == 0x80:
+        nb = nb[:-1]
+    zs = [k[1] for k in keys if k[0] == 0] or [0]
+    head = struct.pack('<4sIII4iBBBB28s', b'VPK1', len(keys), 64, 64 + len(index),
+                       *(int(round(v * 1e6)) for v in box), min(zs), max(zs), 1 if contours else 0, 0, nb)
+    assert len(head) == 64
+    with open(path + '.tmp', 'wb') as f:
+        f.write(head)
+        f.write(index)
+        for k in keys:
+            f.write(files[k])
+    os.replace(path + '.tmp', path)
 
 
 if __name__ == '__main__':

tools/maps/pbf.py

@@ -0,0 +1,65 @@
+"""OSM PBF extract -> the elements osm_vector.py works on (as Overpass gives
+them with `out geom;`), for a box. Needs pyosmium (pip install osmium).
+
+Two passes over the file: relations first (hiking routes, multipolygons: which
+ways they need), then nodes and ways with their node locations. A way is kept
+if it touches the box; a relation if one of its ways was kept.
+"""
+import sys
+
+WAY_KEYS = ('highway', 'waterway', 'natural', 'landuse', 'building', 'tourism')
+AREA_KEYS = ('natural', 'landuse', 'building', 'water')
+NODE_KEYS = ('natural', 'tourism', 'amenity', 'place')
+
+
+def read_pbf(path, box):
+    """box: (lon0, lat0, lon1, lat1). Returns a list of element dicts."""
+    import osmium
+    lon0, lat0, lon1, lat1 = box
+
+    rels, want = [], {}   # relations kept; way id -> [(relation index, role)]
+    for r in osmium.FileProcessor(path, osmium.osm.RELATION):
+        t = dict(r.tags)
+        if not (t.get('route') == 'hiking' or (t.get('type') == 'multipolygon' and any(k in t for k in AREA_KEYS))):
+            continue
+        i = len(rels)
+        rels.append({'type': 'relation', 'id': r.id, 'tags': t, 'members': []})
+        for m in r.members:
+            if m.type == 'w':
+                want.setdefault(m.ref, []).append((i, m.role))
+    print(f'PBF: {len(rels)} relations', file=sys.stderr)
+
+    elements, geoms = [], {}
+    # Untagged nodes (most of them) never reach Python; ways do, untagged ones
+    # can be a multipolygon's outline.
+    fp = osmium.FileProcessor(path, osmium.osm.NODE | osmium.osm.WAY).with_locations() \
+        .with_filter(osmium.filter.EmptyTagFilter().enable_for(osmium.osm.NODE))
+    for o in fp:
+        if o.is_node():
+            if not any(k in o.tags for k in NODE_KEYS):
+                continue
+            loc = o.location
+            if loc.valid() and lon0 <= loc.lon <= lon1 and lat0 <= loc.lat <= lat1:
+                elements.append({'type': 'node', 'id': o.id, 'lat': loc.lat, 'lon': loc.lon, 'tags': dict(o.tags)})
+            continue
+        in_rel = o.id in want
+        tagged = any(k in o.tags for k in WAY_KEYS)
+        if not in_rel and not tagged:
+            continue
+        g = [{'lat': n.lat, 'lon': n.lon} for n in o.nodes if n.location.valid()]
+        if len(g) < 2:
+            continue
+        if (max(p['lon'] for p in g) < lon0 or min(p['lon'] for p in g) > lon1 or
+                max(p['lat'] for p in g) < lat0 or min(p['lat'] for p in g) > lat1):
+            continue
+        if in_rel:
+            geoms[o.id] = g
+        if tagged:
+            elements.append({'type': 'way', 'id': o.id, 'tags': dict(o.tags), 'geometry': g})
+    for wid, uses in want.items():
+        if wid in geoms:
+            for i, role in uses:
+                rels[i]['members'].append({'type': 'way', 'ref': wid, 'role': role, 'geometry': geoms[wid]})
+    kept = [r for r in rels if r['members']]
+    print(f'PBF: {len(elements)} nodes / ways, {len(kept)} relations in the box', file=sys.stderr)
+    return elements + kept