Complete rewrite: PapaParse + leaflet.glify + RBush + Web Worker approach

This commit is contained in:
Iliyas Kyrykbayev 2026-05-26 18:17:33 +05:00
parent 48e63b12da
commit 89e5dd9e6d
5 changed files with 76142 additions and 613883 deletions

11
.gitignore vendored Normal file
View File

@ -0,0 +1,11 @@
# Generated data files (no longer needed with the new approach)
data/regions/
data/tiles/
data/region_index.json
data/tile_index.json
data/hulls.json
data/speedtest_providers_avg_speeds_by_location — копия.csv
# Python
__pycache__/
*.pyc

View File

@ -1,76 +0,0 @@
provider,latitude,longitude,avg_download_mbps,avg_upload_mbps
Казахтелеком,43.2561,76.9285,87.4,42.1
Казахтелеком,43.2350,76.9100,91.2,44.7
Казахтелеком,43.2700,76.9500,78.5,38.2
Казахтелеком,43.2200,76.8900,95.0,46.0
Казахтелеком,43.2900,76.9700,82.3,40.5
Казахтелеком,43.2100,76.8800,88.1,43.0
Казахтелеком,43.2450,76.9150,76.4,36.8
Казахтелеком,43.2600,76.9350,93.7,45.2
Казахтелеком,43.2750,76.9550,79.9,39.1
Казахтелеком,43.2250,76.8950,85.6,41.3
Казахтелеком,51.1800,71.4450,72.5,35.0
Казахтелеком,51.1600,71.4250,68.3,32.1
Казахтелеком,51.2000,71.4650,74.1,36.2
Казахтелеком,51.1400,71.4050,70.8,34.5
Казахтелеком,51.2200,71.4850,76.2,37.8
Казахтелеком,49.8000,73.1200,65.4,31.0
Казахтелеком,49.8200,73.1400,62.1,29.8
Казахтелеком,49.7800,73.1000,67.8,32.5
Казахтелеком,49.8400,73.1600,63.5,30.2
Казахтелеком,49.7600,73.0800,69.2,33.1
Beeline,43.2561,76.9285,120.5,58.3
Beeline,43.2350,76.9100,115.2,55.1
Beeline,43.2700,76.9500,130.8,62.4
Beeline,43.2200,76.8900,108.6,52.0
Beeline,43.2900,76.9700,125.3,60.1
Beeline,43.2100,76.8800,112.4,54.7
Beeline,43.2450,76.9150,118.9,57.2
Beeline,43.2600,76.9350,135.1,64.8
Beeline,43.2750,76.9550,122.7,59.3
Beeline,43.2250,76.8950,110.3,53.5
Beeline,51.1800,71.4450,95.2,45.0
Beeline,51.1600,71.4250,88.7,42.3
Beeline,51.2000,71.4650,92.4,44.1
Beeline,51.1400,71.4050,85.6,40.8
Beeline,51.2200,71.4850,98.1,46.5
Beeline,49.8000,73.1200,78.3,37.0
Beeline,49.8200,73.1400,82.5,39.2
Beeline,49.7800,73.1000,75.1,35.8
Beeline,49.8400,73.1600,80.4,38.5
Beeline,49.7600,73.0800,76.8,36.4
Tele2,43.2561,76.9285,45.2,21.0
Tele2,43.2350,76.9100,42.8,19.5
Tele2,43.2700,76.9500,48.1,22.3
Tele2,43.2200,76.8900,40.5,18.8
Tele2,43.2900,76.9700,46.7,21.8
Tele2,43.2100,76.8800,43.4,20.1
Tele2,43.2450,76.9150,44.9,20.8
Tele2,43.2600,76.9350,49.2,22.9
Tele2,43.2750,76.9550,41.6,19.2
Tele2,43.2250,76.8950,47.3,22.0
Tele2,51.1800,71.4450,38.5,17.2
Tele2,51.1600,71.4250,35.2,16.0
Tele2,51.2000,71.4650,40.1,18.5
Tele2,51.1400,71.4050,36.8,16.8
Tele2,51.2200,71.4850,39.4,18.1
Kcell,43.2561,76.9285,65.3,31.2
Kcell,43.2350,76.9100,62.1,29.8
Kcell,43.2700,76.9500,68.5,32.7
Kcell,43.2200,76.8900,60.4,28.5
Kcell,43.2900,76.9700,66.8,31.9
Kcell,43.2100,76.8800,63.2,30.1
Kcell,43.2450,76.9150,64.7,30.9
Kcell,43.2600,76.9350,70.1,33.5
Kcell,43.2750,76.9550,61.5,29.2
Kcell,43.2250,76.8950,67.2,32.0
Altel,43.2561,76.9285,55.8,26.4
Altel,43.2350,76.9100,52.3,24.8
Altel,43.2700,76.9500,58.1,27.5
Altel,43.2200,76.8900,50.6,23.9
Altel,43.2900,76.9700,56.4,26.8
Altel,43.2100,76.8800,53.7,25.2
Altel,43.2450,76.9150,54.9,26.0
Altel,43.2600,76.9350,59.2,28.1
Altel,43.2750,76.9550,51.8,24.5
Altel,43.2250,76.8950,57.1,27.0
1 provider latitude longitude avg_download_mbps avg_upload_mbps
2 Казахтелеком 43.2561 76.9285 87.4 42.1
3 Казахтелеком 43.2350 76.9100 91.2 44.7
4 Казахтелеком 43.2700 76.9500 78.5 38.2
5 Казахтелеком 43.2200 76.8900 95.0 46.0
6 Казахтелеком 43.2900 76.9700 82.3 40.5
7 Казахтелеком 43.2100 76.8800 88.1 43.0
8 Казахтелеком 43.2450 76.9150 76.4 36.8
9 Казахтелеком 43.2600 76.9350 93.7 45.2
10 Казахтелеком 43.2750 76.9550 79.9 39.1
11 Казахтелеком 43.2250 76.8950 85.6 41.3
12 Казахтелеком 51.1800 71.4450 72.5 35.0
13 Казахтелеком 51.1600 71.4250 68.3 32.1
14 Казахтелеком 51.2000 71.4650 74.1 36.2
15 Казахтелеком 51.1400 71.4050 70.8 34.5
16 Казахтелеком 51.2200 71.4850 76.2 37.8
17 Казахтелеком 49.8000 73.1200 65.4 31.0
18 Казахтелеком 49.8200 73.1400 62.1 29.8
19 Казахтелеком 49.7800 73.1000 67.8 32.5
20 Казахтелеком 49.8400 73.1600 63.5 30.2
21 Казахтелеком 49.7600 73.0800 69.2 33.1
22 Beeline 43.2561 76.9285 120.5 58.3
23 Beeline 43.2350 76.9100 115.2 55.1
24 Beeline 43.2700 76.9500 130.8 62.4
25 Beeline 43.2200 76.8900 108.6 52.0
26 Beeline 43.2900 76.9700 125.3 60.1
27 Beeline 43.2100 76.8800 112.4 54.7
28 Beeline 43.2450 76.9150 118.9 57.2
29 Beeline 43.2600 76.9350 135.1 64.8
30 Beeline 43.2750 76.9550 122.7 59.3
31 Beeline 43.2250 76.8950 110.3 53.5
32 Beeline 51.1800 71.4450 95.2 45.0
33 Beeline 51.1600 71.4250 88.7 42.3
34 Beeline 51.2000 71.4650 92.4 44.1
35 Beeline 51.1400 71.4050 85.6 40.8
36 Beeline 51.2200 71.4850 98.1 46.5
37 Beeline 49.8000 73.1200 78.3 37.0
38 Beeline 49.8200 73.1400 82.5 39.2
39 Beeline 49.7800 73.1000 75.1 35.8
40 Beeline 49.8400 73.1600 80.4 38.5
41 Beeline 49.7600 73.0800 76.8 36.4
42 Tele2 43.2561 76.9285 45.2 21.0
43 Tele2 43.2350 76.9100 42.8 19.5
44 Tele2 43.2700 76.9500 48.1 22.3
45 Tele2 43.2200 76.8900 40.5 18.8
46 Tele2 43.2900 76.9700 46.7 21.8
47 Tele2 43.2100 76.8800 43.4 20.1
48 Tele2 43.2450 76.9150 44.9 20.8
49 Tele2 43.2600 76.9350 49.2 22.9
50 Tele2 43.2750 76.9550 41.6 19.2
51 Tele2 43.2250 76.8950 47.3 22.0
52 Tele2 51.1800 71.4450 38.5 17.2
53 Tele2 51.1600 71.4250 35.2 16.0
54 Tele2 51.2000 71.4650 40.1 18.5
55 Tele2 51.1400 71.4050 36.8 16.8
56 Tele2 51.2200 71.4850 39.4 18.1
57 Kcell 43.2561 76.9285 65.3 31.2
58 Kcell 43.2350 76.9100 62.1 29.8
59 Kcell 43.2700 76.9500 68.5 32.7
60 Kcell 43.2200 76.8900 60.4 28.5
61 Kcell 43.2900 76.9700 66.8 31.9
62 Kcell 43.2100 76.8800 63.2 30.1
63 Kcell 43.2450 76.9150 64.7 30.9
64 Kcell 43.2600 76.9350 70.1 33.5
65 Kcell 43.2750 76.9550 61.5 29.2
66 Kcell 43.2250 76.8950 67.2 32.0
67 Altel 43.2561 76.9285 55.8 26.4
68 Altel 43.2350 76.9100 52.3 24.8
69 Altel 43.2700 76.9500 58.1 27.5
70 Altel 43.2200 76.8900 50.6 23.9
71 Altel 43.2900 76.9700 56.4 26.8
72 Altel 43.2100 76.8800 53.7 25.2
73 Altel 43.2450 76.9150 54.9 26.0
74 Altel 43.2600 76.9350 59.2 28.1
75 Altel 43.2750 76.9550 51.8 24.5
76 Altel 43.2250 76.8950 57.1 27.0

1764
index.html

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

644
tools/tile-csv.py Normal file
View File

@ -0,0 +1,644 @@
#!/usr/bin/env python3
"""
tile-csv.py — Pre-process CSV into tiles + per-region data files + LOD files.
Output:
data/tiles/<key>.json — Flat array per tile
data/tile_index.json — Tile list + provider metadata
data/hulls.json — Pre-computed convex hulls per provider
data/regions/<slug>.json — Full detail flat array per region
data/regions/<slug>_lod0.json — ~200 aggregated points (zoom <= 10)
data/regions/<slug>_lod1.json — ~2000 aggregated points (zoom 11-12)
data/regions/<slug>_lod2.json — Full detail (zoom >= 13)
data/region_index.json — Region metadata with LOD info
Usage:
python tools/tile-csv.py
"""
import csv
import json
import math
import os
import sys
from collections import defaultdict
from shapely.geometry import shape, Point
# === CONFIG ===
CSV_PATH = 'speedtest_providers_avg_speeds_by_location.csv'
DELIMITER = ';'
TILE_SIZE_DEG = 0.5 # 0.5° × 0.5° tiles (~55km at equator)
OUT_DIR = 'data/tiles'
INDEX_PATH = 'data/tile_index.json'
HULLS_PATH = 'data/hulls.json'
REGION_GEOJSON_PATH = 'data/kaz_regions.geojson'
REGION_OUT_DIR = 'data/regions'
REGION_INDEX_PATH = 'data/region_index.json'
EPS_KM = 20
MIN_CLUSTER_POINTS = 3
DEFAULT_PROVIDERS = ['Kazakhtelecom', 'Beeline', 'Alma Plus', 'Transtelecom', 'NLS']
# CSV field names (from the file header)
FIELD_PROVIDER = 'provider'
FIELD_LATITUDE = 'latitude'
FIELD_LONGITUDE = 'longitude'
FIELD_DOWNLOAD = 'avg_download_speed'
FIELD_UPLOAD = 'avg_upload_speed'
def get_tile_key(lat, lng):
"""Return tile key string for given lat/lng."""
gi = int(math.floor(lat / TILE_SIZE_DEG))
gj = int(math.floor(lng / TILE_SIZE_DEG))
return f"{gi}_{gj}"
def convex_hull(points):
"""
Compute convex hull of a list of (lng, lat) pairs.
Returns list of [lng, lat] pairs forming the hull, or None if < 3 points.
"""
pts = sorted(points, key=lambda p: (p[0], p[1]))
if len(pts) < 3:
return None
def cross(o, a, b):
return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0])
lower = []
for p in pts:
while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0:
lower.pop()
lower.append(p)
upper = []
for p in reversed(pts):
while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0:
upper.pop()
upper.append(p)
lower.pop()
upper.pop()
hull = lower + upper
if len(hull) < 3:
return None
# Check collinear
is_collinear = True
for i in range(2, len(hull)):
if cross(hull[0], hull[1], hull[i]) != 0:
is_collinear = False
break
if is_collinear:
return None
# Close the ring
hull.append(hull[0])
return hull
def chaikin_smooth(ring, iterations=2):
"""Smooth a polygon ring using Chaikin's algorithm."""
if not ring or len(ring) < 4:
return ring
pts = ring[:-1] # remove closing point
for _ in range(iterations):
new_pts = []
for i in range(len(pts)):
p0 = pts[i]
p1 = pts[(i + 1) % len(pts)]
new_pts.append([0.75 * p0[0] + 0.25 * p1[0], 0.75 * p0[1] + 0.25 * p1[1]])
new_pts.append([0.25 * p0[0] + 0.75 * p1[0], 0.25 * p0[1] + 0.75 * p1[1]])
pts = new_pts
closed = pts + [pts[0]]
return closed
def aggregate_lod(flat_arr, count, stride, max_points):
"""
Aggregate points into a smaller set by binning into a grid.
flat_arr: list of [lat, lng, provId, dl, ul, ...]
count: number of points
stride: values per point (5)
max_points: target maximum number of output points
Returns a new flat list with averaged points.
"""
if count <= max_points:
return list(flat_arr)
# Find bounds
lats = [flat_arr[i * stride] for i in range(count)]
lngs = [flat_arr[i * stride + 1] for i in range(count)]
min_lat, max_lat = min(lats), max(lats)
min_lng, max_lng = min(lngs), max(lngs)
lat_range = max_lat - min_lat
lng_range = max_lng - min_lng
if lat_range < 1e-6:
lat_range = 1.0
if lng_range < 1e-6:
lng_range = 1.0
# Count unique providers
provider_set = set()
for i in range(count):
provider_set.add(int(flat_arr[i * stride + 2]))
num_providers = len(provider_set)
# We want roughly max_points output points.
# Each grid cell can have up to num_providers points (one per provider).
# But in practice, most cells have far fewer providers.
# Use a correction factor: assume ~30% of cells have multiple providers
avg_providers_per_cell = max(1, int(num_providers * 0.3))
target_cells = max(1, max_points // avg_providers_per_cell)
# Grid dimensions: maintain aspect ratio of the bounding box
aspect = lng_range / max(lat_range, 1e-6)
grid_rows = max(1, int(math.sqrt(target_cells / max(aspect, 0.01))))
grid_cols = max(1, int(grid_rows * aspect))
# Cap to avoid excessive memory
grid_rows = min(grid_rows, 500)
grid_cols = min(grid_cols, 500)
# Bin points into grid cells
bins = {} # (row, col) -> {provId: [count, sum_lat, sum_lng, sum_dl, sum_ul]}
for i in range(count):
off = i * stride
lat = flat_arr[off]
lng = flat_arr[off + 1]
pid = int(flat_arr[off + 2])
dl = flat_arr[off + 3]
ul = flat_arr[off + 4]
col = int((lng - min_lng) / lng_range * grid_cols)
row = int((lat - min_lat) / lat_range * grid_rows)
col = min(col, grid_cols - 1)
row = min(row, grid_rows - 1)
key = (row, col)
if key not in bins:
bins[key] = {}
if pid not in bins[key]:
bins[key][pid] = [0, 0.0, 0.0, 0.0, 0.0]
b = bins[key][pid]
b[0] += 1
b[1] += lat
b[2] += lng
b[3] += dl
b[4] += ul
# Flatten bins into output array
result = []
for cell_bins in bins.values():
for pid, b in cell_bins.items():
cnt = b[0]
result.append(b[1] / cnt)
result.append(b[2] / cnt)
result.append(float(pid))
result.append(b[3] / cnt)
result.append(b[4] / cnt)
return result
def main():
print(f"Reading CSV: {CSV_PATH}")
print(f"Tile size: {TILE_SIZE_DEG} deg x {TILE_SIZE_DEG} deg")
# Ensure output directory exists
os.makedirs(OUT_DIR, exist_ok=True)
# Phase 1: Read CSV and group into tiles
# We'll store points per tile as flat arrays: [lat, lng, provId, dl, ul, ...]
tile_data = {} # key -> list of numbers
tile_counts = {} # key -> point count
provider_name_to_id = {}
provider_id_to_name = []
provider_counts = defaultdict(int)
total_rows = 0
skipped_rows = 0
with open(CSV_PATH, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f, delimiter=DELIMITER)
for row in reader:
total_rows += 1
# Extract fields
prov_name = row.get(FIELD_PROVIDER, '').strip()
if not prov_name:
skipped_rows += 1
continue
try:
lat = float(row.get(FIELD_LATITUDE, ''))
lng = float(row.get(FIELD_LONGITUDE, ''))
except (ValueError, TypeError):
skipped_rows += 1
continue
if math.isnan(lat) or math.isnan(lng):
skipped_rows += 1
continue
try:
dl = float(row.get(FIELD_DOWNLOAD, 0) or 0)
except (ValueError, TypeError):
dl = 0.0
try:
ul = float(row.get(FIELD_UPLOAD, 0) or 0)
except (ValueError, TypeError):
ul = 0.0
# Assign provider ID
if prov_name not in provider_name_to_id:
pid = len(provider_id_to_name)
provider_name_to_id[prov_name] = pid
provider_id_to_name.append(prov_name)
else:
pid = provider_name_to_id[prov_name]
provider_counts[prov_name] += 1
# Determine tile
key = get_tile_key(lat, lng)
if key not in tile_data:
tile_data[key] = []
tile_counts[key] = 0
tile_data[key].extend([lat, lng, pid, dl, ul])
tile_counts[key] = tile_counts.get(key, 0) + 1
if total_rows % 100000 == 0:
print(f" Processed {total_rows:,} rows...")
print(f"\nTotal rows: {total_rows:,}")
print(f"Skipped: {skipped_rows:,}")
print(f"Providers: {len(provider_id_to_name)}")
print(f"Tiles: {len(tile_data)}")
# Filter providers with < 3 points
valid_providers = set()
for name, count in provider_counts.items():
if count >= MIN_CLUSTER_POINTS:
valid_providers.add(name)
print(f"Valid providers (>= {MIN_CLUSTER_POINTS} points): {len(valid_providers)}")
# Phase 2: Write tile files
print("\nWriting tile files...")
tile_list = []
total_points_written = 0
for key, flat_arr in tile_data.items():
count = tile_counts[key]
# Filter out points from invalid providers
filtered = []
filtered_count = 0
for i in range(count):
off = i * 5
pid = int(flat_arr[off + 2])
pname = provider_id_to_name[pid]
if pname in valid_providers:
filtered.extend(flat_arr[off:off + 5])
filtered_count += 1
if filtered_count == 0:
continue
tile_path = os.path.join(OUT_DIR, f"{key}.json")
with open(tile_path, 'w', encoding='utf-8') as tf:
json.dump(filtered, tf, separators=(',', ':'))
tile_list.append({
'key': key,
'count': filtered_count,
'file': f"tiles/{key}.json"
})
total_points_written += filtered_count
if len(tile_list) % 100 == 0:
print(f" Written {len(tile_list)} tiles...")
print(f"Total tiles written: {len(tile_list)}")
print(f"Total points written: {total_points_written:,}")
# Phase 3: Build provider list for tile_index.json
print("\nBuilding provider list...")
provider_list = []
default_set = set(p.lower() for p in DEFAULT_PROVIDERS)
for pid, name in enumerate(provider_id_to_name):
if name not in valid_providers:
continue
color_idx = pid % 20 # matches CONFIG.PALETTE
provider_list.append({
'id': pid,
'name': name,
'count': provider_counts[name],
'colorIdx': color_idx,
'isDefault': name.lower() in default_set
})
# Phase 4: Write tile_index.json
index_data = {
'tileSizeDeg': TILE_SIZE_DEG,
'pointStride': 5,
'providers': provider_list,
'tiles': tile_list
}
with open(INDEX_PATH, 'w', encoding='utf-8') as f:
json.dump(index_data, f, separators=(',', ':'))
print(f"\nWritten: {INDEX_PATH}")
# Phase 5: Pre-compute convex hulls per provider
print("\nComputing convex hulls...")
hulls = []
# Group points by provider (from all tiles)
provider_points = defaultdict(list)
for key, flat_arr in tile_data.items():
count = tile_counts[key]
for i in range(count):
off = i * 5
pid = int(flat_arr[off + 2])
pname = provider_id_to_name[pid]
if pname not in valid_providers:
continue
lng = flat_arr[off + 1]
lat = flat_arr[off + 0]
dl = flat_arr[off + 3]
ul = flat_arr[off + 4]
provider_points[pid].append((lng, lat, dl, ul))
for pid, pts in provider_points.items():
if len(pts) < MIN_CLUSTER_POINTS:
continue
pname = provider_id_to_name[pid]
coords = [(p[0], p[1]) for p in pts] # (lng, lat)
hull = convex_hull(coords)
if hull is None:
continue
hull = chaikin_smooth(hull, 2)
if hull is None:
continue
sum_dl = sum(p[2] for p in pts)
sum_ul = sum(p[3] for p in pts)
avg_dl = sum_dl / len(pts)
avg_ul = sum_ul / len(pts)
feature = {
'type': 'Feature',
'geometry': {
'type': 'Polygon',
'coordinates': [hull]
},
'properties': {
'provider': pname,
'providerId': pid,
'count': len(pts),
'avg_download': round(avg_dl, 1),
'avg_upload': round(avg_ul, 1)
}
}
hulls.append(feature)
hulls_geojson = {
'type': 'FeatureCollection',
'features': hulls
}
with open(HULLS_PATH, 'w', encoding='utf-8') as f:
json.dump(hulls_geojson, f, separators=(',', ':'))
print(f"Written: {HULLS_PATH}")
print(f"Hulls: {len(hulls)}")
# Clean up tile_data to free memory
tile_data.clear()
provider_points.clear()
# Print tile size stats
tile_sizes = [t['count'] for t in tile_list]
if tile_sizes:
print(f"\n--- Tile Stats ---")
print(f"Min points/tile: {min(tile_sizes)}")
print(f"Max points/tile: {max(tile_sizes)}")
print(f"Avg points/tile: {sum(tile_sizes) / len(tile_sizes):.1f}")
print(f"Median points/tile: {sorted(tile_sizes)[len(tile_sizes) // 2]}")
# ============================================================
# Phase 6: Map points to administrative regions
# ============================================================
print("\n--- Phase 6: Mapping points to regions ---")
# Load region boundaries
print(f"Loading region boundaries: {REGION_GEOJSON_PATH}")
with open(REGION_GEOJSON_PATH, 'r', encoding='utf-8') as f:
regions_geojson = json.load(f)
# Build region polygon list with spatial indexing
region_list = [] # [(slug, name, shapely_polygon, bounds)]
for feature in regions_geojson['features']:
props = feature['properties']
name = props.get('NAME_1', 'Unknown')
slug = name.lower().replace(' ', '_').replace('-', '_').replace("'", '')
poly = shape(feature['geometry'])
# Compute bounding box for quick pre-filter
bounds = poly.bounds # (minx, miny, maxx, maxy) = (lng_min, lat_min, lng_max, lat_max)
region_list.append((slug, name, poly, bounds))
print(f"Loaded {len(region_list)} regions")
# Re-read CSV to assign each point to a region
# We store per-region flat arrays
region_data = defaultdict(list) # slug -> [lat, lng, provId, dl, ul, ...]
region_counts = defaultdict(int) # slug -> point count
region_provider_counts = defaultdict(lambda: defaultdict(int)) # slug -> {provId: count}
region_unassigned = 0
region_total = 0
print("Re-reading CSV for region assignment...")
with open(CSV_PATH, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f, delimiter=DELIMITER)
for row in reader:
prov_name = row.get(FIELD_PROVIDER, '').strip()
if not prov_name:
continue
if prov_name not in valid_providers:
continue
try:
lat = float(row.get(FIELD_LATITUDE, ''))
lng = float(row.get(FIELD_LONGITUDE, ''))
except (ValueError, TypeError):
continue
if math.isnan(lat) or math.isnan(lng):
continue
try:
dl = float(row.get(FIELD_DOWNLOAD, 0) or 0)
except (ValueError, TypeError):
dl = 0.0
try:
ul = float(row.get(FIELD_UPLOAD, 0) or 0)
except (ValueError, TypeError):
ul = 0.0
pid = provider_name_to_id[prov_name]
# Find which region this point belongs to
point = Point(lng, lat)
assigned = False
for slug, name, poly, bounds in region_list:
# Quick bounding box check first
if (lng < bounds[0] or lng > bounds[2] or
lat < bounds[1] or lat > bounds[3]):
continue
if poly.contains(point):
region_data[slug].extend([lat, lng, pid, dl, ul])
region_counts[slug] += 1
region_provider_counts[slug][pid] += 1
assigned = True
break
if not assigned:
region_unassigned += 1
region_total += 1
if region_total % 100000 == 0:
print(f" Assigned {region_total:,} points...")
print(f"\nRegion assignment complete:")
print(f" Total points: {region_total:,}")
print(f" Unassigned (outside any region): {region_unassigned:,}")
print(f" Regions with data: {len(region_data)}")
# Write per-region data files
print("\nWriting region data files...")
os.makedirs(REGION_OUT_DIR, exist_ok=True)
region_index_list = []
for slug, flat_arr in region_data.items():
count = region_counts[slug]
if count == 0:
continue
# Write full detail file
file_path = os.path.join(REGION_OUT_DIR, f"{slug}.json")
with open(file_path, 'w', encoding='utf-8') as rf:
json.dump(flat_arr, rf, separators=(',', ':'))
# Compute bounds from data
lats = [flat_arr[i] for i in range(0, len(flat_arr), 5)]
lngs = [flat_arr[i + 1] for i in range(0, len(flat_arr), 5)]
min_lat = min(lats)
max_lat = max(lats)
min_lng = min(lngs)
max_lng = max(lngs)
# Build provider counts dict
prov_counts = dict(region_provider_counts[slug])
# Find the name for this slug
region_name = slug
for s, n, _, _ in region_list:
if s == slug:
region_name = n
break
# Generate LOD files — write as raw binary (Float64Array bytes)
# This eliminates JSON.parse() overhead in the browser
import struct
STRIDE = 5
lod0_max = 200 # zoom <= 10 (~3KB)
lod1_max = 3000 # zoom 11-12 (~40KB)
lod2_max = 20000 # zoom >= 13 (~300KB) — never load full detail
lod0_arr = aggregate_lod(flat_arr, count, STRIDE, lod0_max)
lod1_arr = aggregate_lod(flat_arr, count, STRIDE, lod1_max)
lod2_arr = aggregate_lod(flat_arr, count, STRIDE, lod2_max)
# Write as raw binary: each float64 = 8 bytes, little-endian
def write_binary(arr, path):
"""Write a list of floats as raw little-endian float64 binary."""
with open(path, 'wb') as f:
f.write(struct.pack(f'<{len(arr)}d', *arr))
lod0_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod0.bin")
lod1_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod1.bin")
lod2_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod2.bin")
write_binary(lod0_arr, lod0_path)
write_binary(lod1_arr, lod1_path)
write_binary(lod2_arr, lod2_path)
lod0_count = len(lod0_arr) // STRIDE
lod1_count = len(lod1_arr) // STRIDE
lod2_count = len(lod2_arr) // STRIDE
region_index_list.append({
'name': region_name,
'slug': slug,
'pointCount': count,
'providerCounts': prov_counts,
'bounds': [min_lat, min_lng, max_lat, max_lng],
'file': f"regions/{slug}.json",
'lod': {
'0': {'file': f"regions/{slug}_lod0.bin", 'count': lod0_count},
'1': {'file': f"regions/{slug}_lod1.bin", 'count': lod1_count},
'2': {'file': f"regions/{slug}_lod2.bin", 'count': lod2_count}
}
})
if len(region_index_list) % 5 == 0:
print(f" Written {len(region_index_list)} regions...")
# Write region_index.json
region_index = {
'regions': region_index_list
}
with open(REGION_INDEX_PATH, 'w', encoding='utf-8') as f:
json.dump(region_index, f, separators=(',', ':'))
print(f"\nWritten: {REGION_INDEX_PATH}")
print(f"Regions: {len(region_index_list)}")
# Print region stats
region_sizes = [(r['name'], r['pointCount']) for r in region_index_list]
region_sizes.sort(key=lambda x: -x[1])
print(f"\n--- Region Stats ---")
for name, count in region_sizes:
safe_name = name.encode('ascii', 'replace').decode('ascii')
print(f" {safe_name}: {count:,} points")
# Print LOD stats for largest region
if region_index_list:
largest = region_index_list[0]
lod = largest['lod']
print(f"\n--- LOD Stats (largest: {largest['name']}) ---")
print(f" LOD0 (~200): {lod['0']['count']} points")
print(f" LOD1 (~2000): {lod['1']['count']} points")
print(f" LOD2 (~10000): {lod['2']['count']} points")
# Clean up region data
region_data.clear()
print("\nDone!")
if __name__ == '__main__':
main()