Complete rewrite: PapaParse + leaflet.glify + RBush + Web Worker approach
This commit is contained in:
parent
48e63b12da
commit
89e5dd9e6d
11
.gitignore
vendored
Normal file
11
.gitignore
vendored
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
# Generated data files (no longer needed with the new approach)
|
||||||
|
data/regions/
|
||||||
|
data/tiles/
|
||||||
|
data/region_index.json
|
||||||
|
data/tile_index.json
|
||||||
|
data/hulls.json
|
||||||
|
data/speedtest_providers_avg_speeds_by_location — копия.csv
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
@ -1,76 +0,0 @@
|
|||||||
provider,latitude,longitude,avg_download_mbps,avg_upload_mbps
|
|
||||||
Казахтелеком,43.2561,76.9285,87.4,42.1
|
|
||||||
Казахтелеком,43.2350,76.9100,91.2,44.7
|
|
||||||
Казахтелеком,43.2700,76.9500,78.5,38.2
|
|
||||||
Казахтелеком,43.2200,76.8900,95.0,46.0
|
|
||||||
Казахтелеком,43.2900,76.9700,82.3,40.5
|
|
||||||
Казахтелеком,43.2100,76.8800,88.1,43.0
|
|
||||||
Казахтелеком,43.2450,76.9150,76.4,36.8
|
|
||||||
Казахтелеком,43.2600,76.9350,93.7,45.2
|
|
||||||
Казахтелеком,43.2750,76.9550,79.9,39.1
|
|
||||||
Казахтелеком,43.2250,76.8950,85.6,41.3
|
|
||||||
Казахтелеком,51.1800,71.4450,72.5,35.0
|
|
||||||
Казахтелеком,51.1600,71.4250,68.3,32.1
|
|
||||||
Казахтелеком,51.2000,71.4650,74.1,36.2
|
|
||||||
Казахтелеком,51.1400,71.4050,70.8,34.5
|
|
||||||
Казахтелеком,51.2200,71.4850,76.2,37.8
|
|
||||||
Казахтелеком,49.8000,73.1200,65.4,31.0
|
|
||||||
Казахтелеком,49.8200,73.1400,62.1,29.8
|
|
||||||
Казахтелеком,49.7800,73.1000,67.8,32.5
|
|
||||||
Казахтелеком,49.8400,73.1600,63.5,30.2
|
|
||||||
Казахтелеком,49.7600,73.0800,69.2,33.1
|
|
||||||
Beeline,43.2561,76.9285,120.5,58.3
|
|
||||||
Beeline,43.2350,76.9100,115.2,55.1
|
|
||||||
Beeline,43.2700,76.9500,130.8,62.4
|
|
||||||
Beeline,43.2200,76.8900,108.6,52.0
|
|
||||||
Beeline,43.2900,76.9700,125.3,60.1
|
|
||||||
Beeline,43.2100,76.8800,112.4,54.7
|
|
||||||
Beeline,43.2450,76.9150,118.9,57.2
|
|
||||||
Beeline,43.2600,76.9350,135.1,64.8
|
|
||||||
Beeline,43.2750,76.9550,122.7,59.3
|
|
||||||
Beeline,43.2250,76.8950,110.3,53.5
|
|
||||||
Beeline,51.1800,71.4450,95.2,45.0
|
|
||||||
Beeline,51.1600,71.4250,88.7,42.3
|
|
||||||
Beeline,51.2000,71.4650,92.4,44.1
|
|
||||||
Beeline,51.1400,71.4050,85.6,40.8
|
|
||||||
Beeline,51.2200,71.4850,98.1,46.5
|
|
||||||
Beeline,49.8000,73.1200,78.3,37.0
|
|
||||||
Beeline,49.8200,73.1400,82.5,39.2
|
|
||||||
Beeline,49.7800,73.1000,75.1,35.8
|
|
||||||
Beeline,49.8400,73.1600,80.4,38.5
|
|
||||||
Beeline,49.7600,73.0800,76.8,36.4
|
|
||||||
Tele2,43.2561,76.9285,45.2,21.0
|
|
||||||
Tele2,43.2350,76.9100,42.8,19.5
|
|
||||||
Tele2,43.2700,76.9500,48.1,22.3
|
|
||||||
Tele2,43.2200,76.8900,40.5,18.8
|
|
||||||
Tele2,43.2900,76.9700,46.7,21.8
|
|
||||||
Tele2,43.2100,76.8800,43.4,20.1
|
|
||||||
Tele2,43.2450,76.9150,44.9,20.8
|
|
||||||
Tele2,43.2600,76.9350,49.2,22.9
|
|
||||||
Tele2,43.2750,76.9550,41.6,19.2
|
|
||||||
Tele2,43.2250,76.8950,47.3,22.0
|
|
||||||
Tele2,51.1800,71.4450,38.5,17.2
|
|
||||||
Tele2,51.1600,71.4250,35.2,16.0
|
|
||||||
Tele2,51.2000,71.4650,40.1,18.5
|
|
||||||
Tele2,51.1400,71.4050,36.8,16.8
|
|
||||||
Tele2,51.2200,71.4850,39.4,18.1
|
|
||||||
Kcell,43.2561,76.9285,65.3,31.2
|
|
||||||
Kcell,43.2350,76.9100,62.1,29.8
|
|
||||||
Kcell,43.2700,76.9500,68.5,32.7
|
|
||||||
Kcell,43.2200,76.8900,60.4,28.5
|
|
||||||
Kcell,43.2900,76.9700,66.8,31.9
|
|
||||||
Kcell,43.2100,76.8800,63.2,30.1
|
|
||||||
Kcell,43.2450,76.9150,64.7,30.9
|
|
||||||
Kcell,43.2600,76.9350,70.1,33.5
|
|
||||||
Kcell,43.2750,76.9550,61.5,29.2
|
|
||||||
Kcell,43.2250,76.8950,67.2,32.0
|
|
||||||
Altel,43.2561,76.9285,55.8,26.4
|
|
||||||
Altel,43.2350,76.9100,52.3,24.8
|
|
||||||
Altel,43.2700,76.9500,58.1,27.5
|
|
||||||
Altel,43.2200,76.8900,50.6,23.9
|
|
||||||
Altel,43.2900,76.9700,56.4,26.8
|
|
||||||
Altel,43.2100,76.8800,53.7,25.2
|
|
||||||
Altel,43.2450,76.9150,54.9,26.0
|
|
||||||
Altel,43.2600,76.9350,59.2,28.1
|
|
||||||
Altel,43.2750,76.9550,51.8,24.5
|
|
||||||
Altel,43.2250,76.8950,57.1,27.0
|
|
||||||
|
1764
index.html
1764
index.html
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
644
tools/tile-csv.py
Normal file
644
tools/tile-csv.py
Normal file
@ -0,0 +1,644 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
tile-csv.py — Pre-process CSV into tiles + per-region data files + LOD files.
|
||||||
|
|
||||||
|
Output:
|
||||||
|
data/tiles/<key>.json — Flat array per tile
|
||||||
|
data/tile_index.json — Tile list + provider metadata
|
||||||
|
data/hulls.json — Pre-computed convex hulls per provider
|
||||||
|
data/regions/<slug>.json — Full detail flat array per region
|
||||||
|
data/regions/<slug>_lod0.json — ~200 aggregated points (zoom <= 10)
|
||||||
|
data/regions/<slug>_lod1.json — ~2000 aggregated points (zoom 11-12)
|
||||||
|
data/regions/<slug>_lod2.json — Full detail (zoom >= 13)
|
||||||
|
data/region_index.json — Region metadata with LOD info
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python tools/tile-csv.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from shapely.geometry import shape, Point
|
||||||
|
|
||||||
|
# === CONFIG ===
|
||||||
|
CSV_PATH = 'speedtest_providers_avg_speeds_by_location.csv'
|
||||||
|
DELIMITER = ';'
|
||||||
|
TILE_SIZE_DEG = 0.5 # 0.5° × 0.5° tiles (~55km at equator)
|
||||||
|
OUT_DIR = 'data/tiles'
|
||||||
|
INDEX_PATH = 'data/tile_index.json'
|
||||||
|
HULLS_PATH = 'data/hulls.json'
|
||||||
|
REGION_GEOJSON_PATH = 'data/kaz_regions.geojson'
|
||||||
|
REGION_OUT_DIR = 'data/regions'
|
||||||
|
REGION_INDEX_PATH = 'data/region_index.json'
|
||||||
|
EPS_KM = 20
|
||||||
|
MIN_CLUSTER_POINTS = 3
|
||||||
|
DEFAULT_PROVIDERS = ['Kazakhtelecom', 'Beeline', 'Alma Plus', 'Transtelecom', 'NLS']
|
||||||
|
|
||||||
|
# CSV field names (from the file header)
|
||||||
|
FIELD_PROVIDER = 'provider'
|
||||||
|
FIELD_LATITUDE = 'latitude'
|
||||||
|
FIELD_LONGITUDE = 'longitude'
|
||||||
|
FIELD_DOWNLOAD = 'avg_download_speed'
|
||||||
|
FIELD_UPLOAD = 'avg_upload_speed'
|
||||||
|
|
||||||
|
|
||||||
|
def get_tile_key(lat, lng):
|
||||||
|
"""Return tile key string for given lat/lng."""
|
||||||
|
gi = int(math.floor(lat / TILE_SIZE_DEG))
|
||||||
|
gj = int(math.floor(lng / TILE_SIZE_DEG))
|
||||||
|
return f"{gi}_{gj}"
|
||||||
|
|
||||||
|
|
||||||
|
def convex_hull(points):
|
||||||
|
"""
|
||||||
|
Compute convex hull of a list of (lng, lat) pairs.
|
||||||
|
Returns list of [lng, lat] pairs forming the hull, or None if < 3 points.
|
||||||
|
"""
|
||||||
|
pts = sorted(points, key=lambda p: (p[0], p[1]))
|
||||||
|
if len(pts) < 3:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def cross(o, a, b):
|
||||||
|
return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0])
|
||||||
|
|
||||||
|
lower = []
|
||||||
|
for p in pts:
|
||||||
|
while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0:
|
||||||
|
lower.pop()
|
||||||
|
lower.append(p)
|
||||||
|
|
||||||
|
upper = []
|
||||||
|
for p in reversed(pts):
|
||||||
|
while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0:
|
||||||
|
upper.pop()
|
||||||
|
upper.append(p)
|
||||||
|
|
||||||
|
lower.pop()
|
||||||
|
upper.pop()
|
||||||
|
hull = lower + upper
|
||||||
|
|
||||||
|
if len(hull) < 3:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Check collinear
|
||||||
|
is_collinear = True
|
||||||
|
for i in range(2, len(hull)):
|
||||||
|
if cross(hull[0], hull[1], hull[i]) != 0:
|
||||||
|
is_collinear = False
|
||||||
|
break
|
||||||
|
if is_collinear:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Close the ring
|
||||||
|
hull.append(hull[0])
|
||||||
|
return hull
|
||||||
|
|
||||||
|
|
||||||
|
def chaikin_smooth(ring, iterations=2):
|
||||||
|
"""Smooth a polygon ring using Chaikin's algorithm."""
|
||||||
|
if not ring or len(ring) < 4:
|
||||||
|
return ring
|
||||||
|
pts = ring[:-1] # remove closing point
|
||||||
|
for _ in range(iterations):
|
||||||
|
new_pts = []
|
||||||
|
for i in range(len(pts)):
|
||||||
|
p0 = pts[i]
|
||||||
|
p1 = pts[(i + 1) % len(pts)]
|
||||||
|
new_pts.append([0.75 * p0[0] + 0.25 * p1[0], 0.75 * p0[1] + 0.25 * p1[1]])
|
||||||
|
new_pts.append([0.25 * p0[0] + 0.75 * p1[0], 0.25 * p0[1] + 0.75 * p1[1]])
|
||||||
|
pts = new_pts
|
||||||
|
closed = pts + [pts[0]]
|
||||||
|
return closed
|
||||||
|
|
||||||
|
|
||||||
|
def aggregate_lod(flat_arr, count, stride, max_points):
|
||||||
|
"""
|
||||||
|
Aggregate points into a smaller set by binning into a grid.
|
||||||
|
|
||||||
|
flat_arr: list of [lat, lng, provId, dl, ul, ...]
|
||||||
|
count: number of points
|
||||||
|
stride: values per point (5)
|
||||||
|
max_points: target maximum number of output points
|
||||||
|
|
||||||
|
Returns a new flat list with averaged points.
|
||||||
|
"""
|
||||||
|
if count <= max_points:
|
||||||
|
return list(flat_arr)
|
||||||
|
|
||||||
|
# Find bounds
|
||||||
|
lats = [flat_arr[i * stride] for i in range(count)]
|
||||||
|
lngs = [flat_arr[i * stride + 1] for i in range(count)]
|
||||||
|
min_lat, max_lat = min(lats), max(lats)
|
||||||
|
min_lng, max_lng = min(lngs), max(lngs)
|
||||||
|
|
||||||
|
lat_range = max_lat - min_lat
|
||||||
|
lng_range = max_lng - min_lng
|
||||||
|
if lat_range < 1e-6:
|
||||||
|
lat_range = 1.0
|
||||||
|
if lng_range < 1e-6:
|
||||||
|
lng_range = 1.0
|
||||||
|
|
||||||
|
# Count unique providers
|
||||||
|
provider_set = set()
|
||||||
|
for i in range(count):
|
||||||
|
provider_set.add(int(flat_arr[i * stride + 2]))
|
||||||
|
num_providers = len(provider_set)
|
||||||
|
|
||||||
|
# We want roughly max_points output points.
|
||||||
|
# Each grid cell can have up to num_providers points (one per provider).
|
||||||
|
# But in practice, most cells have far fewer providers.
|
||||||
|
# Use a correction factor: assume ~30% of cells have multiple providers
|
||||||
|
avg_providers_per_cell = max(1, int(num_providers * 0.3))
|
||||||
|
target_cells = max(1, max_points // avg_providers_per_cell)
|
||||||
|
|
||||||
|
# Grid dimensions: maintain aspect ratio of the bounding box
|
||||||
|
aspect = lng_range / max(lat_range, 1e-6)
|
||||||
|
grid_rows = max(1, int(math.sqrt(target_cells / max(aspect, 0.01))))
|
||||||
|
grid_cols = max(1, int(grid_rows * aspect))
|
||||||
|
|
||||||
|
# Cap to avoid excessive memory
|
||||||
|
grid_rows = min(grid_rows, 500)
|
||||||
|
grid_cols = min(grid_cols, 500)
|
||||||
|
|
||||||
|
# Bin points into grid cells
|
||||||
|
bins = {} # (row, col) -> {provId: [count, sum_lat, sum_lng, sum_dl, sum_ul]}
|
||||||
|
|
||||||
|
for i in range(count):
|
||||||
|
off = i * stride
|
||||||
|
lat = flat_arr[off]
|
||||||
|
lng = flat_arr[off + 1]
|
||||||
|
pid = int(flat_arr[off + 2])
|
||||||
|
dl = flat_arr[off + 3]
|
||||||
|
ul = flat_arr[off + 4]
|
||||||
|
|
||||||
|
col = int((lng - min_lng) / lng_range * grid_cols)
|
||||||
|
row = int((lat - min_lat) / lat_range * grid_rows)
|
||||||
|
col = min(col, grid_cols - 1)
|
||||||
|
row = min(row, grid_rows - 1)
|
||||||
|
|
||||||
|
key = (row, col)
|
||||||
|
if key not in bins:
|
||||||
|
bins[key] = {}
|
||||||
|
|
||||||
|
if pid not in bins[key]:
|
||||||
|
bins[key][pid] = [0, 0.0, 0.0, 0.0, 0.0]
|
||||||
|
|
||||||
|
b = bins[key][pid]
|
||||||
|
b[0] += 1
|
||||||
|
b[1] += lat
|
||||||
|
b[2] += lng
|
||||||
|
b[3] += dl
|
||||||
|
b[4] += ul
|
||||||
|
|
||||||
|
# Flatten bins into output array
|
||||||
|
result = []
|
||||||
|
for cell_bins in bins.values():
|
||||||
|
for pid, b in cell_bins.items():
|
||||||
|
cnt = b[0]
|
||||||
|
result.append(b[1] / cnt)
|
||||||
|
result.append(b[2] / cnt)
|
||||||
|
result.append(float(pid))
|
||||||
|
result.append(b[3] / cnt)
|
||||||
|
result.append(b[4] / cnt)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print(f"Reading CSV: {CSV_PATH}")
|
||||||
|
print(f"Tile size: {TILE_SIZE_DEG} deg x {TILE_SIZE_DEG} deg")
|
||||||
|
|
||||||
|
# Ensure output directory exists
|
||||||
|
os.makedirs(OUT_DIR, exist_ok=True)
|
||||||
|
|
||||||
|
# Phase 1: Read CSV and group into tiles
|
||||||
|
# We'll store points per tile as flat arrays: [lat, lng, provId, dl, ul, ...]
|
||||||
|
tile_data = {} # key -> list of numbers
|
||||||
|
tile_counts = {} # key -> point count
|
||||||
|
provider_name_to_id = {}
|
||||||
|
provider_id_to_name = []
|
||||||
|
provider_counts = defaultdict(int)
|
||||||
|
total_rows = 0
|
||||||
|
skipped_rows = 0
|
||||||
|
|
||||||
|
with open(CSV_PATH, 'r', encoding='utf-8-sig') as f:
|
||||||
|
reader = csv.DictReader(f, delimiter=DELIMITER)
|
||||||
|
|
||||||
|
for row in reader:
|
||||||
|
total_rows += 1
|
||||||
|
|
||||||
|
# Extract fields
|
||||||
|
prov_name = row.get(FIELD_PROVIDER, '').strip()
|
||||||
|
if not prov_name:
|
||||||
|
skipped_rows += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
lat = float(row.get(FIELD_LATITUDE, ''))
|
||||||
|
lng = float(row.get(FIELD_LONGITUDE, ''))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
skipped_rows += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
if math.isnan(lat) or math.isnan(lng):
|
||||||
|
skipped_rows += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
dl = float(row.get(FIELD_DOWNLOAD, 0) or 0)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
dl = 0.0
|
||||||
|
try:
|
||||||
|
ul = float(row.get(FIELD_UPLOAD, 0) or 0)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ul = 0.0
|
||||||
|
|
||||||
|
# Assign provider ID
|
||||||
|
if prov_name not in provider_name_to_id:
|
||||||
|
pid = len(provider_id_to_name)
|
||||||
|
provider_name_to_id[prov_name] = pid
|
||||||
|
provider_id_to_name.append(prov_name)
|
||||||
|
else:
|
||||||
|
pid = provider_name_to_id[prov_name]
|
||||||
|
|
||||||
|
provider_counts[prov_name] += 1
|
||||||
|
|
||||||
|
# Determine tile
|
||||||
|
key = get_tile_key(lat, lng)
|
||||||
|
|
||||||
|
if key not in tile_data:
|
||||||
|
tile_data[key] = []
|
||||||
|
tile_counts[key] = 0
|
||||||
|
|
||||||
|
tile_data[key].extend([lat, lng, pid, dl, ul])
|
||||||
|
tile_counts[key] = tile_counts.get(key, 0) + 1
|
||||||
|
|
||||||
|
if total_rows % 100000 == 0:
|
||||||
|
print(f" Processed {total_rows:,} rows...")
|
||||||
|
|
||||||
|
print(f"\nTotal rows: {total_rows:,}")
|
||||||
|
print(f"Skipped: {skipped_rows:,}")
|
||||||
|
print(f"Providers: {len(provider_id_to_name)}")
|
||||||
|
print(f"Tiles: {len(tile_data)}")
|
||||||
|
|
||||||
|
# Filter providers with < 3 points
|
||||||
|
valid_providers = set()
|
||||||
|
for name, count in provider_counts.items():
|
||||||
|
if count >= MIN_CLUSTER_POINTS:
|
||||||
|
valid_providers.add(name)
|
||||||
|
|
||||||
|
print(f"Valid providers (>= {MIN_CLUSTER_POINTS} points): {len(valid_providers)}")
|
||||||
|
|
||||||
|
# Phase 2: Write tile files
|
||||||
|
print("\nWriting tile files...")
|
||||||
|
tile_list = []
|
||||||
|
total_points_written = 0
|
||||||
|
|
||||||
|
for key, flat_arr in tile_data.items():
|
||||||
|
count = tile_counts[key]
|
||||||
|
|
||||||
|
# Filter out points from invalid providers
|
||||||
|
filtered = []
|
||||||
|
filtered_count = 0
|
||||||
|
for i in range(count):
|
||||||
|
off = i * 5
|
||||||
|
pid = int(flat_arr[off + 2])
|
||||||
|
pname = provider_id_to_name[pid]
|
||||||
|
if pname in valid_providers:
|
||||||
|
filtered.extend(flat_arr[off:off + 5])
|
||||||
|
filtered_count += 1
|
||||||
|
|
||||||
|
if filtered_count == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
tile_path = os.path.join(OUT_DIR, f"{key}.json")
|
||||||
|
with open(tile_path, 'w', encoding='utf-8') as tf:
|
||||||
|
json.dump(filtered, tf, separators=(',', ':'))
|
||||||
|
|
||||||
|
tile_list.append({
|
||||||
|
'key': key,
|
||||||
|
'count': filtered_count,
|
||||||
|
'file': f"tiles/{key}.json"
|
||||||
|
})
|
||||||
|
total_points_written += filtered_count
|
||||||
|
|
||||||
|
if len(tile_list) % 100 == 0:
|
||||||
|
print(f" Written {len(tile_list)} tiles...")
|
||||||
|
|
||||||
|
print(f"Total tiles written: {len(tile_list)}")
|
||||||
|
print(f"Total points written: {total_points_written:,}")
|
||||||
|
|
||||||
|
# Phase 3: Build provider list for tile_index.json
|
||||||
|
print("\nBuilding provider list...")
|
||||||
|
provider_list = []
|
||||||
|
default_set = set(p.lower() for p in DEFAULT_PROVIDERS)
|
||||||
|
|
||||||
|
for pid, name in enumerate(provider_id_to_name):
|
||||||
|
if name not in valid_providers:
|
||||||
|
continue
|
||||||
|
color_idx = pid % 20 # matches CONFIG.PALETTE
|
||||||
|
provider_list.append({
|
||||||
|
'id': pid,
|
||||||
|
'name': name,
|
||||||
|
'count': provider_counts[name],
|
||||||
|
'colorIdx': color_idx,
|
||||||
|
'isDefault': name.lower() in default_set
|
||||||
|
})
|
||||||
|
|
||||||
|
# Phase 4: Write tile_index.json
|
||||||
|
index_data = {
|
||||||
|
'tileSizeDeg': TILE_SIZE_DEG,
|
||||||
|
'pointStride': 5,
|
||||||
|
'providers': provider_list,
|
||||||
|
'tiles': tile_list
|
||||||
|
}
|
||||||
|
|
||||||
|
with open(INDEX_PATH, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(index_data, f, separators=(',', ':'))
|
||||||
|
|
||||||
|
print(f"\nWritten: {INDEX_PATH}")
|
||||||
|
|
||||||
|
# Phase 5: Pre-compute convex hulls per provider
|
||||||
|
print("\nComputing convex hulls...")
|
||||||
|
hulls = []
|
||||||
|
|
||||||
|
# Group points by provider (from all tiles)
|
||||||
|
provider_points = defaultdict(list)
|
||||||
|
for key, flat_arr in tile_data.items():
|
||||||
|
count = tile_counts[key]
|
||||||
|
for i in range(count):
|
||||||
|
off = i * 5
|
||||||
|
pid = int(flat_arr[off + 2])
|
||||||
|
pname = provider_id_to_name[pid]
|
||||||
|
if pname not in valid_providers:
|
||||||
|
continue
|
||||||
|
lng = flat_arr[off + 1]
|
||||||
|
lat = flat_arr[off + 0]
|
||||||
|
dl = flat_arr[off + 3]
|
||||||
|
ul = flat_arr[off + 4]
|
||||||
|
provider_points[pid].append((lng, lat, dl, ul))
|
||||||
|
|
||||||
|
for pid, pts in provider_points.items():
|
||||||
|
if len(pts) < MIN_CLUSTER_POINTS:
|
||||||
|
continue
|
||||||
|
|
||||||
|
pname = provider_id_to_name[pid]
|
||||||
|
coords = [(p[0], p[1]) for p in pts] # (lng, lat)
|
||||||
|
hull = convex_hull(coords)
|
||||||
|
if hull is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
hull = chaikin_smooth(hull, 2)
|
||||||
|
if hull is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
sum_dl = sum(p[2] for p in pts)
|
||||||
|
sum_ul = sum(p[3] for p in pts)
|
||||||
|
avg_dl = sum_dl / len(pts)
|
||||||
|
avg_ul = sum_ul / len(pts)
|
||||||
|
|
||||||
|
feature = {
|
||||||
|
'type': 'Feature',
|
||||||
|
'geometry': {
|
||||||
|
'type': 'Polygon',
|
||||||
|
'coordinates': [hull]
|
||||||
|
},
|
||||||
|
'properties': {
|
||||||
|
'provider': pname,
|
||||||
|
'providerId': pid,
|
||||||
|
'count': len(pts),
|
||||||
|
'avg_download': round(avg_dl, 1),
|
||||||
|
'avg_upload': round(avg_ul, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hulls.append(feature)
|
||||||
|
|
||||||
|
hulls_geojson = {
|
||||||
|
'type': 'FeatureCollection',
|
||||||
|
'features': hulls
|
||||||
|
}
|
||||||
|
|
||||||
|
with open(HULLS_PATH, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(hulls_geojson, f, separators=(',', ':'))
|
||||||
|
|
||||||
|
print(f"Written: {HULLS_PATH}")
|
||||||
|
print(f"Hulls: {len(hulls)}")
|
||||||
|
|
||||||
|
# Clean up tile_data to free memory
|
||||||
|
tile_data.clear()
|
||||||
|
provider_points.clear()
|
||||||
|
|
||||||
|
# Print tile size stats
|
||||||
|
tile_sizes = [t['count'] for t in tile_list]
|
||||||
|
if tile_sizes:
|
||||||
|
print(f"\n--- Tile Stats ---")
|
||||||
|
print(f"Min points/tile: {min(tile_sizes)}")
|
||||||
|
print(f"Max points/tile: {max(tile_sizes)}")
|
||||||
|
print(f"Avg points/tile: {sum(tile_sizes) / len(tile_sizes):.1f}")
|
||||||
|
print(f"Median points/tile: {sorted(tile_sizes)[len(tile_sizes) // 2]}")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# Phase 6: Map points to administrative regions
|
||||||
|
# ============================================================
|
||||||
|
print("\n--- Phase 6: Mapping points to regions ---")
|
||||||
|
|
||||||
|
# Load region boundaries
|
||||||
|
print(f"Loading region boundaries: {REGION_GEOJSON_PATH}")
|
||||||
|
with open(REGION_GEOJSON_PATH, 'r', encoding='utf-8') as f:
|
||||||
|
regions_geojson = json.load(f)
|
||||||
|
|
||||||
|
# Build region polygon list with spatial indexing
|
||||||
|
region_list = [] # [(slug, name, shapely_polygon, bounds)]
|
||||||
|
for feature in regions_geojson['features']:
|
||||||
|
props = feature['properties']
|
||||||
|
name = props.get('NAME_1', 'Unknown')
|
||||||
|
slug = name.lower().replace(' ', '_').replace('-', '_').replace("'", '')
|
||||||
|
poly = shape(feature['geometry'])
|
||||||
|
# Compute bounding box for quick pre-filter
|
||||||
|
bounds = poly.bounds # (minx, miny, maxx, maxy) = (lng_min, lat_min, lng_max, lat_max)
|
||||||
|
region_list.append((slug, name, poly, bounds))
|
||||||
|
|
||||||
|
print(f"Loaded {len(region_list)} regions")
|
||||||
|
|
||||||
|
# Re-read CSV to assign each point to a region
|
||||||
|
# We store per-region flat arrays
|
||||||
|
region_data = defaultdict(list) # slug -> [lat, lng, provId, dl, ul, ...]
|
||||||
|
region_counts = defaultdict(int) # slug -> point count
|
||||||
|
region_provider_counts = defaultdict(lambda: defaultdict(int)) # slug -> {provId: count}
|
||||||
|
region_unassigned = 0
|
||||||
|
region_total = 0
|
||||||
|
|
||||||
|
print("Re-reading CSV for region assignment...")
|
||||||
|
with open(CSV_PATH, 'r', encoding='utf-8-sig') as f:
|
||||||
|
reader = csv.DictReader(f, delimiter=DELIMITER)
|
||||||
|
for row in reader:
|
||||||
|
prov_name = row.get(FIELD_PROVIDER, '').strip()
|
||||||
|
if not prov_name:
|
||||||
|
continue
|
||||||
|
if prov_name not in valid_providers:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
lat = float(row.get(FIELD_LATITUDE, ''))
|
||||||
|
lng = float(row.get(FIELD_LONGITUDE, ''))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
if math.isnan(lat) or math.isnan(lng):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
dl = float(row.get(FIELD_DOWNLOAD, 0) or 0)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
dl = 0.0
|
||||||
|
try:
|
||||||
|
ul = float(row.get(FIELD_UPLOAD, 0) or 0)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ul = 0.0
|
||||||
|
|
||||||
|
pid = provider_name_to_id[prov_name]
|
||||||
|
|
||||||
|
# Find which region this point belongs to
|
||||||
|
point = Point(lng, lat)
|
||||||
|
assigned = False
|
||||||
|
for slug, name, poly, bounds in region_list:
|
||||||
|
# Quick bounding box check first
|
||||||
|
if (lng < bounds[0] or lng > bounds[2] or
|
||||||
|
lat < bounds[1] or lat > bounds[3]):
|
||||||
|
continue
|
||||||
|
if poly.contains(point):
|
||||||
|
region_data[slug].extend([lat, lng, pid, dl, ul])
|
||||||
|
region_counts[slug] += 1
|
||||||
|
region_provider_counts[slug][pid] += 1
|
||||||
|
assigned = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if not assigned:
|
||||||
|
region_unassigned += 1
|
||||||
|
|
||||||
|
region_total += 1
|
||||||
|
if region_total % 100000 == 0:
|
||||||
|
print(f" Assigned {region_total:,} points...")
|
||||||
|
|
||||||
|
print(f"\nRegion assignment complete:")
|
||||||
|
print(f" Total points: {region_total:,}")
|
||||||
|
print(f" Unassigned (outside any region): {region_unassigned:,}")
|
||||||
|
print(f" Regions with data: {len(region_data)}")
|
||||||
|
|
||||||
|
# Write per-region data files
|
||||||
|
print("\nWriting region data files...")
|
||||||
|
os.makedirs(REGION_OUT_DIR, exist_ok=True)
|
||||||
|
region_index_list = []
|
||||||
|
|
||||||
|
for slug, flat_arr in region_data.items():
|
||||||
|
count = region_counts[slug]
|
||||||
|
if count == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Write full detail file
|
||||||
|
file_path = os.path.join(REGION_OUT_DIR, f"{slug}.json")
|
||||||
|
with open(file_path, 'w', encoding='utf-8') as rf:
|
||||||
|
json.dump(flat_arr, rf, separators=(',', ':'))
|
||||||
|
|
||||||
|
# Compute bounds from data
|
||||||
|
lats = [flat_arr[i] for i in range(0, len(flat_arr), 5)]
|
||||||
|
lngs = [flat_arr[i + 1] for i in range(0, len(flat_arr), 5)]
|
||||||
|
min_lat = min(lats)
|
||||||
|
max_lat = max(lats)
|
||||||
|
min_lng = min(lngs)
|
||||||
|
max_lng = max(lngs)
|
||||||
|
|
||||||
|
# Build provider counts dict
|
||||||
|
prov_counts = dict(region_provider_counts[slug])
|
||||||
|
|
||||||
|
# Find the name for this slug
|
||||||
|
region_name = slug
|
||||||
|
for s, n, _, _ in region_list:
|
||||||
|
if s == slug:
|
||||||
|
region_name = n
|
||||||
|
break
|
||||||
|
|
||||||
|
# Generate LOD files — write as raw binary (Float64Array bytes)
|
||||||
|
# This eliminates JSON.parse() overhead in the browser
|
||||||
|
import struct
|
||||||
|
STRIDE = 5
|
||||||
|
lod0_max = 200 # zoom <= 10 (~3KB)
|
||||||
|
lod1_max = 3000 # zoom 11-12 (~40KB)
|
||||||
|
lod2_max = 20000 # zoom >= 13 (~300KB) — never load full detail
|
||||||
|
lod0_arr = aggregate_lod(flat_arr, count, STRIDE, lod0_max)
|
||||||
|
lod1_arr = aggregate_lod(flat_arr, count, STRIDE, lod1_max)
|
||||||
|
lod2_arr = aggregate_lod(flat_arr, count, STRIDE, lod2_max)
|
||||||
|
|
||||||
|
# Write as raw binary: each float64 = 8 bytes, little-endian
|
||||||
|
def write_binary(arr, path):
|
||||||
|
"""Write a list of floats as raw little-endian float64 binary."""
|
||||||
|
with open(path, 'wb') as f:
|
||||||
|
f.write(struct.pack(f'<{len(arr)}d', *arr))
|
||||||
|
|
||||||
|
lod0_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod0.bin")
|
||||||
|
lod1_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod1.bin")
|
||||||
|
lod2_path = os.path.join(REGION_OUT_DIR, f"{slug}_lod2.bin")
|
||||||
|
write_binary(lod0_arr, lod0_path)
|
||||||
|
write_binary(lod1_arr, lod1_path)
|
||||||
|
write_binary(lod2_arr, lod2_path)
|
||||||
|
|
||||||
|
lod0_count = len(lod0_arr) // STRIDE
|
||||||
|
lod1_count = len(lod1_arr) // STRIDE
|
||||||
|
lod2_count = len(lod2_arr) // STRIDE
|
||||||
|
|
||||||
|
region_index_list.append({
|
||||||
|
'name': region_name,
|
||||||
|
'slug': slug,
|
||||||
|
'pointCount': count,
|
||||||
|
'providerCounts': prov_counts,
|
||||||
|
'bounds': [min_lat, min_lng, max_lat, max_lng],
|
||||||
|
'file': f"regions/{slug}.json",
|
||||||
|
'lod': {
|
||||||
|
'0': {'file': f"regions/{slug}_lod0.bin", 'count': lod0_count},
|
||||||
|
'1': {'file': f"regions/{slug}_lod1.bin", 'count': lod1_count},
|
||||||
|
'2': {'file': f"regions/{slug}_lod2.bin", 'count': lod2_count}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(region_index_list) % 5 == 0:
|
||||||
|
print(f" Written {len(region_index_list)} regions...")
|
||||||
|
|
||||||
|
# Write region_index.json
|
||||||
|
region_index = {
|
||||||
|
'regions': region_index_list
|
||||||
|
}
|
||||||
|
with open(REGION_INDEX_PATH, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(region_index, f, separators=(',', ':'))
|
||||||
|
|
||||||
|
print(f"\nWritten: {REGION_INDEX_PATH}")
|
||||||
|
print(f"Regions: {len(region_index_list)}")
|
||||||
|
|
||||||
|
# Print region stats
|
||||||
|
region_sizes = [(r['name'], r['pointCount']) for r in region_index_list]
|
||||||
|
region_sizes.sort(key=lambda x: -x[1])
|
||||||
|
print(f"\n--- Region Stats ---")
|
||||||
|
for name, count in region_sizes:
|
||||||
|
safe_name = name.encode('ascii', 'replace').decode('ascii')
|
||||||
|
print(f" {safe_name}: {count:,} points")
|
||||||
|
|
||||||
|
# Print LOD stats for largest region
|
||||||
|
if region_index_list:
|
||||||
|
largest = region_index_list[0]
|
||||||
|
lod = largest['lod']
|
||||||
|
print(f"\n--- LOD Stats (largest: {largest['name']}) ---")
|
||||||
|
print(f" LOD0 (~200): {lod['0']['count']} points")
|
||||||
|
print(f" LOD1 (~2000): {lod['1']['count']} points")
|
||||||
|
print(f" LOD2 (~10000): {lod['2']['count']} points")
|
||||||
|
|
||||||
|
# Clean up region data
|
||||||
|
region_data.clear()
|
||||||
|
|
||||||
|
print("\nDone!")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user