halkyon/apps/learn/tool/maps/world_countries.py

#!/usr/bin/env python3
"""Builds the app's world map of countries and territories.

The outlines come from Natural Earth, whose vector data is in the public
domain: its Admin 0 (countries) layer at 1:50m, release v5.1.2, which draws
borders as they are on the ground, generalised for maps of this scale with
neighbours' borders kept identical; Gibraltar and the US Outlying Islands,
which that layer leaves out, come from its 1:10m layer. The names come from
the Unicode CLDR,
release 48.2.3 (Unicode License v3), in every language a studyset can be in:
the names phones and computers give countries. Each download is checked
against the hash recorded below.

Each region is a country or territory with its own ISO 3166-1 code (CY, GL,
XK for Kosovo as the EU and CLDR use it), 239 in all. Areas without a code
of their own count where ISO counts them: Northern Cyprus as Cyprus,
Somaliland as Somalia, and the outlying parts Natural Earth draws apart
(Australia's islands in the Indian Ocean) as their country. Siachen Glacier,
which belongs to no code, stays as land that can't be picked.

The outlines are kept as Natural Earth generalised them, so neighbours'
borders still meet; only islands under 0.001 square degrees are dropped
(unless they are all a region has), and coordinates rounded to 3 decimals,
about 100 m. Each region keeps Natural Earth's label point, where the app
marks a region too small to tap.

Run from the repository root with Python 3 and no other packages:

    python3 tool/maps/world_countries.py

It rewrites halkyon_learn_flutter/assets/maps/world_countries.geojson.
"""

import json
import sys
from collections import defaultdict
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))
from outlines import area, download, polygons_of, reduce  # noqa: E402

SOURCE = (
    "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/"
    "v5.1.2/geojson/ne_50m_admin_0_countries.geojson"
)
SOURCE_SHA256 = "3e458fc036ad0a66411f2c1e6cac49c5d7bfb81cb1123bc513b22511a2b7fdeb"
# The finer layer, for what the coarser one leaves out.
DETAILED = (
    "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/"
    "v5.1.2/geojson/ne_10m_admin_0_countries.geojson"
)
DETAILED_SHA256 = (
    "239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255")
FROM_DETAILED = {"GI", "UM"}

NAMES = (
    "https://raw.githubusercontent.com/unicode-org/cldr-json/48.2.3/"
    "cldr-json/cldr-localenames-full/main/{}/territories.json"
)
# The languages a studyset's cards can be in (content_languages.dart), as
# CLDR names them: zh is Chinese in simplified characters.
NAMES_SHA256 = {
    "ar": "b78c4a11a9ad8358048998bcda2eb49e205d5ab281f85991ed097af7b972c3d7",
    "de": "e5b7a831cd70afeaad10a56360c08e2d970e9fe79f254ee8d937a7dfa13686b7",
    "el": "489974d90d8ccc4647b267467e70cbe9abad3c5995e878a6238a853a3315975c",
    "en": "158c1d575308f7e46912edbeda435c8fe2ef5dad280798231f3a432e406b1807",
    "es": "027b9c91d4e923d506b404415648691ef41ebde721aa454cee4d9f7f66b3ea62",
    "fr": "42d67b78cb74ffe87380db1d702d06be995c3082d7f711ffd6e75a721133c5e0",
    "he": "f487b312ed883f3840ea9d3d2b7754023b6897cbd986f77e81d319104531128d",
    "hi": "f77f9c0c4f8bb7062dfe40edca416beae34e17b89714dabedca2f2b41ed77121",
    "it": "d41462485baabf9b0b70de0092a173a45a7af8c415ace686e6cbe1909baf390f",
    "ja": "2e9976a1d5e6a02e48b5a5c18987bd6246b0a21369513386bb5874a6887056d2",
    "ko": "47b2f98d9cbf823068b0594074fe5db7fde80f9baf6a8038bd24ed81aefe7829",
    "nl": "51f5d8ffc56cee8b610b2857eb491d151ad2866da49eccbd7b9ff0d6461f9b3c",
    "pl": "ae91e6cb2c761bc158f08ae3bf9a6d495b05914ac945cc6eea1bee2b8b532aab",
    "pt": "89604c7cd2eecc83c7a7f65129e7e6fe70a04f2d52263345530d714146f99ee8",
    "ru": "305ba1f9eecb1b0151ab081a111aaa8a5898d38c9d57bfbfa125734e508a2389",
    "sv": "8f11c17116f5740ca6f0ccbe404542b65c678033594d1f0f027942131bd981cd",
    "th": "e4c6fd21ec75686f51e9bf986856cac2d6a4cddefe9065373282a7510bcefcd0",
    "tr": "4b9b52e756c79f8b4c81e7c2c5bd2c2ba9338f0c80f2dceeee53ba4561d872b5",
    "uk": "be0b7a34fb070cb4e7749a69eed93e62af6acf7a298437d4eb436ef0163284af",
    "vi": "1bf0ad4f7286b12b326f9d4387af6d4bcc4e1099a269c2ec9a6c9e5d6db20476",
    "zh": "18f1426f1e8981a671517a4857a8d4e56060909906c4cfd9b9a43a8a18b9ab6f",
    "zh-Hant": "2a316ff8da8e54295d4d13ec76e1b6e9d07f3a1a11c76af2ac85ec7d6a7960c6",
}
# The tags the app gives these languages.
TAGS = {"zh": "zh-Hans"}
# CLDR's short names where its usual one says more than a map label needs
# ("Hong Kong SAR China", "Myanmar (Burma)").
SHORT = {"HK", "MO", "MM", "PS"}

TARGET = Path("halkyon_learn_flutter/assets/maps/world_countries.geojson")

TOLERANCE = 0  # degrees: only points on a straight line go
MIN_AREA = 0.001  # square degrees; smaller islands are dropped
DECIMALS = 3

# Areas Natural Earth gives no code of their own, counted where ISO counts
# them; the rest of those stays as land without a code.
JOINED = {
    "N. Cyprus": "CY",
    "Somaliland": "SO",
}


def names_by_code():
    """Every territory's name in each language, by ISO code."""
    names = defaultdict(dict)
    for language, sha256 in NAMES_SHA256.items():
        data = json.loads(download(NAMES.format(language), sha256))
        territories = data["main"][language]["localeDisplayNames"][
            "territories"]
        for code, name in territories.items():
            if len(code) == 2 and code.isalpha():
                short = territories.get(f"{code}-alt-short")
                names[code][TAGS.get(language, language)] = (
                    short if code in SHORT and short else name)
    return names


def main():
    source = json.loads(download(SOURCE, SOURCE_SHA256))["features"]
    detailed = json.loads(download(DETAILED, DETAILED_SHA256))["features"]
    source += [feature for feature in detailed
               if feature["properties"]["ISO_A2_EH"] in FROM_DETAILED]
    names = names_by_code()
    parts = defaultdict(list)
    # Each code's label: its largest area's (France's, not Clipperton
    # Island's), with that area's size.
    labels = {}
    land = []
    for feature in source:
        properties = feature["properties"]
        code = JOINED.get(properties["NAME"], properties["ISO_A2_EH"])
        polygons = polygons_of(feature["geometry"])
        if code == "-99":
            land.extend(polygons)
            continue
        parts[code].extend(polygons)
        size = sum(area(polygon[0]) for polygon in polygons)
        if code not in labels or labels[code][1] < size:
            labels[code] = ([round(properties["LABEL_X"], DECIMALS),
                             round(properties["LABEL_Y"], DECIMALS)], size)
    features = []
    for code in sorted(parts):
        if "en" not in names[code]:
            sys.exit(f"CLDR has no English name for {code}.")
        features.append({
            "type": "Feature",
            "properties": {
                "code": code,
                "names": dict(sorted(names[code].items())),
                "label": labels[code][0],
            },
            "geometry": reduce(parts[code], TOLERANCE, MIN_AREA, DECIMALS),
        })
    features.append({
        "type": "Feature",
        "properties": {},
        "geometry": reduce(land, TOLERANCE, MIN_AREA, DECIMALS),
    })
    if len(features) - 1 != 239:
        sys.exit(f"Expected 239 regions, got {len(features) - 1}.")
    TARGET.write_text(json.dumps(
        {"type": "FeatureCollection", "features": features},
        ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")
    print(f"Wrote {TARGET} ({TARGET.stat().st_size} bytes)", file=sys.stderr)


if __name__ == "__main__":
    main()