From 26167b26d6e98ba34f0f7072d495fb8b483b3373 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 12:41:05 +1300 Subject: [PATCH 001/111] Source layers can now have an optional background defined in a map's manifest. These are rasterised for the viewer. --- mapmaker/__init__.py | 3 + mapmaker/flatmap/__init__.py | 11 +-- mapmaker/flatmap/layers.py | 78 ++++++++++++---------- mapmaker/flatmap/manifest.py | 23 ++++++- mapmaker/geometry/__init__.py | 6 +- mapmaker/sources/__init__.py | 47 ++++++++++--- mapmaker/sources/fc_powerpoint/__init__.py | 7 +- mapmaker/sources/mbfbioscience/__init__.py | 4 +- mapmaker/sources/powerpoint/__init__.py | 7 +- mapmaker/sources/svg/__init__.py | 14 +++- 10 files changed, 135 insertions(+), 65 deletions(-) diff --git a/mapmaker/__init__.py b/mapmaker/__init__.py index 9aa474f0..0bd92708 100644 --- a/mapmaker/__init__.py +++ b/mapmaker/__init__.py @@ -31,6 +31,9 @@ MIN_ZOOM = 2 #: Default minimum zoom level for generated flatmaps MAX_ZOOM = 10 #: Default maximum zoom level for generated flatmaps + +ZOOM_OFFSET_FROM_BASE = 1 + #=============================================================================== from .maker import MapMaker diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index c19546e2..5e141001 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -45,7 +45,7 @@ from .layers import FEATURES_TILE_LAYER, MapLayer # Exports -from .manifest import Manifest, SourceManifest +from .manifest import Manifest, SourceBackground, SourceManifest if TYPE_CHECKING: from mapmaker.annotation import Annotator @@ -354,7 +354,7 @@ def add_source_layers(self, layer_number: int, source: 'MapSource'): for layer in source.layers: self.add_layer(layer) if layer.exported: - layer.add_raster_layer(layer.id, source.extent, source) + layer.add_raster_layers(layer.id, source.extent, source) # The first layer is used as the base map if layer_number == 0: if source.kind == 'details': @@ -380,7 +380,8 @@ def layer_metadata(self): { 'id': raster_layer.id, 'options': { 'max-zoom': raster_layer.max_zoom, - 'min-zoom': raster_layer.min_zoom + 'min-zoom': raster_layer.min_zoom, + 'background': raster_layer.background_layer } } for raster_layer in layer.raster_layers ] @@ -472,9 +473,9 @@ def __add_detail_features(self, layer, detail_layer, lowres_features): else: # nerve feature.pop_property('maxzoom') - if hires_layer.source.raster_source is not None: + if len(hires_layer.source.raster_sources): extent = transform.transform_extent(hires_layer.source.extent) - layer.add_raster_layer('{}_{}'.format(detail_layer.id, hires_layer.id), + layer.add_raster_layers(f'{detail_layer.id}_{hires_layer.id}', extent, hires_layer.source, minzoom, local_world_to_base=transform) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index e0ddf786..71cfcf7c 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -29,16 +29,16 @@ #=============================================================================== -from mapmaker import MIN_ZOOM +from mapmaker import ZOOM_OFFSET_FROM_BASE from mapmaker.exceptions import GroupValueError -from mapmaker.geometry import connect_dividers, extend_line, make_boundary -from mapmaker.geometry import save_geometry +from mapmaker.geometry import connect_dividers, extend_line, make_boundary, merge_bounds +from mapmaker.geometry import MapBounds, save_geometry, Transform from mapmaker.settings import settings -from mapmaker.utils import log +from mapmaker.utils import FilePath, log if TYPE_CHECKING: - from mapmaker.sources import MapSource - from . import FlatMap + from mapmaker.sources import MapSource, RasterSource + from . import FlatMap, SourceBackground from .feature import Feature @@ -194,17 +194,18 @@ def add_feature(self, feature: Feature): # type: ignore if feature.has_property('details'): self.__detail_features.append(feature) - def add_raster_layer(self, layer_id: str, extent, map_source: 'MapSource', min_zoom: Optional[int]=None, local_world_to_base=None): - #================================================================================================================================== - if map_source.raster_source is not None: - if min_zoom is not None: - min_zoom += 1 - else: - min_zoom = self.min_zoom - if map_source.base_feature is not None: - min_zoom -= 3 - self.__raster_layers.append(RasterLayer(layer_id.replace('/', '_'), extent, map_source, min_zoom, - local_world_to_base)) + def add_raster_layers(self, layer_id: str, extent: MapBounds, map_source: 'MapSource', + min_zoom: Optional[int]=None, local_world_to_base: Optional[Transform]=None): + #================================================================================================== + if min_zoom is not None: + min_zoom += 1 + else: + min_zoom = self.min_zoom + if map_source.base_feature is not None: + min_zoom -= ZOOM_OFFSET_FROM_BASE + self.__raster_layers = [RasterLayer(raster_source, extent, + min_zoom=min_zoom, local_world_to_base=local_world_to_base) + for raster_source in map_source.raster_sources] def add_group_features(self, group_name: str, features: list[Feature], tile_layer=FEATURES_TILE_LAYER, outermost=False) -> Optional[Feature]: #=========================================================================================================================================== @@ -402,31 +403,32 @@ class RasterLayer(object): """ Details of layer for creating raster tiles. - :param id: the ``id`` of the source layer to rasterise - :type id: str + :param raster_source: the source to be rasterised :param extent: the extent of the base map in which the layer is to be rasterised as decimal latitude and longitude coordinates. - :type extent: tuple(south, west, north, east) - :param map_source: the source of the layer's data - :type map_source: :class:`~mapmaker.sources.MapSource` :param min_zoom: The minimum zoom level to generate tiles for. Optional, defaults to ``min_zoom`` of ``map_source``. - :type map_zoom: int :param local_world_to_base: an optional transform from the raster layer's local world coordinates to the base map's world coordinates. Defaults to ``None``, meaning the :class:`~mapmaker.geometry.Transform.Identity()` transform - :type local_world_to_base: :class:`~mapmaker.geometry.Transform` """ - def __init__(self, id: str, extent, map_source: 'MapSource', - min_zoom:Optional[int]=None, max_zoom: Optional[int]=None, local_world_to_base=None): - self.__id = '{}_image'.format(id) + def __init__(self, raster_source: 'RasterSource', extent: MapBounds, + min_zoom:Optional[int]=None, max_zoom: Optional[int]=None, + local_world_to_base: Optional[Transform]=None): + self.__id = raster_source.id self.__extent = extent - self.__map_source = map_source - self.__flatmap = map_source.flatmap - self.__max_zoom = max_zoom if max_zoom is not None else settings.get('maxRasterZoom', map_source.max_zoom) - self.__min_zoom = min_zoom if min_zoom is not None else map_source.min_zoom + self.__raster_source = raster_source + self.__map_source = raster_source.map_source + self.__flatmap = self.__map_source.flatmap + self.__max_zoom = max_zoom if max_zoom is not None else settings.get('maxRasterZoom', self.__map_source.max_zoom) + self.__min_zoom = min_zoom if min_zoom is not None else self.__map_source.min_zoom self.__local_world_to_base = local_world_to_base + self.__background_layer = raster_source.background_layer + + @property + def background_layer(self) -> bool: + return self.__background_layer @property def extent(self): @@ -458,22 +460,26 @@ def min_zoom(self): @property def source_data(self) -> bytes: - return self.__map_source.raster_source.data + return self.__raster_source.data @property def source_extent(self): return self.__map_source.extent @property - def source_kind(self): - return self.__map_source.raster_source.kind + def source_kind(self) -> str: + return self.__raster_source.kind @property - def source_path(self): - return self.__map_source.raster_source.source_path + def source_path(self) -> Optional[FilePath]: + return self.__raster_source.source_path @property def source_range(self): return self.__map_source.source_range + @property + def transform(self) -> Optional[Transform]: + return self.__raster_source.transform + #=============================================================================== diff --git a/mapmaker/flatmap/manifest.py b/mapmaker/flatmap/manifest.py index 9784dda5..256e2828 100644 --- a/mapmaker/flatmap/manifest.py +++ b/mapmaker/flatmap/manifest.py @@ -20,6 +20,7 @@ from collections import namedtuple from copy import deepcopy +from dataclasses import dataclass from datetime import datetime from enum import Enum import multiprocessing @@ -166,6 +167,14 @@ def path_blob_url(self, path): #=============================================================================== +@dataclass +class SourceBackground: + href: str + scale: float + translate: tuple[float, float] + +#=============================================================================== + class SourceManifest: def __init__(self, description: dict, manifest: 'Manifest'): self.__id = description['id'] @@ -182,6 +191,18 @@ def __init__(self, description: dict, manifest: 'Manifest'): if (source_range := description.get('slides')) is not None else None) self.__zoom = description['zoom'] if self.__feature is not None else 0 + if (background := description.get('background')) is not None: + if (href := manifest.check_and_normalise_path(background.get('href'), 'Background source file')) is None: + raise ValueError(f'Background for source {self.__id} has no `href`') + self.__background_source = SourceBackground(href, + float(background.get('scale', 1.0)), + tuple(float(t) for t in background.get('translate', [0, 0]))[0:2]) # type: ignore + else: + self.__background_source = None + + @property + def background_source(self) -> Optional[SourceBackground]: + return self.__background_source @property def boundary(self) -> Optional[str]: @@ -399,7 +420,7 @@ def clean_up(self): def check_and_normalise_path(self, path: str, desc: str='') -> str|None: #======================================================================= - if path.strip() == '': + if path is None or path.strip() == '': return None normalised_path = self.__path.join_url(path) if not self.__ignore_git: diff --git a/mapmaker/geometry/__init__.py b/mapmaker/geometry/__init__.py index b085c13e..188b279a 100644 --- a/mapmaker/geometry/__init__.py +++ b/mapmaker/geometry/__init__.py @@ -120,8 +120,10 @@ def scale(cls, scale: float): return cls([[scale, 0, 0], [0, scale, 0], [0, 0, 1]]) @classmethod - def translate(cls, tx: float, ty: float): - return cls([[1, 0, tx], [0, 1, ty], [0, 0, 1]]) + def translate(cls, translate: tuple[float, float]): + return cls([[1, 0, translate[0]], + [0, 1, translate[1]], + [0, 0, 1]]) @property def matrix(self): diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 86a49ec1..09c7504f 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -29,7 +29,7 @@ #=============================================================================== from mapmaker.geometry import bounds_to_extent, MapBounds, Transform -from mapmaker.flatmap import SourceManifest, SOURCE_DETAIL_KINDS +from mapmaker.flatmap import SourceBackground, SourceManifest, SOURCE_DETAIL_KINDS from mapmaker.flatmap.layers import PATHWAYS_TILE_LAYER from mapmaker.properties.markup import parse_markup from mapmaker.utils import FilePath @@ -133,7 +133,8 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): self.__errors: list[tuple[str, str]] = [] self.__layers: list['MapLayer'] = [] self.__bounds: MapBounds = (0, 0, 0, 0) - self.__raster_source = None + self.__raster_sources = None + self.__background_raster_source = source_manifest.background_source if self.__kind in SOURCE_DETAIL_KINDS: if source_manifest.feature is None: raise ValueError('A `detail` source must specify an existing `feature`') @@ -154,6 +155,10 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): def annotator(self): return None + @property + def background_raster_source(self) -> Optional[SourceBackground]: + return self.__background_raster_source + @property def base_feature(self): return self.__base_feature @@ -207,10 +212,10 @@ def min_zoom(self): return self.__min_zoom @property - def raster_source(self) -> 'RasterSource': - if self.__raster_source is None: - self.__raster_source = self.get_raster_source() - return self.__raster_source # type: ignore + def raster_sources(self) -> list['RasterSource']: + if self.__raster_sources is None: + self.__raster_sources = self.get_raster_sources() + return self.__raster_sources # type: ignore @property def href(self): @@ -272,18 +277,28 @@ def process(self) -> None: #========================= raise TypeError('`process()` must be implemented by `MapSource` sub-class') - def get_raster_source(self) -> Optional['RasterSource']: - #======================================================= - return None + def get_raster_sources(self) -> list['RasterSource']: + #==================================================== + return [] #=============================================================================== class RasterSource(object): - def __init__(self, kind: str, get_data: Callable[[], bytes], source_path: Optional[FilePath]=None): + def __init__(self, id: str, kind: str, get_data: Callable[[], bytes], + map_source: MapSource, source_path: Optional[FilePath]=None, + background_layer: bool=False, transform: Optional[Transform]=None): + self.__id = id self.__kind = kind self.__get_data = get_data self.__data = None + self.__map_source = map_source self.__source_path = source_path + self.__background_layer = background_layer + self.__transform = transform + + @property + def background_layer(self): + return self.__background_layer @property def data(self) -> bytes: @@ -291,14 +306,26 @@ def data(self) -> bytes: self.__data = self.__get_data() return self.__data + @property + def id(self) -> str: + return self.__id + @property def kind(self) -> str: return self.__kind + @property + def map_source(self): + return self.__map_source + @property def source_path(self) -> Optional[FilePath]: return self.__source_path + @property + def transform(self) -> Optional[Transform]: + return self.__transform + #=============================================================================== # Export our sources here to avoid circular imports diff --git a/mapmaker/sources/fc_powerpoint/__init__.py b/mapmaker/sources/fc_powerpoint/__init__.py index 8f55a774..05188dcd 100644 --- a/mapmaker/sources/fc_powerpoint/__init__.py +++ b/mapmaker/sources/fc_powerpoint/__init__.py @@ -39,6 +39,7 @@ from mapmaker.shapes.shapefilter import ShapeFilter from mapmaker.utils import log +from .. import RasterSource from ..powerpoint import PowerpointSource, Slide from ..powerpoint.colour import ColourTheme @@ -84,9 +85,9 @@ def __init__(self, flatmap, source_manifest: SourceManifest, ), **kwds) - def get_raster_source(self): - #=========================== - return None # We don't rasterise FC maps + def get_raster_sources(self) -> list[RasterSource]: + #================================================== + return [] # We don't rasterise FC maps #=============================================================================== diff --git a/mapmaker/sources/mbfbioscience/__init__.py b/mapmaker/sources/mbfbioscience/__init__.py index 9160194f..54d42c3b 100644 --- a/mapmaker/sources/mbfbioscience/__init__.py +++ b/mapmaker/sources/mbfbioscience/__init__.py @@ -159,8 +159,8 @@ def process(self): self.__image = mask_image(self.__image, self.__world_to_image.transform_geometry(boundary_geometry)) - def get_raster_source(self): - #============================ return RasterSource('image', lambda: self.__image) + def get_raster_sources(self) -> list[RasterSource]: + #================================================== #=============================================================================== diff --git a/mapmaker/sources/powerpoint/__init__.py b/mapmaker/sources/powerpoint/__init__.py index 9ff333ba..0ab5aff1 100644 --- a/mapmaker/sources/powerpoint/__init__.py +++ b/mapmaker/sources/powerpoint/__init__.py @@ -175,10 +175,11 @@ def process(self): if 'exportSVG' in settings: self.__make_svg() - def get_raster_source(self): - #=========================== + def get_raster_sources(self) -> list[RasterSource]: + #================================================== if self.kind == 'base': # Only rasterise base source layer - return RasterSource('svg', self.get_raster_data) + return [RasterSource(f'{self.id}_image', 'svg', self.__get_raster_data, self)] + return [] def get_raster_data(self): #========================= diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index dbe4222d..256cd1fa 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -174,9 +174,17 @@ def create_preview(self): with open(cleaned_svg, 'wb') as fp: cleaner.save(fp) - def get_raster_source(self): - #=========================== - return RasterSource('svg', self.__get_raster_data, source_path=self.__source_file) + def get_raster_sources(self) -> list[RasterSource]: + #================================================== + raster_sources = [] + if (background := self.background_raster_source) is not None: + background_path = FilePath(background.href) + raster_sources.append(RasterSource(f'{self.id}_background', 'svg', background_path.get_data, self, + source_path=background_path, background_layer=True, + transform=Transform.translate(background.translate)@Transform.scale(background.scale))) + raster_sources.append(RasterSource(f'{self.id}_image', 'svg', self.__get_raster_data, self, + source_path=self.__source_file)) + return raster_sources def __get_raster_data(self) -> bytes: #==================================== From aae15c581c55b2370e2817fba5c4119e0ad86ab9 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 13:06:45 +1300 Subject: [PATCH 002/111] Implement a `feature-groups` section in `properties.json` to group features for detailed maps. --- mapmaker/flatmap/layers.py | 19 +++++++++++++++++++ mapmaker/properties/__init__.py | 17 +++++++++++++++++ mapmaker/sources/__init__.py | 1 + 3 files changed, 37 insertions(+) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 71cfcf7c..eca2e58d 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -194,6 +194,25 @@ def add_feature(self, feature: Feature): # type: ignore if feature.has_property('details'): self.__detail_features.append(feature) + def create_feature_groups(self): + #=============================== + for (group_id, feature_ids) in self.flatmap.properties_store.feature_groups(self.id).items(): + if len(feature_ids): + group_bounds = None + for feature_id in feature_ids: + if ((feature := self.flatmap.get_feature_by_name(feature_id)) is None + and (feature := self.flatmap.get_feature(feature_id)) is None): + log.warning('Cannot find source feature for feature group', group=group_id, feature=feature_id) + elif group_bounds is None: + group_bounds = feature.bounds + else: + group_bounds = merge_bounds(group_bounds, feature.bounds) + if group_bounds is not None: + self.flatmap.new_feature(self.id, shapely.box(*group_bounds), { + 'id': group_id, + 'exclude': True + }) + def add_raster_layers(self, layer_id: str, extent: MapBounds, map_source: 'MapSource', min_zoom: Optional[int]=None, local_world_to_base: Optional[Transform]=None): #================================================================================================== diff --git a/mapmaker/properties/__init__.py b/mapmaker/properties/__init__.py index 2076f445..7d510184 100644 --- a/mapmaker/properties/__init__.py +++ b/mapmaker/properties/__init__.py @@ -109,6 +109,16 @@ def __init__(self, flatmap: "FlatMap", manifest: "Manifest"): if manifest.proxy_features is not None else []) + # Feature groups by layer + self.__feature_groups: dict[str, dict[str, list[str]]] = {} + for group_defns in properties_dict.get('feature-groups', []): + if 'layer' in group_defns: + feature_groups: dict[str, list[str]] = {} + for group in group_defns.get('groups', []): + if 'id' in group: + feature_groups[group['id']] = group.get('features', []) + self.__feature_groups[group_defns['layer']] = feature_groups + @property def connectivity(self): return self.__pathways.connectivity @@ -129,6 +139,13 @@ def pathways(self): def proxies(self): return self.__proxies + """ + Get the feature group definitions for a layer + """ + def feature_groups(self, layer_id: str) -> dict[str, list[str]]: + #============================================================== + return self.__feature_groups.get(layer_id, {}) + def network_feature(self, feature): #================================== # Is the ``feature`` included in some network? diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 09c7504f..d535e677 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -231,6 +231,7 @@ def transform(self) -> Optional[Transform]: def add_layer(self, layer: 'MapLayer'): #====================================== + layer.create_feature_groups() self.__layers.append(layer) def create_preview(self): From aaf8bf5adb437f6775deddcfbaf43c2ad30055b7 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:23:43 +1300 Subject: [PATCH 003/111] Add Python type annotations and tidy code. --- mapmaker/flatmap/layers.py | 32 ++++++++--------- mapmaker/flatmap/manifest.py | 9 +++-- mapmaker/geometry/beziers.py | 42 ++++++++++++---------- mapmaker/output/__init__.py | 2 +- mapmaker/sources/mbfbioscience/__init__.py | 2 +- mapmaker/sources/powerpoint/__init__.py | 4 +-- 6 files changed, 47 insertions(+), 44 deletions(-) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index eca2e58d..2f89248a 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -144,11 +144,10 @@ def __init__(self, id: str, source: 'MapSource', exported=False, min_zoom: Optio self.__max_zoom = source.max_zoom @property - def boundary_feature(self): + def boundary_feature(self) -> Optional[Feature]: return self.__boundary_feature - @boundary_feature.setter - def boundary_feature(self, value): + def boundary_feature(self, value: Feature): self.__boundary_feature = value @property @@ -164,10 +163,10 @@ def detail_layer(self) -> bool: return (self.__source.base_feature is not None) @property - def max_zoom(self) -> Optional[int]: + def max_zoom(self) -> int: return self.__max_zoom @max_zoom.setter - def max_zoom(self, zoom): + def max_zoom(self, zoom: int): self.__max_zoom = zoom @property @@ -226,8 +225,9 @@ def add_raster_layers(self, layer_id: str, extent: MapBounds, map_source: 'MapSo min_zoom=min_zoom, local_world_to_base=local_world_to_base) for raster_source in map_source.raster_sources] - def add_group_features(self, group_name: str, features: list[Feature], tile_layer=FEATURES_TILE_LAYER, outermost=False) -> Optional[Feature]: - #=========================================================================================================================================== + def add_group_features(self, group_name: str, features: list[Feature], + tile_layer=FEATURES_TILE_LAYER, outermost=False) -> Optional[Feature]: + #============================================================================================ base_properties = { 'tile-layer': tile_layer } @@ -450,31 +450,31 @@ def background_layer(self) -> bool: return self.__background_layer @property - def extent(self): + def extent(self) -> MapBounds: return self.__extent @property - def flatmap(self): + def flatmap(self) -> 'FlatMap': return self.__flatmap @property - def id(self): + def id(self) -> str: return self.__id @property - def local_world_to_base(self): + def local_world_to_base(self) -> Optional[Transform]: return self.__local_world_to_base @property - def map_source(self): + def map_source(self) -> 'MapSource': return self.__map_source @property - def max_zoom(self): + def max_zoom(self) -> int: return self.__max_zoom @property - def min_zoom(self): + def min_zoom(self) -> int: return self.__min_zoom @property @@ -482,7 +482,7 @@ def source_data(self) -> bytes: return self.__raster_source.data @property - def source_extent(self): + def source_extent(self) -> MapBounds: return self.__map_source.extent @property @@ -494,7 +494,7 @@ def source_path(self) -> Optional[FilePath]: return self.__raster_source.source_path @property - def source_range(self): + def source_range(self) -> Optional[list[int]]: return self.__map_source.source_range @property diff --git a/mapmaker/flatmap/manifest.py b/mapmaker/flatmap/manifest.py index 256e2828..898c6377 100644 --- a/mapmaker/flatmap/manifest.py +++ b/mapmaker/flatmap/manifest.py @@ -178,9 +178,8 @@ class SourceBackground: class SourceManifest: def __init__(self, description: dict, manifest: 'Manifest'): self.__id = description['id'] - href = manifest.check_and_normalise_path(description['href'], 'Flatmap source file') - if href is None: - raise ValueError('Source in manifest has no `href`') + if (href := manifest.check_and_normalise_path(description.get('href'), 'Flatmap source file')) is None: + raise ValueError(f'Source {self.__id} in manifest has no `href`') self.__href = href self.__kind = description.get('kind', '') self.__boundary = description.get('boundary') @@ -418,8 +417,8 @@ def clean_up(self): if self.__temp_directory is not None: shutil.rmtree(self.__temp_directory) - def check_and_normalise_path(self, path: str, desc: str='') -> str|None: - #======================================================================= + def check_and_normalise_path(self, path: Optional[str], desc: str='') -> str|None: + #================================================================================= if path is None or path.strip() == '': return None normalised_path = self.__path.join_url(path) diff --git a/mapmaker/geometry/beziers.py b/mapmaker/geometry/beziers.py index 9548f9b6..8501f4a4 100644 --- a/mapmaker/geometry/beziers.py +++ b/mapmaker/geometry/beziers.py @@ -28,14 +28,18 @@ from beziers.segment import Segment as BezierSegment from shapely.geometry.base import BaseGeometry -import shapely.geometry +import shapely #=============================================================================== -def coords_to_point(pt: tuple[float, float]) -> BezierPoint: +type Coordinate = tuple[float, float] + +#=============================================================================== + +def coords_to_point(pt: Coordinate) -> BezierPoint: return BezierPoint(*pt) -def point_to_coords(pt: BezierPoint) -> tuple[float, float]: +def point_to_coords(pt: BezierPoint) -> Coordinate: return (pt.x, pt.y) #=============================================================================== @@ -47,12 +51,12 @@ def width_along_line(geometry: BaseGeometry, point: BezierPoint, dirn: BezierPoi point in a given direction. """ bounds = geometry.bounds - max_width = shapely.geometry.Point(*bounds[0:2]).distance(shapely.geometry.Point(*bounds[2:4])) - line = shapely.geometry.LineString([point_to_coords(point - dirn*max_width), + max_width = shapely.Point(*bounds[0:2]).distance(shapely.Point(*bounds[2:4])) + line = shapely.LineString([point_to_coords(point - dirn*max_width), point_to_coords(point + dirn*max_width)]) if geometry.intersects(line): intersection = geometry.boundary.intersection(line) - if isinstance(intersection, shapely.geometry.MultiPoint): + if isinstance(intersection, shapely.MultiPoint): intersecting_points = intersection.geoms if len(intersecting_points) == 2: return intersecting_points[0].distance(intersecting_points[1]) @@ -60,31 +64,31 @@ def width_along_line(geometry: BaseGeometry, point: BezierPoint, dirn: BezierPoi #=============================================================================== -def bezier_sample(bz, num_points=100): -#===================================== +def bezier_sample(bz, num_points=100) -> list[Coordinate]: +#========================================================= return [(pt.x, pt.y) for pt in bz.sample(num_points)] -def bezier_to_linestring(bz, num_points=100, offset=0): -#====================================================== - line = shapely.geometry.LineString(bezier_sample(bz, num_points)) +def bezier_to_linestring(bz, num_points=100, offset=0) -> shapely.LineString|shapely.MultiLineString: +#==================================================================================================== + line = shapely.LineString(bezier_sample(bz, num_points)) if offset == 0: return line else: return line.parallel_offset(abs(offset), 'left' if offset >= 0 else 'right') -def bezier_to_line_coords(bz, num_points=100, offset=0): -#======================================================= +def bezier_to_line_coords(bz, num_points=100, offset=0) -> list[Coordinate]: +#=========================================================================== line = bezier_to_linestring(bz, num_points=num_points, offset=offset) - if 'Multi' not in line.geom_type: - return line.coords - coords = [] + if isinstance(line, shapely.LineString): + return list(line.coords) + coords: list[Coordinate] = [] for l in line.geoms: coords.extend(l.coords if offset >= 0 else reversed(l.coords)) return coords #=============================================================================== -def bezier_connect(a: BezierPoint, b: BezierPoint, start_angle: float, end_angle: Optional[float]=None) -> CubicBezier: +def bezier_connect(a: BezierPoint, b: BezierPoint, start_angle: float, end_angle: Optional[float]=None) -> Optional[CubicBezier]: # Connect points ``a`` and ``b`` with a Bezier curve with a slope # at ``a`` of ``start_angle`` and a slope at ''b'' of ``pi + end_angle``. d = a.distanceFrom(b) @@ -97,7 +101,7 @@ def bezier_connect(a: BezierPoint, b: BezierPoint, start_angle: float, end_angle #=============================================================================== -def closest_time_distance(bz: 'BezierPath | BezierSegment', pt: BezierPoint, steps: int=100) -> tuple[float, float]: +def closest_time_distance(bz: BezierPath, pt: BezierPoint, steps: int=100) -> Coordinate: def subdivide_search(t0: float, t1: float, steps: int) -> tuple[float, float, float]: closest_d = -1 closest_t = t0 @@ -136,7 +140,7 @@ def set_bezier_path_end_to_point(bz_path: BezierPath, point: BezierPoint) -> flo #=============================================================================== -def split_bezier_path_at_point(bz_path: BezierPath, point: BezierPoint): +def split_bezier_path_at_point(bz_path: BezierPath, point: BezierPoint) -> tuple[BezierPath, BezierPath]: segments = bz_path.asSegments() # Find segment that is closest to the point closest_distance = None diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index 18d504a1..1e359efb 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -27,7 +27,7 @@ 'centreline', # bool 'children', # list[int] 'class', - 'colour', + 'colour', # str 'description', 'error', 'fc-class', diff --git a/mapmaker/sources/mbfbioscience/__init__.py b/mapmaker/sources/mbfbioscience/__init__.py index 54d42c3b..9453e01d 100644 --- a/mapmaker/sources/mbfbioscience/__init__.py +++ b/mapmaker/sources/mbfbioscience/__init__.py @@ -159,8 +159,8 @@ def process(self): self.__image = mask_image(self.__image, self.__world_to_image.transform_geometry(boundary_geometry)) - return RasterSource('image', lambda: self.__image) def get_raster_sources(self) -> list[RasterSource]: #================================================== + return [RasterSource(f'{self.id}_image', 'image', lambda: self.__image.tobytes(), self)] #=============================================================================== diff --git a/mapmaker/sources/powerpoint/__init__.py b/mapmaker/sources/powerpoint/__init__.py index 0ab5aff1..bc4ad48e 100644 --- a/mapmaker/sources/powerpoint/__init__.py +++ b/mapmaker/sources/powerpoint/__init__.py @@ -181,8 +181,8 @@ def get_raster_sources(self) -> list[RasterSource]: return [RasterSource(f'{self.id}_image', 'svg', self.__get_raster_data, self)] return [] - def get_raster_data(self): - #========================= + def __get_raster_data(self) -> bytes: + #==================================== svg_maker = SvgMaker(self.__powerpoint) svg_maker.add_slides(self.__slides) svg_data = BytesIO(svg_maker.svg_bytes()) From fb1686b169dfbed145e528b0a172985a2ea6ab24 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:23:59 +1300 Subject: [PATCH 004/111] Add missing import. --- mapmaker/sources/svg/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 256cd1fa..a256346d 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -19,6 +19,7 @@ #=============================================================================== import os +import math import tempfile import typing from typing import Optional From 9b4a20a2092e208a95be074879f80bacb60109ef Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:29:22 +1300 Subject: [PATCH 005/111] Has constants to specify colours of Shapes with errors. --- mapmaker/shapes/classify.py | 10 ++++++---- mapmaker/shapes/constants.py | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index b71dc294..f123749e 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -39,6 +39,8 @@ from mapmaker.utils import log from .constants import COMPONENT_BORDER_WIDTH, CONNECTION_STROKE_WIDTH, MAX_LINE_WIDTH +from .constants import SHAPE_ERROR_COLOUR, SHAPE_ERROR_BORDER + from .line_finder import Line, LineFinder, XYPair from .text_finder import TextFinder @@ -142,7 +144,7 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float connection_joiners.append(shape) elif not self.__add_connection(shape): log.warning('Unclassifiable shape', shape=shape.id) - shape.properties['colour'] = 'yellow' + shape.properties['colour'] = SHAPE_ERROR_COLOUR if not shape.properties.get('exclude', False): self.__shapes_by_type[shape.shape_type].append(shape) if shape.shape_type != SHAPE_TYPE.CONNECTION: @@ -160,8 +162,8 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float (connection_0, connection_1) = self.__extend_joined_connections(ends) joined_connection_graph.add_edge(connection_0, connection_1) else: - joiner.properties['colour'] = 'yellow' - joiner.properties['stroke'] = 'red' + joiner.properties['colour'] = SHAPE_ERROR_COLOUR + joiner.properties['stroke'] = SHAPE_ERROR_BORDER joiner.properties['stroke-width'] = COMPONENT_BORDER_WIDTH joiner.geometry = joiner.geometry.buffer(self.__max_line_width) for joined_connection in nx.connected_components(joined_connection_graph): @@ -195,7 +197,7 @@ def __add_connection(self, shape: Shape) -> bool: if 'Polygon' in shape.geometry.geom_type: if (line := self.__line_finder.get_line(shape)) is None: shape.properties['exclude'] = not settings.get('authoring', False) - shape.properties['colour'] = 'yellow' + shape.properties['colour'] = SHAPE_ERROR_COLOUR return False shape.geometry = line kind = VASCULAR_KINDS.lookup(shape.properties.get('fill')) diff --git a/mapmaker/shapes/constants.py b/mapmaker/shapes/constants.py index 97a06788..a85e587c 100644 --- a/mapmaker/shapes/constants.py +++ b/mapmaker/shapes/constants.py @@ -48,3 +48,8 @@ CONNECTION_STROKE_WIDTH = 2 #=============================================================================== + +SHAPE_ERROR_COLOUR = 'yellow' +SHAPE_ERROR_BORDER = 'red' + +#=============================================================================== From 3dcd84ece1df7554cbc44a1325f6b34a7a21d472 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:30:52 +1300 Subject: [PATCH 006/111] Put any superscript before any subscript when generating LaTeX for a Shape's text. --- mapmaker/shapes/text_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 472bf0be..596eef37 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -69,7 +69,7 @@ def get_text(self, shape: Shape) -> Optional[str]: superscript = self.__text_block_to_text(cluster.shapes) else: base_text = self.__text_block_to_text(cluster.shapes) - text = f'${base_text}{f"_{{{subscript}}}" if subscript != "" else ""}{f"^{{{superscript}}}" if superscript != "" else ""}$' + text = f'${base_text}{f"^{{{superscript}}}" if superscript != "" else ""}{f"_{{{subscript}}}" if subscript != "" else ""}$' return text if text != '' else None def __text_block_to_text(self, text_block: list[Shape]) -> str: From adc58327904817c2906627f44f1ef6ee59649158 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:32:41 +1300 Subject: [PATCH 007/111] Add a new layer to the flatmap **after** its source has been processed, so its features are available for grouping... --- mapmaker/sources/svg/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index a256346d..4ab25028 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -142,7 +142,6 @@ def __init__(self, flatmap: FlatMap, source_manifest: SourceManifest): # maker # southwest and northeast corners self.bounds = (top_left[0], bottom_right[1], bottom_right[0], top_left[1]) self.__layer = SVGLayer(self.id, self, svg, exported=self.__exported, min_zoom=self.min_zoom) - self.add_layer(self.__layer) self.__boundary_geometry = None @property @@ -162,6 +161,7 @@ def process(self): self.__layer.process() if self.__layer.boundary_feature is not None: self.__boundary_geometry = self.__layer.boundary_feature.geometry + self.add_layer(self.__layer) def create_preview(self): #======================== From 4441ac5fd0748221d765aadb4296862cb48c6630 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:34:48 +1300 Subject: [PATCH 008/111] Better detect arrows at ends of thin polygon shapes representing connections. --- mapmaker/shapes/constants.py | 5 +++++ mapmaker/shapes/line_finder.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/constants.py b/mapmaker/shapes/constants.py index a85e587c..c3097d5d 100644 --- a/mapmaker/shapes/constants.py +++ b/mapmaker/shapes/constants.py @@ -23,6 +23,11 @@ #=============================================================================== +# The ratio of the difference between the lengths of a candidate arrow's point +# edges and their sum must be less than 0.001 + +ARROW_POINT_EPSILON = 1e-3 + # The ratio of the actual overlap to the combined length of parallel edges needs # to be at least 0.6 for them to be candidates for merging into a line diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index d76d8c81..6f678bd9 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -362,7 +362,7 @@ def get_line(self, shape: Shape) -> Optional[LineString]: distances = [ p0.distance(p1) for (p0, p1) in itertools.pairwise(points + [points[0]])] for (n, (d0, d1)) in enumerate(itertools.pairwise([distances[-1]] + distances)): - if abs(d0 - d1) <= EPSILON: + if abs(d0 - d1)/(d0 + d1) <= ARROW_POINT_EPSILON: arrow_line = Line(points[n-2].midpoint(points[n-1]), points[n]) if arrow_line.p0.distance(line_points[0]) < arrow_line.p0.distance(line_points[-1]): line_points[0] = arrow_line.p1 From b0a3833994bb33e479902940f486124e70566707 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:42:42 +1300 Subject: [PATCH 009/111] Improve detection of connection lines drawn as long thin polygons. --- mapmaker/shapes/line_finder.py | 45 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index 6f678bd9..fc2e4050 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -34,7 +34,7 @@ from mapmaker.utils import log from mapmaker.shapes import Shape -from mapmaker.shapes.constants import EPSILON, LINE_OVERLAP_RATIO +from mapmaker.shapes.constants import ARROW_POINT_EPSILON, EPSILON, LINE_OVERLAP_RATIO from mapmaker.shapes.constants import MAX_PARALLEL_SKEW, MAX_LINE_WIDTH, MIN_LINE_ASPECT_RATIO #=============================================================================== @@ -276,19 +276,17 @@ def get_line(self, shape: Shape) -> Optional[LineString]: ends_graph = nx.Graph() used_lines: set[Line] = set() mid_lines: list[Line] = [] + unused_boundary_lines: set[Line] = set() boundary_coords = shape.geometry.boundary.simplify(self.__epsilon).coords - shapely.prepare(shape.geometry) - boundary_line_coords = zip(boundary_coords, boundary_coords[1:]) - for (line0, line1) in itertools.combinations(boundary_line_coords, 2): - l0 = Line.from_coords(line0) - l1 = Line.from_coords(line1) - if l0.parallel(l1): - p0 = HorizontalLine.from_line(l0) - p1 = p0.project(l1) - - if trace: - print('PAR', p0.separation(p1), self.__max_line_width, p0.overlap(p1), shape.id, p0, p1) + shapely.prepare(shape.geometry) + boundary_line_coord_pairs = zip(boundary_coords, boundary_coords[1:]) + boundary_lines = [Line.from_coords(coords) for coords in boundary_line_coord_pairs] + unused_boundary_lines = set(boundary_lines) + for (line0, line1) in itertools.combinations(boundary_lines, 2): + if line0.parallel(line1): + p0 = HorizontalLine.from_line(line0) + p1 = p0.project(line1) # reject if centroid of overlapping region isn't inside the shape's polygon if ((pt := p0.mid_point(p1)) is not None and shapely.contains_xy(shape.geometry, pt.x, pt.y)): @@ -296,12 +294,25 @@ def get_line(self, shape: Shape) -> Optional[LineString]: and p0.overlap(p1, True) > MIN_LINE_ASPECT_RATIO*w and p0.overlap(p1, False)/p0.overlap(p1, True) >= LINE_OVERLAP_RATIO): mid_lines.append(p0.mid_line(p1)) - used_lines.update([l0, l1]) - elif (pt := l0.intersection(l1)) is not None: - ends_graph.add_edge(l0, l1, intersection=pt) + used_lines.update([line0, line1]) + unused_boundary_lines.remove(line0) + unused_boundary_lines.remove(line1) + elif (pt := line0.intersection(line1)) is not None: + ends_graph.add_edge(line0, line1, intersection=pt) + + # ``Use`` any boundary line parallel to a mid-line and within + # MAX_LINE_WIDTH/2 of it + for ml in mid_lines: + for ul in unused_boundary_lines: + if ul.parallel(ml): + mh = HorizontalLine.from_line(ml) + uh = mh.project(ul) + if (uh.separation(mh) < self.__max_line_width/2 + and uh.overlap(mh, False) > self.__epsilon): + used_lines.add(ul) + ends_graph.remove_nodes_from(used_lines) - if len(mid_lines) == 1: - # Only a single line segment + if len(mid_lines) == 1: # Only a single line segment line_points = [mid_lines[0].p0, mid_lines[0].p1] elif len(mid_lines) == 0: line_points = [] From 669813bba9ac3d802f39db54a2c391bc08108102 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:46:00 +1300 Subject: [PATCH 010/111] Note if a possible connection line has a non-arrow shape along it. --- mapmaker/shapes/line_finder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index fc2e4050..fa3d54ec 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -382,6 +382,9 @@ def get_line(self, shape: Shape) -> Optional[LineString]: line_points[-1] = arrow_line.p1 shape.properties['directional'] = True break + elif len(end_nodes) != 1: + shape.properties['stroke'] = SHAPE_ERROR_COLOUR + log.warning('Bad arrow shape?', shape=shape.id, nodes=len(end_nodes)) return LineString([pt.coords for pt in line_points]) if len(line_points) >= 2 else None From 429518a0449cf5dfdd4cd65bfd3694711925f6eb Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:47:11 +1300 Subject: [PATCH 011/111] Add comment. --- mapmaker/sources/svg/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 4ab25028..e0a2f017 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -361,6 +361,9 @@ def __add_clip_geometry(self, clip_path_element, transform): and (geometry := self.__get_clip_geometry(clip_path_element, transform)) is not None): self.__clip_geometries.add(clip_id, geometry) + """ + Get the geometry described by the children of a ``clipPath`` element + """ def __get_clip_geometry(self, clip_path_element, transform) -> Optional[BaseGeometry]: #==================================================================================== geometries = [] From 7e21b0fe50853a6bdf5a39a2484b1331a07daf3a Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 21 Jan 2025 15:49:37 +1300 Subject: [PATCH 012/111] Handle SVG paths with multiple segments. --- mapmaker/sources/svg/utils.py | 75 +++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/mapmaker/sources/svg/utils.py b/mapmaker/sources/svg/utils.py index 8620697e..06a8b43b 100644 --- a/mapmaker/sources/svg/utils.py +++ b/mapmaker/sources/svg/utils.py @@ -44,7 +44,7 @@ from mapmaker.exceptions import MakerException from mapmaker.flatmap import Feature from mapmaker.geometry import Transform, reflect_point -from mapmaker.geometry.beziers import bezier_sample +from mapmaker.geometry.beziers import bezier_sample, Coordinate from mapmaker.geometry.arc_to_bezier import bezier_segments_from_arc_endpoints, tuple2 from mapmaker.output.path_colours import get_path_colour from mapmaker.utils import log @@ -234,10 +234,48 @@ def element_from_etree(svg: etree.Element) -> etree.Element: #=============================================================================== +def __geometry_from_coordinates(coordinates: list[Coordinate], closed: bool, must_close: Optional[bool]) -> Optional[BaseGeometry]: + if must_close == False and closed: + raise ValueError("Shape can't have closed geometry") + elif must_close == True and not closed: + raise ValueError("Shape must have closed geometry") + + if closed and len(coordinates) >= 3: + geometry = shapely.geometry.Polygon(coordinates).buffer(0) + elif must_close == True and len(coordinates) >= 3: + # Return a polygon if flagged as `closed` + coordinates.append(coordinates[0]) + geometry = shapely.geometry.Polygon(coordinates).buffer(0) + elif len(coordinates) >= 2: + ## Warn if start and end point are ``close`` wrt to the length of the line as shape + ## may be intended to be closed... (test with ``cardio_8-1``) + geometry = shapely.geometry.LineString(coordinates) + else: + geometry = None + + if geometry is not None and not geometry.is_valid: + if 'Polygon' in geometry.geom_type: + # Try smoothing out boundary irregularities + geometry = geometry.buffer(20) + if not geometry.is_valid: + log.error(f'{geometry.geom_type} geometry is invalid') + geometry = None + + return geometry + +#=============================================================================== + +type GeometricObject = tuple[Optional[BaseGeometry], list[BezierSegment]] + +#=============================================================================== + def geometry_from_svg_path(path_tokens: list[str|float], transform: Transform, - must_close: Optional[bool]=None) -> tuple[Optional[BaseGeometry], list[BezierSegment]]: - coordinates = [] - bezier_segments = [] + must_close: Optional[bool]=None) -> GeometricObject: + + geometries: list[BaseGeometry] = [] + + coordinates: list[Coordinate] = [] + bezier_segments: list[BezierSegment] = [] closed = False moved = False @@ -333,6 +371,12 @@ def geometry_from_svg_path(path_tokens: list[str|float], transform: Transform, current_point = pt elif cmd in ['m', 'M']: + if len(coordinates): + if (geometry := __geometry_from_coordinates(coordinates, closed, must_close)) is not None: + geometries.append(geometry) + coordinates: list[Coordinate] = [] + closed = False + params = [float(x) for x in path_tokens[pos:pos+2]] pos += 2 pt = params[0:2] @@ -386,25 +430,12 @@ def geometry_from_svg_path(path_tokens: list[str|float], transform: Transform, elif must_close == True and not closed: raise ValueError("Shape must have closed geometry") - if closed and len(coordinates) >= 3: - geometry = shapely.geometry.Polygon(coordinates).buffer(0) - elif must_close == True and len(coordinates) >= 3: - # Return a polygon if flagged as `closed` - coordinates.append(coordinates[0]) - geometry = shapely.geometry.Polygon(coordinates).buffer(0) - elif len(coordinates) >= 2: - ## Warn if start and end point are ``close`` wrt to the length of the line as shape - ## may be intended to be closed... (test with ``cardio_8-1``) - geometry = shapely.geometry.LineString(coordinates) - else: - geometry = None + if (geometry := __geometry_from_coordinates(coordinates, closed, must_close)) is not None: + geometries.append(geometry) - if geometry is not None and not geometry.is_valid: - if 'Polygon' in geometry.geom_type: - # Try smoothing out boundary irregularities - geometry = geometry.buffer(20) - if not geometry.is_valid: - raise ValueError(f'{geometry.geom_type} geometry is invalid') + geometry = (None if len(geometries) == 0 + else geometries[0] if len(geometries) == 1 + else shapely.unary_union(geometries)) return (geometry, bezier_segments) From ec5a70c0b1952a02197433912d9f340b0f53ca04 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 08:38:02 +1300 Subject: [PATCH 013/111] Tidy up how margins are added to functional map layers. --- mapmaker/sources/svg/__init__.py | 21 ++++++++++----- mapmaker/sources/svg/rasteriser.py | 43 +++++++++++++----------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index e0a2f017..42147577 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -59,7 +59,8 @@ #=============================================================================== -FUNCTIONAL_MAP_MARGIN = 200 # pixels +DETAILED_MAP_BORDER = 50 # pixels +FUNCTIONAL_MAP_MARGIN = 500 # pixels in SVG source space #=============================================================================== @@ -123,7 +124,9 @@ def __init__(self, flatmap: FlatMap, source_manifest: SourceManifest): # maker [0.0, 0.0, 1.0]])) self.__metres_per_pixel = scale else: - if self.flatmap.map_kind == MAP_KIND.FUNCTIONAL: + # Add a margin around the base layer of a functional map + if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL + and self.kind == 'base'): left -= FUNCTIONAL_MAP_MARGIN top -= FUNCTIONAL_MAP_MARGIN width += 2*FUNCTIONAL_MAP_MARGIN @@ -229,12 +232,16 @@ def __process_shapes(self, shapes: TreeList[Shape]) -> list[Feature]: # CellDL conversion mode... sc = ShapeClassifier(shapes.flatten(), self.source.map_area(), self.source.metres_per_pixel) shapes = TreeList(sc.classify()) - if self.source.base_feature is not None: + # Add a background shape behind a detailed functional map + if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL + and self.source.kind == 'functional'): bounds = self.source.bounds - margin = 0.02*(abs(bounds[2]-bounds[0]) - + abs(bounds[3]-bounds[1])) - bbox = shapely.geometry.box(*bounds).buffer(margin) - shapes.insert(0, Shape('background', bbox, { + margin = self.source.metres_per_pixel*DETAILED_MAP_BORDER + bounds = (bounds[0] + margin, bounds[1] - margin, + bounds[2] - margin, bounds[3] - margin) + bbox = shapely.geometry.box(*bounds).buffer(2*margin) + shapes.insert(0, Shape(None, bbox, { + 'id': 'background', 'tooltip': False, 'colour': 'white', 'kind': 'background' diff --git a/mapmaker/sources/svg/rasteriser.py b/mapmaker/sources/svg/rasteriser.py index ededc5ca..74ee4cc9 100644 --- a/mapmaker/sources/svg/rasteriser.py +++ b/mapmaker/sources/svg/rasteriser.py @@ -44,7 +44,7 @@ from mapmaker.settings import MAP_KIND from mapmaker.utils import FilePath, ProgressBar, log -from . import FUNCTIONAL_MAP_MARGIN, SVGSource +from . import DETAILED_MAP_BORDER, FUNCTIONAL_MAP_MARGIN, SVGSource from .definitions import DefinitionStore, ObjectStore from .styling import ElementStyleDict, StyleMatcher, wrap_element from .transform import SVGTransform @@ -308,13 +308,18 @@ def __init__(self, raster_layer: 'RasterLayer', tile_set: 'TileSet'): (left, top) = (0, 0) self.__size = (length_as_pixels(self.__svg.attrib['width']), length_as_pixels(self.__svg.attrib['height'])) - if (raster_layer.map_source.base_feature is None - and raster_layer.flatmap.map_kind == MAP_KIND.FUNCTIONAL): - left -= FUNCTIONAL_MAP_MARGIN - top -= FUNCTIONAL_MAP_MARGIN - self.__size = (self.__size[0] + 2*FUNCTIONAL_MAP_MARGIN, - self.__size[1] + 2*FUNCTIONAL_MAP_MARGIN) - + self.__add_background_rect = False + if raster_layer.flatmap.map_kind == MAP_KIND.FUNCTIONAL: + margin = 0 + if raster_layer.map_source.kind == 'base': + if not raster_layer.background_layer: + margin = FUNCTIONAL_MAP_MARGIN + elif raster_layer.map_source.kind == 'functional': + self.__add_background_rect = (raster_layer.map_source.base_feature is not None) + if margin: + left -= margin + top -= margin + self.__size = (self.__size[0] + 2*margin, self.__size[1] + 2*margin) self.__left_top = (left, top) self.__scaling = (tile_set.pixel_rect.width/self.__size[0], tile_set.pixel_rect.height/self.__size[1]) @@ -330,6 +335,7 @@ def __init__(self, raster_layer: 'RasterLayer', tile_set: 'TileSet'): [0.0, 0.0, 1.0]]) self.__svg_source = typing.cast(SVGSource, raster_layer.map_source) metres_per_pixel = self.__svg_source.metres_per_pixel + # Transform from SVG pixels to world coordinates self.__image_to_world = (Transform([ [metres_per_pixel/self.__scaling[0], 0, 0], @@ -338,18 +344,6 @@ def __init__(self, raster_layer: 'RasterLayer', tile_set: 'TileSet'): @np.array([[1.0, 0.0, -self.__scaling[0]*self.__size[0]/2.0], [0.0, -1.0, self.__scaling[1]*self.__size[1]/2.0], [0.0, 0.0, 1.0]])) -## ``image_to_world`` is used for rasterising details and may be wrong, esp. if the -## SVG's viewport origin is not (0, 0). -## -## The following might be correct, but needs testing... -## -## svg_origin = (left+self.__size[0]/2.0, top+self.__size[1]/2) -## @np.array([[1.0, 0.0, -svg_origin[0]], -## [0.0, -1.0, svg_origin[1]], -## [0.0, 0.0, 1.0]])) -## -## And do we need to multiply by scaling?? -## self.__tile_size = tile_set.tile_size self.__tile_origin = tile_set.start_coords self.__pixel_offset = tuple(tile_set.pixel_rect)[0:2] @@ -426,11 +420,10 @@ def __draw_svg(self, svg_to_tile_transform, show_progress=False) -> CanvasGroup: svg_to_tile_transform if transform is None else svg_to_tile_transform@transform, None, show_progress=show_progress) - if self.__svg_source.base_feature is not None: - margin = 0.02*(self.__size[0] + self.__size[1]) - path = skia.Path.RRect((self.__left_top[0] - margin, self.__left_top[1] - margin, - self.__size[0] + 2*margin, self.__size[1] + 2*margin), - margin, margin) + if self.__add_background_rect: + path = skia.Path.RRect((self.__left_top[0], self.__left_top[1], + self.__size[0], self.__size[1]), + DETAILED_MAP_BORDER/2, DETAILED_MAP_BORDER/2) paint = skia.Paint(AntiAlias=True, Color=make_colour('#FEFEFE', 1.0)) drawing_objects.insert(0, CanvasPath(path, paint, svg_to_tile_transform, transform, None)) return CanvasGroup(drawing_objects, svg_to_tile_transform, transform, None, outermost=True) From 0d3a6407288f981372c26bab5656da24fb8c65eb Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 13:40:44 +1300 Subject: [PATCH 014/111] A detailed source's manifest description can have an `alignment` list of pairs of feature ids/names, used to offset the layer relative to its base. --- mapmaker/flatmap/layers.py | 45 +++++++++++++++++++++++++++++++++-- mapmaker/flatmap/manifest.py | 5 ++++ mapmaker/geometry/__init__.py | 14 +++++++++++ mapmaker/output/geojson.py | 4 ++++ mapmaker/sources/__init__.py | 3 +++ 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 2f89248a..7bed6ef8 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -31,8 +31,9 @@ from mapmaker import ZOOM_OFFSET_FROM_BASE from mapmaker.exceptions import GroupValueError -from mapmaker.geometry import connect_dividers, extend_line, make_boundary, merge_bounds -from mapmaker.geometry import MapBounds, save_geometry, Transform +from mapmaker.geometry import connect_dividers, extend_line, make_boundary +from mapmaker.geometry import bounds_centroid, MapBounds, merge_bounds, translate_extent +from mapmaker.geometry import save_geometry, Transform from mapmaker.settings import settings from mapmaker.utils import FilePath, log @@ -98,6 +99,10 @@ def max_zoom(self) -> Optional[int]: def min_zoom(self) -> Optional[int]: return None + @property + def offset(self) -> tuple[float, float]: + return (0.0, 0.0) + @property def raster_layers(self) -> list['RasterLayer']: return [] @@ -142,6 +147,7 @@ def __init__(self, id: str, source: 'MapSource', exported=False, min_zoom: Optio self.__raster_layers: list[RasterLayer] = [] self.__min_zoom = min_zoom if min_zoom is not None else source.min_zoom self.__max_zoom = source.max_zoom + self.__offset = (0.0, 0.0) @property def boundary_feature(self) -> Optional[Feature]: @@ -173,6 +179,10 @@ def max_zoom(self, zoom: int): def min_zoom(self) -> int: return self.__min_zoom + @property + def offset(self) -> tuple[float, float]: + return self.__offset + @property def outer_geometry(self) -> BaseGeometry: return self.__outer_geometry @@ -193,6 +203,35 @@ def add_feature(self, feature: Feature): # type: ignore if feature.has_property('details'): self.__detail_features.append(feature) + def __find_feature(self, feature_id: str) -> Optional[Feature]: + #============================================================== + if (feature := self.flatmap.get_feature_by_name(feature_id)) is None: + feature = self.flatmap.get_feature(feature_id) + return feature + + def calculate_offset(self, feature_alignment: list[tuple[str, str]]): + #==================================================================== + base_feature_bounds = None + layer_feature_bounds = None + for (base_feature_id, layer_feature_id) in feature_alignment: + if (base_feature := self.__find_feature(base_feature_id)) is None: + log.warning('Cannot find base feature for layer alignment', layer=self.id, feature=base_feature_id) + elif base_feature_bounds is None: + base_feature_bounds = base_feature.bounds + else: + base_feature_bounds = merge_bounds(base_feature_bounds, base_feature.bounds) + if (layer_feature := self.__find_feature(layer_feature_id)) is None: + log.warning("Cannot find layer's feature for alignment", layer=self.id, feature=layer_feature_id) + elif layer_feature_bounds is None: + layer_feature_bounds = layer_feature.bounds + else: + layer_feature_bounds = merge_bounds(layer_feature_bounds, layer_feature.bounds) + if base_feature_bounds is not None and layer_feature_bounds is not None: + base_centroid = bounds_centroid(base_feature_bounds) + layer_centroid = bounds_centroid(layer_feature_bounds) + self.__offset = ((base_centroid[0] - layer_centroid[0]), + (base_centroid[1] - layer_centroid[1])) + def create_feature_groups(self): #=============================== for (group_id, feature_ids) in self.flatmap.properties_store.feature_groups(self.id).items(): @@ -221,6 +260,8 @@ def add_raster_layers(self, layer_id: str, extent: MapBounds, map_source: 'MapSo min_zoom = self.min_zoom if map_source.base_feature is not None: min_zoom -= ZOOM_OFFSET_FROM_BASE + if self.__offset != (0.0, 0.0): + extent = translate_extent(extent, self.__offset) self.__raster_layers = [RasterLayer(raster_source, extent, min_zoom=min_zoom, local_world_to_base=local_world_to_base) for raster_source in map_source.raster_sources] diff --git a/mapmaker/flatmap/manifest.py b/mapmaker/flatmap/manifest.py index 898c6377..018c097b 100644 --- a/mapmaker/flatmap/manifest.py +++ b/mapmaker/flatmap/manifest.py @@ -185,6 +185,7 @@ def __init__(self, description: dict, manifest: 'Manifest'): self.__boundary = description.get('boundary') self.__detail_fit = description.get('detail-fit') self.__feature = description.get('feature') + self.__alignment = [(features[0], features[1]) for features in description.get('alignment', [])] self.__source_range = (([int(n) for n in source_range] if isinstance(source_range, list) else [int(source_range)]) if (source_range := description.get('slides')) is not None @@ -199,6 +200,10 @@ def __init__(self, description: dict, manifest: 'Manifest'): else: self.__background_source = None + @property + def alignment(self) -> list[tuple[str, str]]: + return self.__alignment + @property def background_source(self) -> Optional[SourceBackground]: return self.__background_source diff --git a/mapmaker/geometry/__init__.py b/mapmaker/geometry/__init__.py index 188b279a..e864068d 100644 --- a/mapmaker/geometry/__init__.py +++ b/mapmaker/geometry/__init__.py @@ -70,8 +70,15 @@ def save_geometry(geo, file): # (SE, NW) bounds as decimal coordinates MapBounds = tuple[float, float, float, float] +# (SW, NE) bounds as lng/lat coordinates +MapExtent = tuple[float, float, float, float] + #=============================================================================== +def bounds_centroid(bounds: MapBounds) -> tuple[float, float]: +#============================================================= + return ((bounds[0] + bounds[2])/2, (bounds[1] + bounds[3])/2) + def bounds_to_extent(bounds): #============================ sw = mercator_transformer.transform(*bounds[:2]) @@ -93,6 +100,13 @@ def merge_bounds(bounds_0: MapBounds, bounds_1: MapBounds) -> MapBounds: return (min(bounds_0[0], bounds_1[0]), min(bounds_0[1], bounds_1[1]), max(bounds_0[2], bounds_1[2]), max(bounds_0[3], bounds_1[3])) +def translate_extent(extent: MapExtent, offset: tuple[float, float]) -> MapBounds: +#================================================================================= + bounds = extent_to_bounds(extent) + bds = (bounds[0] + offset[0], bounds[1] + offset[1], + bounds[2] + offset[0], bounds[3] + offset[1]) + return bounds_to_extent(bds) + #=============================================================================== class Transform(object): diff --git a/mapmaker/output/geojson.py b/mapmaker/output/geojson.py index 7e830131..1858d052 100644 --- a/mapmaker/output/geojson.py +++ b/mapmaker/output/geojson.py @@ -25,6 +25,7 @@ #=============================================================================== +import shapely.affinity import shapely.geometry #=============================================================================== @@ -74,6 +75,7 @@ def __save_features(self, features): unit='ftr', ncols=40, bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt}') + layer_offset = self.__layer.offset for feature in features: if not settings.get('authoring', False): feature.properties.pop('warning', None) @@ -94,6 +96,8 @@ def __save_features(self, features): if name in ENCODED_FEATURE_PROPERTIES }) geometry = feature.geometry + if layer_offset != (0.0, 0.0): + geometry = shapely.affinity.translate(geometry, xoff=layer_offset[0], yoff=layer_offset[1]) area = geometry.area mercator_geometry = mercator_transform(geometry) tile_layer = properties['tile-layer'] diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index d535e677..736d3cfa 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -150,6 +150,7 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): else: self.__min_zoom = flatmap.min_zoom self.__base_feature = None + self.__feature_alignment = source_manifest.alignment @property def annotator(self): @@ -232,6 +233,8 @@ def transform(self) -> Optional[Transform]: def add_layer(self, layer: 'MapLayer'): #====================================== layer.create_feature_groups() + if len(self.__feature_alignment): + layer.calculate_offset(self.__feature_alignment) self.__layers.append(layer) def create_preview(self): From fceae36332bbdc37bbd6e9d5a71b01c91891fa31 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 15:28:22 +1300 Subject: [PATCH 015/111] Add Python type annotations. --- mapmaker/flatmap/layers.py | 2 +- mapmaker/geometry/__init__.py | 35 ++++++++++++++++---------------- mapmaker/shapes/__init__.py | 2 +- mapmaker/sources/__init__.py | 17 ++++++++-------- mapmaker/sources/svg/__init__.py | 2 +- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 7bed6ef8..8e29299b 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -157,7 +157,7 @@ def boundary_feature(self, value: Feature): self.__boundary_feature = value @property - def bounds(self) -> tuple[float, float, float, float]: + def bounds(self) -> MapBounds: return self.__bounds @property diff --git a/mapmaker/geometry/__init__.py b/mapmaker/geometry/__init__.py index e864068d..5ad1dfc8 100644 --- a/mapmaker/geometry/__init__.py +++ b/mapmaker/geometry/__init__.py @@ -19,6 +19,7 @@ #=============================================================================== from math import acos, cos, sin, sqrt, pi as PI +from typing import Self import warnings #=============================================================================== @@ -67,7 +68,7 @@ def save_geometry(geo, file): warnings.simplefilter(action='default', category=FutureWarning) -# (SE, NW) bounds as decimal coordinates +# (SW, NE) bounds as decimal coordinates MapBounds = tuple[float, float, float, float] # (SW, NE) bounds as lng/lat coordinates @@ -79,20 +80,20 @@ def bounds_centroid(bounds: MapBounds) -> tuple[float, float]: #============================================================= return ((bounds[0] + bounds[2])/2, (bounds[1] + bounds[3])/2) -def bounds_to_extent(bounds): -#============================ +def bounds_to_extent(bounds: MapBounds) -> MapExtent: +#==================================================== sw = mercator_transformer.transform(*bounds[:2]) ne = mercator_transformer.transform(*bounds[2:]) return (sw[0], sw[1], ne[0], ne[1]) -def extent_to_bounds(extent): -#============================ +def extent_to_bounds(extent: MapExtent) -> MapBounds: +#==================================================== sw = mercator_transformer.transform(*extent[:2], direction=pyproj.enums.TransformDirection.INVERSE) # type: ignore ne = mercator_transformer.transform(*extent[2:], direction=pyproj.enums.TransformDirection.INVERSE) # type: ignore return (sw[0], sw[1], ne[0], ne[1]) -def mercator_transform(geometry): -#================================ +def mercator_transform(geometry: BaseGeometry) -> BaseGeometry: +#============================================================== return shapely.ops.transform(mercator_transformer.transform, geometry) def merge_bounds(bounds_0: MapBounds, bounds_1: MapBounds) -> MapBounds: @@ -116,7 +117,7 @@ def __init__(self, matrix): self.__matrix[1, 0:2], self.__matrix[0:2, 2]), axis=None).tolist() - def __matmul__(self, transform): + def __matmul__(self, transform) -> 'Transform': if isinstance(transform, Transform): return Transform(self.__matrix@np.array(transform.__matrix)) else: @@ -126,35 +127,35 @@ def __str__(self): return str(self.__matrix) @classmethod - def Identity(cls): + def Identity(cls) -> Self: return cls(np.identity(3)) @classmethod - def scale(cls, scale: float): + def scale(cls, scale: float) -> Self: return cls([[scale, 0, 0], [0, scale, 0], [0, 0, 1]]) @classmethod - def translate(cls, translate: tuple[float, float]): + def translate(cls, translate: tuple[float, float]) -> Self: return cls([[1, 0, translate[0]], [0, 1, translate[1]], [0, 0, 1]]) @property - def matrix(self): + def matrix(self) -> np.ndarray: return self.__matrix @property - def svg_matrix(self): + def svg_matrix(self) -> np.ndarray: return np.array([self.__matrix[0, 0], self.__matrix[1, 0], self.__matrix[0, 1], self.__matrix[1, 1], self.__matrix[0, 2], self.__matrix[1, 2]]) - def flatten(self): - #================= + def flatten(self) -> np.ndarray: + #=============================== return self.__matrix.flatten() - def inverse(self): - #================= + def inverse(self) -> 'Transform': + #================================ return Transform(np.linalg.inv(self.__matrix)) def rotate_angle(self, angle): diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index d376466a..dccc0eb6 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -139,7 +139,7 @@ def global_shape(self) -> 'Shape': # The shape that excluded this o return self.get_property('global-shape', self) @property - def id(self) -> str: + def id(self) -> Optional[str]: return self.__id @property diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 736d3cfa..808f6d68 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -30,12 +30,13 @@ from mapmaker.geometry import bounds_to_extent, MapBounds, Transform from mapmaker.flatmap import SourceBackground, SourceManifest, SOURCE_DETAIL_KINDS -from mapmaker.flatmap.layers import PATHWAYS_TILE_LAYER +from mapmaker.flatmap.layers import MapLayer, PATHWAYS_TILE_LAYER from mapmaker.properties.markup import parse_markup +from mapmaker.shapes import Shape from mapmaker.utils import FilePath if TYPE_CHECKING: - from mapmaker.flatmap import FlatMap, MapLayer + from mapmaker.flatmap import FlatMap #=============================================================================== @@ -131,7 +132,7 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): self.__kind = source_manifest.kind self.__source_range = source_manifest.source_range self.__errors: list[tuple[str, str]] = [] - self.__layers: list['MapLayer'] = [] + self.__layers: list[MapLayer] = [] self.__bounds: MapBounds = (0, 0, 0, 0) self.__raster_sources = None self.__background_raster_source = source_manifest.background_source @@ -201,7 +202,7 @@ def kind(self) -> str: return self.__kind @property - def layers(self) -> list['MapLayer']: + def layers(self) -> list[MapLayer]: return self.__layers @property @@ -230,8 +231,8 @@ def source_range(self) -> Optional[list[int]]: def transform(self) -> Optional[Transform]: return None - def add_layer(self, layer: 'MapLayer'): - #====================================== + def add_layer(self, layer: MapLayer): + #==================================== layer.create_feature_groups() if len(self.__feature_alignment): layer.calculate_offset(self.__feature_alignment) @@ -245,8 +246,8 @@ def error(self, kind: str, msg: str): #==================================== self.__errors.append((kind, msg)) - def filter_map_shape(self, shape): - #================================= + def filter_map_shape(self, shape: Shape): + #======================================== return def map_area(self) -> float: diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 42147577..f362d3eb 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -91,7 +91,7 @@ def __init__(self, flatmap: FlatMap, source_manifest: SourceManifest): # maker super().__init__(flatmap, source_manifest) self.__source_file = FilePath(source_manifest.href) self.__exported = (self.kind == 'base' or self.kind in SOURCE_DETAIL_KINDS) - svg = etree.parse(self.__source_file.get_fp()).getroot() + svg: etree.Element = etree.parse(self.__source_file.get_fp()).getroot() if 'viewBox' in svg.attrib: viewbox = [float(x) for x in svg.attrib.get('viewBox').split()] (left, top) = tuple(viewbox[:2]) From bb63d4155fcd869841665f93b956b006111a51ec Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 15:31:51 +1300 Subject: [PATCH 016/111] Cody tidying. --- mapmaker/flatmap/__init__.py | 8 ++++---- mapmaker/flatmap/layers.py | 9 ++++----- mapmaker/sources/__init__.py | 1 - mapmaker/sources/svg/__init__.py | 1 - 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 5e141001..1ce91361 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -354,7 +354,7 @@ def add_source_layers(self, layer_number: int, source: 'MapSource'): for layer in source.layers: self.add_layer(layer) if layer.exported: - layer.add_raster_layers(layer.id, source.extent, source) + layer.add_raster_layers(source.extent, source) # The first layer is used as the base map if layer_number == 0: if source.kind == 'details': @@ -475,9 +475,9 @@ def __add_detail_features(self, layer, detail_layer, lowres_features): if len(hires_layer.source.raster_sources): extent = transform.transform_extent(hires_layer.source.extent) - layer.add_raster_layers(f'{detail_layer.id}_{hires_layer.id}', - extent, hires_layer.source, minzoom, - local_world_to_base=transform) + layer.add_raster_layers(extent, hires_layer.source, + id=f'{detail_layer.id}_{hires_layer.id}', + min_zoom=minzoom, local_world_to_base=transform) # The detail layer gets a scaled copy of each high-resolution feature for hires_feature in hires_layer.features: diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 8e29299b..d1cdeaa7 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -238,9 +238,8 @@ def create_feature_groups(self): if len(feature_ids): group_bounds = None for feature_id in feature_ids: - if ((feature := self.flatmap.get_feature_by_name(feature_id)) is None - and (feature := self.flatmap.get_feature(feature_id)) is None): - log.warning('Cannot find source feature for feature group', group=group_id, feature=feature_id) + if (feature := self.__find_feature(feature_id)) is None: + log.warning('Cannot find source feature for feature group', layer=self.id, group=group_id, feature=feature_id) elif group_bounds is None: group_bounds = feature.bounds else: @@ -251,7 +250,7 @@ def create_feature_groups(self): 'exclude': True }) - def add_raster_layers(self, layer_id: str, extent: MapBounds, map_source: 'MapSource', + def add_raster_layers(self, extent: MapBounds, map_source: 'MapSource', layer_id: Optional[str]=None, min_zoom: Optional[int]=None, local_world_to_base: Optional[Transform]=None): #================================================================================================== if min_zoom is not None: @@ -475,7 +474,7 @@ class RasterLayer(object): """ def __init__(self, raster_source: 'RasterSource', extent: MapBounds, min_zoom:Optional[int]=None, max_zoom: Optional[int]=None, - local_world_to_base: Optional[Transform]=None): + local_world_to_base: Optional[Transform]=None): self.__id = raster_source.id self.__extent = extent self.__raster_source = raster_source diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 808f6d68..261127d5 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -18,7 +18,6 @@ # #=============================================================================== -from io import BytesIO from typing import Callable, Optional, TYPE_CHECKING #=============================================================================== diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index f362d3eb..e749fd71 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -432,7 +432,6 @@ def __process_element(self, wrapped_element: ElementWrapper, transform, parent_p return self.__process_group(wrapped_element, properties, transform, parent_style) elif element.tag == SVG_TAG('text'): geometry = self.__process_text(element, properties, transform) - if geometry is not None: return Shape(shape_id, geometry, properties, shape_type=SHAPE_TYPE.TEXT, svg_element=element) else: From 0cdaeed6cbda68a1644cdcf1826b129b3d83ce9f Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 15:32:54 +1300 Subject: [PATCH 017/111] `--max-raster-zoom` should only be for base map rasterisation, not detail maps. --- mapmaker/flatmap/layers.py | 2 +- mapmaker/maker.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index d1cdeaa7..8394b096 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -480,7 +480,7 @@ def __init__(self, raster_source: 'RasterSource', extent: MapBounds, self.__raster_source = raster_source self.__map_source = raster_source.map_source self.__flatmap = self.__map_source.flatmap - self.__max_zoom = max_zoom if max_zoom is not None else settings.get('maxRasterZoom', self.__map_source.max_zoom) + self.__max_zoom = max_zoom if max_zoom is not None else self.__map_source.max_zoom self.__min_zoom = min_zoom if min_zoom is not None else self.__map_source.min_zoom self.__local_world_to_base = local_world_to_base self.__background_layer = raster_source.background_layer diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 237e8aed..d1a92356 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -417,8 +417,11 @@ def __check_raster_tiles(self): tilemakers = [] for layer in self.__flatmap.layers: for raster_layer in layer.raster_layers: - tilemaker = RasterTileMaker(raster_layer, self.__map_dir, - settings.get('maxRasterZoom', layer.max_zoom)) + max_zoom = layer.max_zoom + if layer.source.kind == 'base': + # maxRasterZoom is only for base maps + max_zoom = settings.get('maxRasterZoom', max_zoom) + tilemaker = RasterTileMaker(raster_layer, self.__map_dir, max_zoom) tilemakers.append(tilemaker) if settings.get('backgroundTiles', False): tilemaker_process = tilemaker.make_tiles() From 64aab041184375eb6d5539a31a26891e263597ae Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 24 Jan 2025 15:35:08 +1300 Subject: [PATCH 018/111] Pass `detail-layer` property to viewer (with `image-layer` options). --- mapmaker/flatmap/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 1ce91361..0bcdaee8 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -381,7 +381,8 @@ def layer_metadata(self): 'options': { 'max-zoom': raster_layer.max_zoom, 'min-zoom': raster_layer.min_zoom, - 'background': raster_layer.background_layer + 'background': raster_layer.background_layer, + 'detail-layer': layer.detail_layer } } for raster_layer in layer.raster_layers ] From 4ba5a43b3431ec806b7104dd94d0387b018792f7 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 30 Jan 2025 10:55:30 +1300 Subject: [PATCH 019/111] Align features in a layer when adding a new layer to the map, not at output time. --- mapmaker/flatmap/layers.py | 11 +++++------ mapmaker/output/geojson.py | 3 --- mapmaker/sources/__init__.py | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 8394b096..67179386 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -179,10 +179,6 @@ def max_zoom(self, zoom: int): def min_zoom(self) -> int: return self.__min_zoom - @property - def offset(self) -> tuple[float, float]: - return self.__offset - @property def outer_geometry(self) -> BaseGeometry: return self.__outer_geometry @@ -209,8 +205,8 @@ def __find_feature(self, feature_id: str) -> Optional[Feature]: feature = self.flatmap.get_feature(feature_id) return feature - def calculate_offset(self, feature_alignment: list[tuple[str, str]]): - #==================================================================== + def align_layer(self, feature_alignment: list[tuple[str, str]]): + #=============================================================== base_feature_bounds = None layer_feature_bounds = None for (base_feature_id, layer_feature_id) in feature_alignment: @@ -231,6 +227,9 @@ def calculate_offset(self, feature_alignment: list[tuple[str, str]]): layer_centroid = bounds_centroid(layer_feature_bounds) self.__offset = ((base_centroid[0] - layer_centroid[0]), (base_centroid[1] - layer_centroid[1])) + if self.__offset != (0.0, 0.0): + for feature in self.features: + feature.geometry = shapely.affinity.translate(feature.geometry, xoff=self.__offset[0], yoff=self.__offset[1]) def create_feature_groups(self): #=============================== diff --git a/mapmaker/output/geojson.py b/mapmaker/output/geojson.py index 1858d052..fb4d0c1c 100644 --- a/mapmaker/output/geojson.py +++ b/mapmaker/output/geojson.py @@ -75,7 +75,6 @@ def __save_features(self, features): unit='ftr', ncols=40, bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt}') - layer_offset = self.__layer.offset for feature in features: if not settings.get('authoring', False): feature.properties.pop('warning', None) @@ -96,8 +95,6 @@ def __save_features(self, features): if name in ENCODED_FEATURE_PROPERTIES }) geometry = feature.geometry - if layer_offset != (0.0, 0.0): - geometry = shapely.affinity.translate(geometry, xoff=layer_offset[0], yoff=layer_offset[1]) area = geometry.area mercator_geometry = mercator_transform(geometry) tile_layer = properties['tile-layer'] diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 261127d5..c6ad1798 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -234,7 +234,7 @@ def add_layer(self, layer: MapLayer): #==================================== layer.create_feature_groups() if len(self.__feature_alignment): - layer.calculate_offset(self.__feature_alignment) + layer.align_layer(self.__feature_alignment) self.__layers.append(layer) def create_preview(self): From 700fefea8c89081415ab926f078c2fe0b1fe7aec Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 30 Jan 2025 17:44:52 +1300 Subject: [PATCH 020/111] A colour as specified in an SVG style might have leading blanks... --- mapmaker/sources/svg/rasteriser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapmaker/sources/svg/rasteriser.py b/mapmaker/sources/svg/rasteriser.py index 74ee4cc9..1e8475db 100644 --- a/mapmaker/sources/svg/rasteriser.py +++ b/mapmaker/sources/svg/rasteriser.py @@ -79,6 +79,7 @@ def round(x: float|int) -> int: #=============================================================================== def make_colour(colour_string, opacity=1.0): + colour_string = colour_string.strip() if colour_string.startswith('#'): colour = webcolors.hex_to_rgb(colour_string) elif colour_string.startswith('rgb('): From 8198e08eec70e4661c29331e7991dddeff3e0406 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 10:48:29 +1300 Subject: [PATCH 021/111] Tidy up shape classification code. --- mapmaker/shapes/classify.py | 52 +++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index f123749e..cbec12f8 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -152,29 +152,9 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float geometries.append(shape.geometry) shape.properties['stroke-width'] = COMPONENT_BORDER_WIDTH - connection_index = shapely.strtree.STRtree(self.__connection_ends) - joined_connection_graph = nx.Graph() - for joiner in connection_joiners: - ends = connection_index.query_nearest(joiner.geometry) #, max_distance=10*metres_per_pixel*MAX_LINE_WIDTH) - if len(ends) == 2: - joiner.properties['exclude'] = True - (connection_0, connection_1) = self.__extend_joined_connections(ends) - joined_connection_graph.add_edge(connection_0, connection_1) - else: - joiner.properties['colour'] = SHAPE_ERROR_COLOUR - joiner.properties['stroke'] = SHAPE_ERROR_BORDER - joiner.properties['stroke-width'] = COMPONENT_BORDER_WIDTH - joiner.geometry = joiner.geometry.buffer(self.__max_line_width) - for joined_connection in nx.connected_components(joined_connection_graph): - connections = list(joined_connection) - connected_line = shapely.line_merge(shapely.unary_union([conn.geometry for conn in connections])) - assert connected_line.geom_type == 'LineString', f'Cannot join connections: {[conn.id for conn in connections]}' - connections[0].geometry = connected_line - for connection in connections[1:]: - if connection.properties.get('directional', False): - connections[0].properties['directional'] = True - connection.properties['exclude'] = True + # If possible, join connections that share a triangular joiner + self.__join_connections(connection_joiners) self.__str_index = shapely.strtree.STRtree(geometries) geometries: list[BaseGeometry] = self.__str_index.geometries # type: ignore @@ -246,5 +226,33 @@ def classify(self) -> list[Shape]: if (label := self.__text_finder.get_text(shape)) is not None: shape.properties['label'] = label return [s for s in self.__shapes if not s.exclude] + def __join_connections(self, connection_joiners): + #================================================ + connection_index = shapely.strtree.STRtree(self.__connection_ends) + joined_connection_graph = nx.Graph() + for joiner in connection_joiners: + ends = connection_index.query_nearest(joiner.geometry) + if len(ends) == 2: + joiner.properties['exclude'] = True + (connection_0, connection_1) = self.__extend_joined_connections(ends) + joined_connection_graph.add_edge(connection_0, connection_1) + else: + joiner.properties['colour'] = SHAPE_ERROR_COLOUR + joiner.properties['stroke'] = SHAPE_ERROR_BORDER + joiner.properties['stroke-width'] = COMPONENT_BORDER_WIDTH + joiner.geometry = joiner.geometry.buffer(self.__max_line_width) + for joined_connection in nx.connected_components(joined_connection_graph): + connections = list(joined_connection) + connected_line = shapely.line_merge(shapely.unary_union([conn.geometry for conn in connections])) + assert connected_line.geom_type == 'LineString', f'Cannot join connections: {[conn.id for conn in connections]}' + + # Need to check all segments have the same colour... + + connections[0].geometry = connected_line + for connection in connections[1:]: + if connection.properties.get('directional', False): + connections[0].properties['directional'] = True + connection.properties['exclude'] = True + #=============================================================================== From 8c4e265f8c3f8fa6b91ac450d2dd84c2c8f8bc59 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 10:53:00 +1300 Subject: [PATCH 022/111] Improve parent/child detection in shape classification, esp. for text shapes. --- mapmaker/shapes/classify.py | 46 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index cbec12f8..48810ae4 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -105,7 +105,7 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float self.__connection_ends_to_shape: dict[int, ConnectionEnd] = {} self.__max_line_width = metres_per_pixel*MAX_LINE_WIDTH connection_joiners: list[Shape] = [] - geometries = [] + component_geometries = [] for n, shape in enumerate(shapes): geometry = shape.geometry area = geometry.area @@ -149,28 +149,19 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float self.__shapes_by_type[shape.shape_type].append(shape) if shape.shape_type != SHAPE_TYPE.CONNECTION: self.__geometry_to_shape[id(shape.geometry)] = shape - geometries.append(shape.geometry) + component_geometries.append(shape.geometry) shape.properties['stroke-width'] = COMPONENT_BORDER_WIDTH + # An index for component geometries + self.__component_index = shapely.strtree.STRtree(component_geometries) + self.__component_geometries: list[BaseGeometry] = self.__component_index.geometries # type: ignore # If possible, join connections that share a triangular joiner self.__join_connections(connection_joiners) - self.__str_index = shapely.strtree.STRtree(geometries) - geometries: list[BaseGeometry] = self.__str_index.geometries # type: ignore - parent_child = [] - for geometry in geometries: - if geometry.area > 0: - parent = self.__geometry_to_shape[id(geometry)] - for child in [self.__geometry_to_shape[id(geometries[c])] - for c in self.__str_index.query(geometry, predicate='contains_properly') - if geometries[c].area > 0]: - parent_child.append((parent, child)) - last_child_id = None - for (parent, child) in sorted(parent_child, key=lambda s: (s[1].id, s[0].geometry.area)): - if child.id != last_child_id: - child.add_parent(parent) - last_child_id = child.id + # Set parent/child relationship for components + self.__set_parent_relationships() + def __add_connection(self, shape: Shape) -> bool: #================================================ @@ -254,5 +245,26 @@ def __join_connections(self, connection_joiners): connections[0].properties['directional'] = True connection.properties['exclude'] = True + def __set_parent_relationships(self): + #==================================== + parent_child = [] + for geometry in self.__component_geometries: + if geometry.area > 0: + parent = self.__geometry_to_shape[id(geometry)] + bbox_intersecting_shapes = [self.__geometry_to_shape[id(self.__component_geometries[c])] + for c in self.__component_index.query(geometry) + if self.__component_geometries[c].area > 0] + for shape in bbox_intersecting_shapes: + if parent.shape_type != SHAPE_TYPE.TEXT and parent.id != shape.id: + # A text shape is always a child even when not properly contained + if (shape.shape_type == SHAPE_TYPE.TEXT + or shapely.contains_properly(parent.geometry, shape.geometry)): + parent_child.append((parent, shape)) + last_child_id = None + for (parent, child) in sorted(parent_child, key=lambda s: (s[1].id, s[0].geometry.area)): + # Sorted by child id with smallest parent first when there are multiple parents + if child.id != last_child_id: + child.add_parent(parent) + last_child_id = child.id #=============================================================================== From 92d053f47ede6cae23e78bcd28e4b700847ef6c9 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 10:58:38 +1300 Subject: [PATCH 023/111] Recognise `dashed` property when classifying functional connections. --- mapmaker/shapes/classify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 48810ae4..21fc6e35 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -183,7 +183,7 @@ def __add_connection(self, shape: Shape) -> bool: shape.properties['shape-type'] = SHAPE_TYPE.CONNECTION shape.properties['tile-layer'] = PATHWAYS_TILE_LAYER shape.properties['stroke-width'] = CONNECTION_STROKE_WIDTH - shape.properties['type'] = 'line' ## or 'line-dash' + shape.properties['type'] = 'line-dash' if shape.get_property('dashed', False) else 'line' return True def __append_connection_ends(self, end: shapely.Point, shape: Shape, index: int): From d22c414f6ce8eb7669230cc57b69a6f920608dd8 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 10:59:29 +1300 Subject: [PATCH 024/111] MultiPolygons aren't considered as possible connections when classifying shapes. --- mapmaker/shapes/classify.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 21fc6e35..1ff6dd3f 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -165,7 +165,9 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float def __add_connection(self, shape: Shape) -> bool: #================================================ - if 'Polygon' in shape.geometry.geom_type: + if shape.geometry.geom_type == 'MultiPolygon': + return False + elif 'Polygon' in shape.geometry.geom_type: if (line := self.__line_finder.get_line(shape)) is None: shape.properties['exclude'] = not settings.get('authoring', False) shape.properties['colour'] = SHAPE_ERROR_COLOUR From 9900bb6e9872cc3581a383c3c8ad057155d74e0d Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:04:05 +1300 Subject: [PATCH 025/111] Recognise `Annotation` and `Container` shapes during classification. --- mapmaker/shapes/classify.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 1ff6dd3f..c1cde216 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -131,16 +131,16 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float elif ((n < len(shapes) - 1) and shapes[n+1].shape_type == SHAPE_TYPE.TEXT and coverage < 0.5 and bbox_coverage < 0.001): shape.properties['exclude'] = True - elif coverage < 0.4 or 'LineString' in geometry.geom_type: + elif 'LineString' in geometry.geom_type or coverage < 0.4 and 'Multi' not in geometry.geom_type: if not self.__add_connection(shape): log.warning('Cannot extract line from polygon', shape=shape.id) elif bbox_coverage > 0.001 and coverage > 0.9: - shape.properties['shape-type'] = SHAPE_TYPE.CONTAINER - elif bbox_coverage < 0.0005 and aspect > 0.9 and 0.7 < coverage <= 0.85: + shape.properties['shape-type'] = SHAPE_TYPE.CONTAINER if bbox_coverage > 0.2 else SHAPE_TYPE.COMPONENT + elif bbox_coverage < 0.0003 and 0.7 < coverage <= 0.8: + shape.properties['shape-type'] = SHAPE_TYPE.ANNOTATION + elif bbox_coverage < 0.001 and coverage > 0.75: shape.properties['shape-type'] = SHAPE_TYPE.COMPONENT - elif bbox_coverage < 0.001 and coverage > 0.85: - shape.properties['shape-type'] = SHAPE_TYPE.COMPONENT - elif len(shape.geometry.boundary.coords) == 4: # A triangle + elif 'Multi' not in geometry.boundary.geom_type and len(shape.geometry.boundary.coords) == 4: # A triangle connection_joiners.append(shape) elif not self.__add_connection(shape): log.warning('Unclassifiable shape', shape=shape.id) @@ -162,6 +162,15 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float # Set parent/child relationship for components self.__set_parent_relationships() + # Assign text labels to components + for shape in self.__shapes: + if shape.shape_type in [SHAPE_TYPE.ANNOTATION, SHAPE_TYPE.COMPONENT]: + if (label_and_shapes := self.__text_finder.get_text(shape)) is not None: + shape.properties['label'] = label_and_shapes[0] + shape.properties['text-shapes'] = label_and_shapes[1] + # Although we do want their text, we don't want annotations to be active features + if shape.shape_type == SHAPE_TYPE.ANNOTATION: + shape.properties['exclude'] = True def __add_connection(self, shape: Shape) -> bool: #================================================ @@ -212,13 +221,6 @@ def __extend_joined_connections(self, ends: ndarray) -> tuple[Shape, Shape]: c1.shape.geometry = l1.line_string return (c0.shape, c1.shape) - def classify(self) -> list[Shape]: - #================================= - for shape in self.__shapes: - if shape.shape_type in [SHAPE_TYPE.COMPONENT, SHAPE_TYPE.CONTAINER]: - if (label := self.__text_finder.get_text(shape)) is not None: - shape.properties['label'] = label - return [s for s in self.__shapes if not s.exclude] def __join_connections(self, connection_joiners): #================================================ connection_index = shapely.strtree.STRtree(self.__connection_ends) From ac3684e39b73fa22ae232492489bc1747483be39 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:05:30 +1300 Subject: [PATCH 026/111] Set `source` and `target` properties of connections as part of classification. --- mapmaker/shapes/classify.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index c1cde216..02c2f1dd 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -162,7 +162,7 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float # Set parent/child relationship for components self.__set_parent_relationships() - # Assign text labels to components + # Assign text labels to components and source and target of connections for shape in self.__shapes: if shape.shape_type in [SHAPE_TYPE.ANNOTATION, SHAPE_TYPE.COMPONENT]: if (label_and_shapes := self.__text_finder.get_text(shape)) is not None: @@ -171,6 +171,10 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float # Although we do want their text, we don't want annotations to be active features if shape.shape_type == SHAPE_TYPE.ANNOTATION: shape.properties['exclude'] = True + elif shape.shape_type == SHAPE_TYPE.CONNECTION: + line_ends: shapely.geometry.base.GeometrySequence[shapely.MultiPoint] = shape.geometry.boundary.geoms # type: ignore + self.__connect_line_end(shape, line_ends[0], 'source') + self.__connect_line_end(shape, line_ends[1], 'target') def __add_connection(self, shape: Shape) -> bool: #================================================ @@ -203,6 +207,15 @@ def __append_connection_ends(self, end: shapely.Point, shape: Shape, index: int) self.__connection_ends.append(end_circle) self.__connection_ends_to_shape[id(end_circle)] = ConnectionEnd(shape, index) + def __connect_line_end(self, shape: Shape, end: shapely.Point, property: str): + #============================================================================= + for child in [self.__geometry_to_shape[id(self.__component_geometries[c])] + for c in self.__component_index.query(end.buffer(self.__max_line_width), predicate='intersects') + if self.__component_geometries[c].area > 0]: + if not child.exclude: + shape.set_property(property, child.id) + return + def __extend_joined_connections(self, ends: ndarray) -> tuple[Shape, Shape]: #=========================================================================== # Extend connection line ends so that they touch... From 2100470d641de1a5842b89dfa4113837a48c9860 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:05:59 +1300 Subject: [PATCH 027/111] Code tidying. --- mapmaker/shapes/classify.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 02c2f1dd..bb607970 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -189,16 +189,16 @@ def __add_connection(self, shape: Shape) -> bool: kind = VASCULAR_KINDS.lookup(shape.properties.get('fill')) else: kind = VASCULAR_KINDS.lookup(shape.properties.get('stroke')) - assert shape.geometry.geom_type == 'LineString', f'Connection not a LineString: {shape.id}' - line_ends: shapely.geometry.base.GeometrySequence[shapely.MultiPoint] = shape.geometry.boundary.geoms # type: ignore - self.__append_connection_ends(line_ends[0], shape, 0) - self.__append_connection_ends(line_ends[1], shape, -1) if kind is not None: shape.properties['kind'] = kind shape.properties['shape-type'] = SHAPE_TYPE.CONNECTION shape.properties['tile-layer'] = PATHWAYS_TILE_LAYER shape.properties['stroke-width'] = CONNECTION_STROKE_WIDTH shape.properties['type'] = 'line-dash' if shape.get_property('dashed', False) else 'line' + assert shape.geometry.geom_type == 'LineString', f'Connection not a LineString: {shape.id}' + line_ends: shapely.geometry.base.GeometrySequence[shapely.MultiPoint] = shape.geometry.boundary.geoms # type: ignore + self.__append_connection_ends(line_ends[0], shape, 0) + self.__append_connection_ends(line_ends[1], shape, -1) return True def __append_connection_ends(self, end: shapely.Point, shape: Shape, index: int): @@ -219,7 +219,6 @@ def __connect_line_end(self, shape: Shape, end: shapely.Point, property: str): def __extend_joined_connections(self, ends: ndarray) -> tuple[Shape, Shape]: #=========================================================================== # Extend connection line ends so that they touch... - c0 = self.__connection_ends_to_shape[id(self.__connection_ends[ends[0]])] c1 = self.__connection_ends_to_shape[id(self.__connection_ends[ends[1]])] l0 = LineString(c0.shape.geometry) From 7a57c69e6204af3d6f6cb3bf3c7e05c75cd22700 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:08:04 +1300 Subject: [PATCH 028/111] Add missing import. --- mapmaker/shapes/line_finder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index fa3d54ec..e6fbefc1 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -37,6 +37,8 @@ from mapmaker.shapes.constants import ARROW_POINT_EPSILON, EPSILON, LINE_OVERLAP_RATIO from mapmaker.shapes.constants import MAX_PARALLEL_SKEW, MAX_LINE_WIDTH, MIN_LINE_ASPECT_RATIO +from mapmaker.shapes.constants import SHAPE_ERROR_COLOUR + #=============================================================================== type Coordinate = tuple[float, float] From a0be2c4e6038b5e6b8235e248ed155762805eca4 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:08:41 +1300 Subject: [PATCH 029/111] Rework shape text finder to handle sub/superscripts within sub/superscripts. --- mapmaker/shapes/text_finder.py | 108 ++++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 596eef37..3a2c8c27 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -38,6 +38,10 @@ def __init__(self, shape: Optional[Shape]=None): def baseline(self) -> float: return self.__baselines/len(self.__shapes) if len(self.__shapes) else 0 + @property + def left(self) -> float: + return self.__shapes[0].left + @property def shapes(self) -> list[Shape]: return self.__shapes @@ -51,47 +55,119 @@ def left_sort_shapes(self): #=============================================================================== +class LatexMaker: + def __init__(self): + self.__latex = [] + self.__state = 0 + self.__subscripted = False + self.__text = [] + + @property + def latex(self) -> str: + #====================== + self.__make_latex() + return ''.join(self.__latex) + + def add_text(self, text: str, state: int): + #========================================= + if state != self.__state: + self.__make_latex() + self.__state = state + if text != '': + self.__text.append(text) + + def __make_latex(self): + #====================== + if len(self.__text): + if self.__state < 0: + self.__latex.append(f'_{{{''.join(self.__text)}}}') + self.__subscripted = True + elif self.__state > 0: + superscript = f'^{{{''.join(self.__text)}}}' + if self.__subscripted: + self.__latex.insert(-1, superscript) + else: + self.__latex.append(superscript) + else: + self.__latex.append(''.join(self.__text)) + self.__subscripted = False + self.__text = [] + +#=============================================================================== + class TextFinder: def __init__(self, scaling: float): self.__max_text_vertical_offset = scaling * MAX_TEXT_VERTICAL_OFFSET self.__text_baseline_offset = scaling * TEXT_BASELINE_OFFSET - def get_text(self, shape: Shape) -> Optional[str]: - #================================================= + def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: + #===================================================================== text_shapes = [s for s in shape.children if s.shape_type == SHAPE_TYPE.TEXT] text_clusters = self.__cluster_text(text_shapes) - text_baseline = (shape.geometry.bounds[1] + shape.geometry.bounds[3])/2 + self.__text_baseline_offset - base_text = superscript = subscript = '' + offset = self.__max_text_vertical_offset + baseline = (shape.geometry.bounds[1] + shape.geometry.bounds[3])/2 + self.__text_baseline_offset + state = 0 + clusters = [] + latex = LatexMaker() + used_text_shapes = [] for cluster in text_clusters: - if (cluster.baseline + self.__max_text_vertical_offset) < text_baseline: - subscript = self.__text_block_to_text(cluster.shapes) - elif (cluster.baseline - self.__max_text_vertical_offset) > text_baseline: - superscript = self.__text_block_to_text(cluster.shapes) + if cluster.baseline < (baseline - offset): + if state > 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + clusters.append(cluster) + state = -1 + elif cluster.baseline > (baseline + offset): + if state < 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + clusters.append(cluster) + state = 1 else: - base_text = self.__text_block_to_text(cluster.shapes) - text = f'${base_text}{f"^{{{superscript}}}" if superscript != "" else ""}{f"_{{{subscript}}}" if subscript != "" else ""}$' - return text if text != '' else None + if state != 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + latex.add_text(self.__text_block_to_text(cluster.shapes), 0) + state = 0 + used_text_shapes.extend(cluster.shapes) + if len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + text = f'${latex.latex}$' + return (text, used_text_shapes) if text != '' else None def __text_block_to_text(self, text_block: list[Shape]) -> str: #============================================================== return f'{''.join([s.text for s in text_block])}'.replace(' ', '\\ ') + def __text_clusters_to_text(self, text_clusters: list[TextShapeCluster]) -> str: + #=============================================================================== + baseline = text_clusters[0].baseline + offset = 0.9*self.__max_text_vertical_offset + latex = LatexMaker() + for cluster in text_clusters: + if cluster.baseline < (baseline - offset): + latex.add_text(self.__text_block_to_text(cluster.shapes), -1) + elif cluster.baseline > (baseline + offset): + latex.add_text(self.__text_block_to_text(cluster.shapes), 1) + else: + latex.add_text(self.__text_block_to_text(cluster.shapes), 0) + return latex.latex + def __cluster_text(self, text_shapes: list[Shape]) -> list[TextShapeCluster]: #============================================================================ - baseline_ordered_shapes = sorted(text_shapes, key=lambda s: s.baseline) + offset = self.__max_text_vertical_offset + left_ordered_shapes = sorted(text_shapes, key=lambda s: s.left) clusters: list[TextShapeCluster] = [] current_cluster = None - for shape in baseline_ordered_shapes: + for shape in left_ordered_shapes: if (current_cluster is None - or (shape.baseline - current_cluster.baseline) > self.__max_text_vertical_offset): + or abs(shape.baseline - current_cluster.baseline) > offset): current_cluster = TextShapeCluster(shape) clusters.append(current_cluster) else: # Note: ``current_cluster.baseline`` is monotonically increasing current_cluster.add_shape(shape) shape.properties['exclude'] = True - for cluster in clusters: - cluster.left_sort_shapes() return clusters #=============================================================================== From 58b040a8e3fe8ed4d9ace715a1b4c94cd52deab4 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:13:47 +1300 Subject: [PATCH 030/111] Use updated shape classification API. --- mapmaker/shapes/classify.py | 5 +++++ mapmaker/sources/svg/__init__.py | 9 +++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index bb607970..226ff560 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -176,6 +176,11 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float self.__connect_line_end(shape, line_ends[0], 'source') self.__connect_line_end(shape, line_ends[1], 'target') + @property + def shapes(self) -> list[Shape]: + #=============================== + return [s for s in self.__shapes if not s.exclude] + def __add_connection(self, shape: Shape) -> bool: #================================================ if shape.geometry.geom_type == 'MultiPolygon': diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index e749fd71..39c40519 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -224,14 +224,15 @@ def process(self): self.__transform, properties, None, show_progress=True) - features = self.__process_shapes(shapes) + self.__process_shapes(shapes) def __process_shapes(self, shapes: TreeList[Shape]) -> list[Feature]: #==================================================================== - if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL and not self.source.kind == 'anatomical'): + if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL + and not self.source.kind == 'anatomical'): # CellDL conversion mode... - sc = ShapeClassifier(shapes.flatten(), self.source.map_area(), self.source.metres_per_pixel) - shapes = TreeList(sc.classify()) + shape_classifier = ShapeClassifier(shapes.flatten(), self.source.map_area(), self.source.metres_per_pixel) + shapes = TreeList(shape_classifier.shapes) # Add a background shape behind a detailed functional map if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL and self.source.kind == 'functional'): From 2306072a0b7bf49ae3f0ae85cb6a49a6a4c5d37f Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:15:06 +1300 Subject: [PATCH 031/111] Add `background` as a markup property -- background shapes don't become features but are rasterised. --- mapmaker/properties/markup.py | 3 ++- mapmaker/shapes/classify.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mapmaker/properties/markup.py b/mapmaker/properties/markup.py index c2dd40b0..76c4c270 100644 --- a/mapmaker/properties/markup.py +++ b/mapmaker/properties/markup.py @@ -74,7 +74,8 @@ FEATURE_PROPERTIES = CLASS | CHILDCLASSES | IDENTIFIER | NAME | STYLE -SHAPE_FLAGS = Group(Keyword('boundary') +SHAPE_FLAGS = Group(Keyword('background') + | Keyword('boundary') | Keyword('closed') | Keyword('exterior') | Keyword('interior') diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 226ff560..5a138d4e 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -107,6 +107,9 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float connection_joiners: list[Shape] = [] component_geometries = [] for n, shape in enumerate(shapes): + if shape.get_property('background', False): + shape.set_property('exclude', True) + continue geometry = shape.geometry area = geometry.area self.__bounds = geometry.bounds From b1d9038ff362bd2668885abf7bcab82fd14c6697 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 11:25:03 +1300 Subject: [PATCH 032/111] Generate CellDL from a functional model when `--export-bondgraphs`. --- README.rst | 6 +- mapmaker/__main__.py | 2 + mapmaker/knowledgebase/celldl.py | 121 ++++++++++------ mapmaker/sources/celldl/__init__.py | 174 ++++++++++++++++++++++++ mapmaker/sources/celldl/definitions.py | 109 +++++++++++++++ mapmaker/sources/powerpoint/svgutils.py | 2 +- mapmaker/sources/svg/__init__.py | 13 +- mapmaker/utils/__init__.py | 19 +++ mapmaker/utils/svg.py | 9 +- 9 files changed, 406 insertions(+), 49 deletions(-) create mode 100644 mapmaker/sources/celldl/__init__.py create mode 100644 mapmaker/sources/celldl/definitions.py diff --git a/README.rst b/README.rst index 655dbafd..c45c2c11 100644 --- a/README.rst +++ b/README.rst @@ -113,8 +113,8 @@ Command line help [--authoring] [--debug] [--only-networks] [--save-drawml] [--save-geojson] [--tippecanoe] [--initial-zoom N] [--max-zoom N] - [--export-features EXPORT_FILE] [--export-neurons EXPORT_FILE] [--export-svg EXPORT_FILE] - [--single-file {celldl,svg}] + [--export-bondgraphs] [--export-features EXPORT_FILE] [--export-neurons EXPORT_FILE] + [--export-svg EXPORT_FILE] [--single-file {celldl,svg}] --output OUTPUT --source SOURCE Generate a flatmap from its source manifest. @@ -162,6 +162,8 @@ Command line help --max-zoom N Maximum zoom level (defaults to 10) Miscellaneous: + --export-bondgraphs + Export functional modelling components as CellDL bondgraphs --export-features EXPORT_FILE Export identifiers and anatomical terms of labelled features as JSON --export-neurons EXPORT_FILE diff --git a/mapmaker/__main__.py b/mapmaker/__main__.py index 0235e4bc..5cec38c0 100644 --- a/mapmaker/__main__.py +++ b/mapmaker/__main__.py @@ -89,6 +89,8 @@ def arg_parser(): misc_options = parser.add_argument_group('Miscellaneous') misc_options.add_argument('--commit', metavar='GIT_COMMIT', help='The branch/tag/commit to use when the source is a Git repository') + misc_options.add_argument('--export-bondgraphs', dest='exportBondgraphs', action='store_true', + help='Export functional modelling components as CellDL bondgraphs') misc_options.add_argument('--export-features', dest='exportFeatures', metavar='EXPORT_FILE', help='Export identifiers and anatomical terms of labelled features as JSON') misc_options.add_argument('--export-neurons', dest='exportNeurons', metavar='EXPORT_FILE', diff --git a/mapmaker/knowledgebase/celldl.py b/mapmaker/knowledgebase/celldl.py index 4b784ed2..e55b66d1 100644 --- a/mapmaker/knowledgebase/celldl.py +++ b/mapmaker/knowledgebase/celldl.py @@ -19,6 +19,8 @@ #=============================================================================== import base64 +from datetime import datetime, UTC +from typing import Optional import zlib #=============================================================================== @@ -27,12 +29,12 @@ #=============================================================================== -from mapmaker.shapes import Shape +from mapmaker.shapes import Shape, SHAPE_TYPE from mapmaker.utils.svg import svg_id #=============================================================================== -CELLDL_SCHEMA_VERSION = '1.0' +CELLDL_SCHEMA_VERSION = '2.0' #=============================================================================== @@ -107,21 +109,33 @@ class FC_KIND: #=============================================================================== -RDF = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') -RDFS = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') +DCT_NS = rdflib.Namespace('http://purl.org/dc/terms/') +RDF_NS = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +RDFS_NS = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') -CELLDL = rdflib.Namespace('http://celldl.org/ontologies/celldl#') -FC = rdflib.Namespace('http://celldl.org/ontologies/functional-connectivity#') +#=============================================================================== + +BG_NS = rdflib.Namespace('http://celldl.org/ontologies/bond-graph#') +CELLDL_NS = rdflib.Namespace('http://celldl.org/ontologies/celldl#') +FC_NS = rdflib.Namespace('http://celldl.org/ontologies/functional-connectivity#') + +STANDARD_NAMESPACES = { + 'celldl': CELLDL_NS, + 'dct': DCT_NS +} -FLATMAP = rdflib.Namespace('#') +KNOWN_NAMESPACES = { + 'bg': BG_NS, + 'fc': FC_NS, +} #=============================================================================== -CELLDL_TYPE = { - CD_CLASS.CONDUIT: CELLDL.Conduit, - CD_CLASS.CONNECTION: CELLDL.Connection, - CD_CLASS.CONNECTOR: CELLDL.Connector, - CD_CLASS.COMPONENT: CELLDL.Component, +CELLDL_TYPE_FROM_CLASS = { + CD_CLASS.CONDUIT: CELLDL_NS.Conduit, + CD_CLASS.CONNECTION: CELLDL_NS.Connection, + CD_CLASS.CONNECTOR: CELLDL_NS.Connector, + CD_CLASS.COMPONENT: CELLDL_NS.Component, } #=============================================================================== @@ -130,47 +144,74 @@ class FC_KIND: #=============================================================================== +CELLDL_CLASS_FROM_SHAPE_TYPE = { + SHAPE_TYPE.COMPONENT: CD_CLASS.COMPONENT, + SHAPE_TYPE.CONNECTION: CD_CLASS.CONNECTION, + SHAPE_TYPE.CONTAINER: CD_CLASS.COMPONENT +} + +#=============================================================================== + +DIAGRAM_NS = rdflib.Namespace('#') + +def make_uri(shape_id: str) -> rdflib.URIRef: + return DIAGRAM_NS[svg_id(shape_id)] + +#=============================================================================== + class CellDLGraph: - def __init__(self): + def __init__(self, diagram_type: Optional[rdflib.URIRef]=None): self.__graph = rdflib.Graph() - self.__graph.bind('celldl', str(CELLDL)) - self.__graph.bind('fc', str(FC)) - this = FLATMAP[''] - self.__graph.add((this, RDF.type, CELLDL.Document)) - self.__graph.add((this, CELLDL.schema, rdflib.Literal(CELLDL_SCHEMA_VERSION))) - self.__graph.add((this, RDF.type, FC.Diagram)) - - def add_metadata(self, shape: Shape): - #==================================== - if shape.exclude or shape.cd_class not in CELLDL_TYPE: + self.__diagram = make_uri('') + self.__graph.bind('', str(DIAGRAM_NS)) + for (prefix, ns) in STANDARD_NAMESPACES.items(): + self.__graph.bind(prefix, str(ns)) + self.__graph.add((self.__diagram, RDF_NS.type, CELLDL_NS.Document)) + self.__graph.add((self.__diagram, CELLDL_NS.schema, rdflib.Literal(CELLDL_SCHEMA_VERSION))) + if diagram_type is not None: + for (prefix, ns) in KNOWN_NAMESPACES.items(): + if str(diagram_type).startswith(str(ns)): + self.__graph.bind(prefix, str(ns)) + self.__graph.add((self.__diagram, RDF_NS.type, diagram_type)) + self.__graph.add((self.__diagram, DCT_NS.created, rdflib.Literal(datetime.now(UTC).isoformat()))) + + def add_shape(self, shape: Shape) -> Optional[str]: + #================================================== + if (shape.exclude + or shape.shape_type not in CELLDL_CLASS_FROM_SHAPE_TYPE + or shape.id is None): return - this = FLATMAP[svg_id(shape.id)] - self.__graph.add((this, RDF.type, CELLDL_TYPE[shape.cd_class])) - self.__graph.add((this, RDF.type, FC[shape.fc_class.split(':')[-1]])) + this = make_uri(shape.id) + celldl_class = CELLDL_CLASS_FROM_SHAPE_TYPE[shape.shape_type] + self.__graph.add((this, RDF_NS.type, CELLDL_TYPE_FROM_CLASS[celldl_class])) if shape.label: - self.__graph.add((this, RDFS.label, rdflib.Literal(shape.label))) ## add port/node in XXX ?? + self.__graph.add((this, RDFS_NS.label, rdflib.Literal(shape.label))) ## add port/node in XXX ?? if (shape.name and (shape.label is None or shape.name.lower() != shape.label.lower())): - self.__graph.add((this, RDFS.comment, rdflib.Literal(shape.name))) - ## models (layers...) - if shape.cd_class == CD_CLASS.CONNECTION: - if shape.fc_class == FC_CLASS.NEURAL: - self.__graph.add((this, FC.connectionType, rdflib.Literal(shape.path_type.name))) - for id in shape.connector_ids: - self.__graph.add((this, CELLDL.hasConnector, FLATMAP[svg_id(id)])) - for id in shape.intermediate_connectors: - self.__graph.add((this, CELLDL.hasIntermediate, FLATMAP[svg_id(id)])) - for id in shape.intermediate_components: - self.__graph.add((this, CELLDL.hasIntermediate, FLATMAP[svg_id(id)])) + self.__graph.add((this, RDFS_NS.comment, rdflib.Literal(shape.name))) + if celldl_class == CD_CLASS.CONNECTION: + if (source := shape.get_property('source')) is not None: + self.__graph.add((this, CELLDL_NS.hasSource, make_uri(source))) + if (target := shape.get_property('target')) is not None: + self.__graph.add((this, CELLDL_NS.hasTarget, make_uri(target))) + return celldl_class.replace(':', '-') def as_encoded_turtle(self): #=========================== turtle = self.__graph.serialize(format='turtle', encoding='utf-8') return f'{GZIP_BASE64_DATA_URI}{base64.b64encode(zlib.compress(turtle)).decode()}' - def as_xml(self): - #================ + def as_turtle(self) -> bytes: + #============================ + return self.__graph.serialize(format='turtle', encoding='utf-8') + + def as_xml(self) -> bytes: + #========================= return self.__graph.serialize(format='xml', encoding='utf-8') + def set_property(self, property: rdflib.URIRef, value: rdflib.Literal|rdflib.URIRef): + #==================================================================================== + self.__graph.add((self.__diagram, property, value)) + #=============================================================================== diff --git a/mapmaker/sources/celldl/__init__.py b/mapmaker/sources/celldl/__init__.py new file mode 100644 index 00000000..b0d1014f --- /dev/null +++ b/mapmaker/sources/celldl/__init__.py @@ -0,0 +1,174 @@ +#=============================================================================== +# +# Flatmap viewer and annotation tools +# +# Copyright (c) 2020 - 2025 David Brooks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#=============================================================================== + +import copy +import itertools +from pathlib import Path +from typing import Optional + +#=============================================================================== + +import lxml.etree as etree +import rdflib + +#=============================================================================== + +from mapmaker.geometry import Transform +from mapmaker.knowledgebase.celldl import BG_NS, CellDLGraph, DCT_NS +from mapmaker.shapes import Shape, SHAPE_TYPE +from mapmaker.sources.svg.utils import length_as_pixels +from mapmaker.utils import TreeList, SVG_NS +from mapmaker.utils.svg import svg_id + +from .definitions import CELLDL_DEFINITIONS_ID, CELLDL_LAYER_CLASS, CELLDL_METADATA_ID, CELLDL_STYLESHEET_ID, DIAGRAM_LAYER +from .definitions import BondgraphStylesheet, BondgraphSvgDefinitions, CellDLStylesheet + +#=============================================================================== + +class EXPORT_TYPE: + BONDGRAPH = 'bondgraph' + +#=============================================================================== + +MAXIMIMUM_SMALLEST_SVG_DIMENSION = 1200 + +#=============================================================================== + +# If successive x-coordinates (y-coordinates) of a path are within this distance then the +# connecting line segment is considered to be horizontal (vertical) + +PATH_EPSILON = 0.5 # pixels + +#=============================================================================== + +etree.register_namespace('svg', str(SVG_NS)) + +#=============================================================================== + +class CellDLExporter: + def __init__(self, svg_element: etree.Element, source_href: str, world_to_pixels: Transform, + export_type: Optional[str]=EXPORT_TYPE.BONDGRAPH): + self.__celldl = CellDLGraph(BG_NS.Model if export_type == EXPORT_TYPE.BONDGRAPH else None) + self.__celldl.set_property(DCT_NS.source, rdflib.URIRef(source_href)) + self.__svg_element = svg_element + self.__world_to_pixels = world_to_pixels + self.__export_type = export_type + celldl_defs = svg_element.find(f'.//{SVG_NS.defs}[@id="{CELLDL_DEFINITIONS_ID}"]') + if celldl_defs is None: + celldl_defs = svg_element.find(f'.//{SVG_NS.defs}') + if celldl_defs is not None: + celldl_defs.attrib['id'] = CELLDL_DEFINITIONS_ID + else: + celldl_defs = etree.SubElement(svg_element, SVG_NS.defs, { + 'id': CELLDL_DEFINITIONS_ID + }) + if export_type == EXPORT_TYPE.BONDGRAPH: + celldl_defs.extend(BondgraphSvgDefinitions) + + celldl_style = celldl_defs.find(f'.//{SVG_NS.style}[@id="{CELLDL_STYLESHEET_ID}"]') + if celldl_style is None: + celldl_style = etree.SubElement(celldl_defs, SVG_NS.style, { + 'id': CELLDL_STYLESHEET_ID + }) + stylesheets = [CellDLStylesheet] + if export_type == EXPORT_TYPE.BONDGRAPH: + stylesheets.append(BondgraphStylesheet) + celldl_style.text = '\n'.join(stylesheets) + + diagram = etree.Element(SVG_NS.g, { + 'id': DIAGRAM_LAYER, + 'class': CELLDL_LAYER_CLASS + }) + viewbox = self.__check_viewbox() + for child in svg_element: + if child.tag != SVG_NS.defs: + diagram.append(child) + svg_element.append(diagram) # Need to append after above copy/move + self.__connection_group = etree.SubElement(diagram, SVG_NS.g) + self.__metadata_element = etree.Element(SVG_NS.metadata, { + 'id': CELLDL_METADATA_ID, + 'data-content-type': 'text/turtle' + }) + svg_element.insert(0, self.__metadata_element) + + def process(self, shapes: TreeList[Shape]): + #========================================== + self.__process_shape_list(shapes) ##, self.__diagram) + + def save(self, path: Path): + #========================== + self.__metadata_element.text = etree.CDATA(self.__celldl.as_turtle()) + svg_tree = etree.ElementTree(self.__svg_element) + svg_tree.write(path, + encoding='utf-8', #inclusive_ns_prefixes=['svg'], + pretty_print=True, xml_declaration=True) + + def __check_viewbox(self) -> tuple[float, float, float, float]: + #============================================================== + width = self.__svg_element.attrib.pop('width', None) + height = self.__svg_element.attrib.pop('height', None) + if 'viewBox' not in self.__svg_element.attrib: + self.__svg_element.attrib['viewBox'] = f'0 0 {length_as_pixels(width):.1f} {length_as_pixels(height):.1f}' + return tuple(float(i) for i in self.__svg_element.attrib['viewBox'].split()) # type: ignore + + def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element): + #======================================================= + for shape in shapes[0:]: + if isinstance(shape, TreeList): + self.__process_shape_list(shape) + elif not shape.properties.get('exclude', False): + if (svg_element := shape.get_property('svg-element')) is not None: + element_id = svg_id(shape.id) + if (svg_class := self.__celldl.add_shape(shape)) is not None: + if shape.shape_type == SHAPE_TYPE.CONNECTION: + geometry = self.__world_to_pixels.transform_geometry(shape.geometry) + coords = [f'{coord[0]} {coord[1]}' for coord in geometry.coords] + classes = [svg_class] + connection_style = 'rectilinear' + for (c1, c2) in itertools.pairwise(geometry.coords): + if (abs(c1[0] - c2[0]) > PATH_EPSILON + and abs(c1[1] - c2[1]) > PATH_EPSILON): + connection_style = 'linear' + break + classes.append(connection_style) + + if self.__export_type == EXPORT_TYPE.BONDGRAPH: + classes.append('bondgraph') + svg_element.getparent().remove(svg_element) + svg_element = etree.SubElement(self.__connection_group, SVG_NS.path, { + 'id': element_id, + 'class': ' '.join(classes), + }) + svg_element.attrib['d'] = f'M{coords[0]}L{"L".join(coords[1:])}' + elif (text_shapes := shape.get_property('text-shapes')) is not None: + shape_element = copy.deepcopy(svg_element) + svg_element.tag = SVG_NS.g + svg_element.attrib.clear() + svg_element.attrib['id'] = element_id + svg_element.attrib['class'] = svg_class + svg_element.append(shape_element) + svg_element.extend([element for text_shape in text_shapes + if (element := text_shape.get_property('svg-element')) is not None]) + shape.set_property('svg-element', svg_element) + else: + svg_element.attrib['id'] = element_id + svg_element.attrib['class'] = ' '.join(svg_element.attrib.get('class', '').split() + [svg_class]) + +#=============================================================================== diff --git a/mapmaker/sources/celldl/definitions.py b/mapmaker/sources/celldl/definitions.py new file mode 100644 index 00000000..b09085ed --- /dev/null +++ b/mapmaker/sources/celldl/definitions.py @@ -0,0 +1,109 @@ +#=============================================================================== +# +# Flatmap viewer and annotation tools +# +# Copyright (c) 2020 - 2025 David Brooks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#=============================================================================== + +import lxml.etree as etree + +#=============================================================================== + +from mapmaker.utils import SVG_NS + +#=============================================================================== + +EM_SIZE = 16 # Pixels, sets ``font-size`` in CellDLStylesheet +INTERFACE_PORT_RADIUS = 4 # pixels + +#=============================================================================== + +ERROR_COLOUR = 'yellow' + +#=============================================================================== + +DIAGRAM_LAYER = 'diagram-layer' + +CELLDL_LAYER_CLASS = 'celldl-Layer' + +CELLDL_DEFINITIONS_ID = "celldl-svg-definitions" +CELLDL_METADATA_ID = "celldl-rdf-metadata" +CELLDL_STYLESHEET_ID = 'celldl-svg-stylesheet' + +#=============================================================================== + +CellDLStylesheet = '\n'.join([ # Copied from ``@renderer/styles/stylesheet.ts`` + f'svg{{font-size:{EM_SIZE}px}}', + # Conduits + '.celldl-Conduit{z-index:9999}', + # Connections + '.celldl-Connection{stroke-width:2;opacity:0.7;fill:none;stroke:currentcolor}', + '.celldl-Connection.dashed{stroke-dasharray:5}', + # Compartments + '.celldl-Compartment>rect.compartment{fill:#CCC;opacity:0.6;stroke:#444;rx:10px;ry:10px}', + # Interfaces + f'.celldl-InterfacePort{{fill:red;r:{INTERFACE_PORT_RADIUS}px}}', + f'.celldl-Unconnected{{fill:red;fill-opacity:0.1;stroke:red;r:{INTERFACE_PORT_RADIUS}px}}' +]) + +#=============================================================================== + +def arrow_marker_definition(markerId: str, markerType: str) -> str: +#================================================================== + # see https://developer.mozilla.org/en-US/docs/Web/SVG/Element/marker + return f""" + +""" + +#=============================================================================== +#=============================================================================== + +def bondgraph_arrow_definition(domain: str) -> str: +#================================================== + return arrow_marker_definition(f'connection-end-arrow-{domain}', domain) + +#=============================================================================== + +BondgraphSvgDefinitions: list[etree.Element] = etree.fromstring( + '\n'.join([ + f'', + bondgraph_arrow_definition('bondgraph'), + bondgraph_arrow_definition('mechanical'), + bondgraph_arrow_definition('electrical'), + bondgraph_arrow_definition('biochemical'), + '' + ]) +).getchildren() + +#=============================================================================== + +BondgraphStylesheet = '\n'.join([ + # Bondgraph specific + 'svg{--biochemical:#2F6EBA;--electrical:#DE8344;--mechanical:#4EAD5B}', + '.bondgraph{color:pink}' + '.biochemical{color:var(--biochemical)}', + '.electrical{color:var(--electrical)}', + '.mechanical{color:var(--mechanical)}', + # use var(--colour), setting them in master stylesheet included in (along with MathJax styles) + '.celldl-Connection.bondgraph{marker-end:url(#connection-end-arrow-bondgraph)}', + '.celldl-Connection.bondgraph.biochemical{marker-end:url(#connection-end-arrow-biochemical)}', + '.celldl-Connection.bondgraph.electrical{marker-end:url(#connection-end-arrow-electrical)}', + '.celldl-Connection.bondgraph.mechanical{marker-end:url(#connection-end-arrow-mechanical)}', +]) + +#=============================================================================== + diff --git a/mapmaker/sources/powerpoint/svgutils.py b/mapmaker/sources/powerpoint/svgutils.py index f432a878..06bda48a 100644 --- a/mapmaker/sources/powerpoint/svgutils.py +++ b/mapmaker/sources/powerpoint/svgutils.py @@ -307,7 +307,7 @@ def __process_shape(self, shape: Shape, svg_parent: SvgElement, shape.id, shape.geometry.bounds, shape.properties, pptx_shape) if self.__celldl is not None: - self.__celldl.add_metadata(shape) + self.__celldl.add_shape(shape) def __process_svg_shape(self, svg_shape, svg_kind, svg_parent, group_colour, shape_id, bounds, properties, pptx_shape): diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 39c40519..a583d45b 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -2,7 +2,7 @@ # # Flatmap viewer and annotation tools # -# Copyright (c) 2020 David Brooks +# Copyright (c) 2020 - 2025 David Brooks # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,13 +42,14 @@ from mapmaker.flatmap import Feature, FlatMap, SourceManifest, SOURCE_DETAIL_KINDS from mapmaker.flatmap.layers import FEATURES_TILE_LAYER, MapLayer from mapmaker.geometry import Transform -from mapmaker.settings import MAP_KIND +from mapmaker.settings import MAP_KIND, settings from mapmaker.shapes import Shape, SHAPE_TYPE from mapmaker.shapes.classify import ShapeClassifier -from mapmaker.utils import FilePath, ProgressBar, log, TreeList +from mapmaker.utils import FilePath, pathlib_path, ProgressBar, log, TreeList from .. import MapSource, RasterSource from .. import WORLD_METRES_PER_PIXEL +from ..celldl import CellDLExporter from .cleaner import SVGCleaner from .definitions import DefinitionStore, ObjectStore @@ -233,6 +234,12 @@ def __process_shapes(self, shapes: TreeList[Shape]) -> list[Feature]: # CellDL conversion mode... shape_classifier = ShapeClassifier(shapes.flatten(), self.source.map_area(), self.source.metres_per_pixel) shapes = TreeList(shape_classifier.shapes) + if settings.get('exportBondgraphs', False): + celldl_file = pathlib_path(self.source.href).with_suffix('.celldl.svg') + log.info(f'Exporting layer `{self.id}` to `{str(celldl_file)}`...') + celldl_export = CellDLExporter(self.__svg, self.source.href, self.source.transform.inverse()) + celldl_export.process(shapes) + celldl_export.save(celldl_file) # Add a background shape behind a detailed functional map if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL and self.source.kind == 'functional'): diff --git a/mapmaker/utils/__init__.py b/mapmaker/utils/__init__.py index 14ad4919..8ef06383 100644 --- a/mapmaker/utils/__init__.py +++ b/mapmaker/utils/__init__.py @@ -37,6 +37,25 @@ #=============================================================================== +""" +For generating ``lxml`` element tags +""" +class XmlNamespace: + def __init__(self, ns: str): + self.__ns = ns + + def __str__(self): + return self.__ns + + def __getattr__(self, attr: str) -> str: + return f'{{{self.__ns}}}{attr}' + +#=============================================================================== + +SVG_NS = XmlNamespace('http://www.w3.org/2000/svg') + +#=============================================================================== + def relative_path(path: str | pathlib.Path) -> bool: return str(path).split(':', 1)[0] not in ['file', 'http', 'https'] diff --git a/mapmaker/utils/svg.py b/mapmaker/utils/svg.py index dd734a64..3f74a912 100644 --- a/mapmaker/utils/svg.py +++ b/mapmaker/utils/svg.py @@ -24,9 +24,12 @@ In practice, feature IDs are have the form ``LAYER_NAME/Slide-N/NNNNN`` and won't contain any embedded periods. """ -def svg_id(id): -#============== - return id.replace('/', '.') +def svg_id(shape_id: str) -> str: +#================================ + shape_id = shape_id.split('/')[-1] + if shape_id.startswith('SHAPE_'): + shape_id = f'ID-{shape_id[6:].zfill(8)}' + return shape_id def name_from_id(id): #==================== From 9751aea5cb915366a092a9e23b2a466938e66bcd Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 12:54:43 +1300 Subject: [PATCH 033/111] Shape classification: Only check for parent/child relationships between annotation, component and text shapes. --- mapmaker/shapes/classify.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 5a138d4e..c53786de 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -150,7 +150,9 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float shape.properties['colour'] = SHAPE_ERROR_COLOUR if not shape.properties.get('exclude', False): self.__shapes_by_type[shape.shape_type].append(shape) - if shape.shape_type != SHAPE_TYPE.CONNECTION: + if shape.shape_type in [SHAPE_TYPE.ANNOTATION, + SHAPE_TYPE.COMPONENT, + SHAPE_TYPE.TEXT]: self.__geometry_to_shape[id(shape.geometry)] = shape component_geometries.append(shape.geometry) shape.properties['stroke-width'] = COMPONENT_BORDER_WIDTH From 27ab6751c2328d9fb861996ca6d43de4256dc0a5 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 12:55:39 +1300 Subject: [PATCH 034/111] Shape classification: exclude unclassifiable shapes unless authoring, in which case highlight them (in yellow). --- mapmaker/shapes/classify.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index c53786de..a9150609 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -147,7 +147,10 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float connection_joiners.append(shape) elif not self.__add_connection(shape): log.warning('Unclassifiable shape', shape=shape.id) - shape.properties['colour'] = SHAPE_ERROR_COLOUR + if settings.get('authoring', False): + shape.properties['exclude'] = True + else: + shape.properties['colour'] = SHAPE_ERROR_COLOUR if not shape.properties.get('exclude', False): self.__shapes_by_type[shape.shape_type].append(shape) if shape.shape_type in [SHAPE_TYPE.ANNOTATION, @@ -192,8 +195,10 @@ def __add_connection(self, shape: Shape) -> bool: return False elif 'Polygon' in shape.geometry.geom_type: if (line := self.__line_finder.get_line(shape)) is None: - shape.properties['exclude'] = not settings.get('authoring', False) - shape.properties['colour'] = SHAPE_ERROR_COLOUR + if settings.get('authoring', False): + shape.properties['exclude'] = True + else: + shape.properties['colour'] = SHAPE_ERROR_COLOUR return False shape.geometry = line kind = VASCULAR_KINDS.lookup(shape.properties.get('fill')) From f4ff2541e4c82a46dcb2efab87390dec07d17853 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 15:44:28 +1300 Subject: [PATCH 035/111] Add a `number` property to a shape, in order as they are created, so we can sort text shapes into their declaration order in the layer's source. --- mapmaker/shapes/__init__.py | 7 +++++++ mapmaker/shapes/text_finder.py | 5 ++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index dccc0eb6..45782ad5 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -50,12 +50,15 @@ class Shape(PropertyMixin): __shape_id_prefix: str = '' __last_shape_id: int = 0 + __last_shape_number: int = 0 def __init__(self, id: Optional[str], geometry: BaseGeometry, properties=None, **kwds): self.__initialising = True super().__init__(properties) for key, value in kwds.items(): self.set_property(key.replace('_', '-'), value) + Shape.__last_shape_number += 1 + self.__number: int = Shape.__last_shape_number if self.has_property('id'): id = self.get_property('id') if Shape.__shape_id_prefix == '': @@ -154,6 +157,10 @@ def metadata(self) -> dict[str, str]: def name(self) -> str: # Any text content associated with the shape: e.g. ``Bladder`` return self.get_property('name', '') + @property + def number(self) -> int: + return self.__number + @property def opacity(self) -> float: return self.get_property('opacity', 1.0) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 3a2c8c27..f6845255 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -156,16 +156,15 @@ def __text_clusters_to_text(self, text_clusters: list[TextShapeCluster]) -> str: def __cluster_text(self, text_shapes: list[Shape]) -> list[TextShapeCluster]: #============================================================================ offset = self.__max_text_vertical_offset - left_ordered_shapes = sorted(text_shapes, key=lambda s: s.left) + shapes_seen_order = sorted(text_shapes, key=lambda s: s.number) clusters: list[TextShapeCluster] = [] current_cluster = None - for shape in left_ordered_shapes: + for shape in shapes_seen_order: if (current_cluster is None or abs(shape.baseline - current_cluster.baseline) > offset): current_cluster = TextShapeCluster(shape) clusters.append(current_cluster) else: - # Note: ``current_cluster.baseline`` is monotonically increasing current_cluster.add_shape(shape) shape.properties['exclude'] = True return clusters From e5b75300a1cfb5141784a2d1cf53da3b2b8c78ca Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 15:45:44 +1300 Subject: [PATCH 036/111] Shape classification: a shape with a 'Multi' shape boundary can't be a Connection. --- mapmaker/shapes/line_finder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index e6fbefc1..a6ed43b2 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -275,6 +275,8 @@ def __init__(self, scaling: float): def get_line(self, shape: Shape) -> Optional[LineString]: #======================================================== + if 'Multi' in shape.geometry.boundary.geom_type: + return ends_graph = nx.Graph() used_lines: set[Line] = set() mid_lines: list[Line] = [] From 9c97a6243a5bb8497b67c095dd00288d8aa169fd Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 15:46:53 +1300 Subject: [PATCH 037/111] Don't show some properties in the string representation of a `Shape`. --- mapmaker/shapes/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index 45782ad5..eac075e9 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -44,6 +44,18 @@ class SHAPE_TYPE(str, Enum): ## Or IntEnum ?? #=============================================================================== KnownProperties = ['name', 'cd-class', 'fc-class', 'fc-kind'] +HiddenProperties = [ + 'area', + 'aspect', + 'bbox-coverage', + 'coverage', + 'fill', + 'geometry', + 'stroke', + 'stroke-width', + 'svg-element', + 'tile-layer', +] class Shape(PropertyMixin): __attributes = ['id', 'geometry', 'parents', 'children'] @@ -96,7 +108,7 @@ def __setattr__(self, key: str, value: Any=None): def __str__(self): properties = {key: value for key, value in self.properties.items() - if key in KnownProperties} + if key != 'id' and key not in HiddenProperties} return f'Shape {self.id}: {properties}' @staticmethod From 575bc4f86c16dc526d8a680e776cceffeab745e2 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 10 Feb 2025 15:48:19 +1300 Subject: [PATCH 038/111] Shape classification: reduce the baseline offset used for detecting text as base, super-, or sub-script. --- mapmaker/shapes/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/shapes/constants.py b/mapmaker/shapes/constants.py index c3097d5d..f01562a3 100644 --- a/mapmaker/shapes/constants.py +++ b/mapmaker/shapes/constants.py @@ -42,7 +42,7 @@ MAX_LINE_WIDTH = 20 # Close together parallel edges a polygons are converted to lines -MAX_TEXT_VERTICAL_OFFSET = 5 # Between cluster baseline and baselines of text in the cluster +MAX_TEXT_VERTICAL_OFFSET = 3 # Between cluster baseline and baselines of text in the cluster TEXT_BASELINE_OFFSET = -14.5 # From vertical centre of a component #=============================================================================== From d9ef047f8d25258ba1525194b8c5fa93c592f08d Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 3 Mar 2025 09:59:11 +1300 Subject: [PATCH 039/111] Rename shape type definitions' file as it applies to more than Powerpoint FC sources. --- mapmaker/annotation/__init__.py | 2 +- mapmaker/shapes/classify.py | 2 +- .../fc_powerpoint => shapes}/colours.py | 0 .../components.py => shapes/types.py} | 2 +- mapmaker/sources/fc_powerpoint/__init__.py | 20 +++++++++---------- mapmaker/sources/fc_powerpoint/connections.py | 7 +++---- mapmaker/sources/powerpoint/powerpoint.py | 5 ++--- 7 files changed, 18 insertions(+), 20 deletions(-) rename mapmaker/{sources/fc_powerpoint => shapes}/colours.py (100%) rename mapmaker/{sources/fc_powerpoint/components.py => shapes/types.py} (99%) diff --git a/mapmaker/annotation/__init__.py b/mapmaker/annotation/__init__.py index 08fbffae..8a4df70f 100644 --- a/mapmaker/annotation/__init__.py +++ b/mapmaker/annotation/__init__.py @@ -26,7 +26,7 @@ from mapmaker.knowledgebase.celldl import FC_CLASS from mapmaker.knowledgebase.sckan import PATH_TYPE from mapmaker.shapes import Shape -from mapmaker.sources.fc_powerpoint.components import is_connector +from mapmaker.shapes.types import is_connector from mapmaker.utils import log from .json_annotations import JsonAnnotations diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index a9150609..8d258889 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -35,7 +35,6 @@ from mapmaker.flatmap.layers import PATHWAYS_TILE_LAYER from mapmaker.settings import settings from mapmaker.shapes import Shape, SHAPE_TYPE -from mapmaker.sources.fc_powerpoint.components import VASCULAR_KINDS from mapmaker.utils import log from .constants import COMPONENT_BORDER_WIDTH, CONNECTION_STROKE_WIDTH, MAX_LINE_WIDTH @@ -43,6 +42,7 @@ from .line_finder import Line, LineFinder, XYPair from .text_finder import TextFinder +from .types import VASCULAR_KINDS #=============================================================================== diff --git a/mapmaker/sources/fc_powerpoint/colours.py b/mapmaker/shapes/colours.py similarity index 100% rename from mapmaker/sources/fc_powerpoint/colours.py rename to mapmaker/shapes/colours.py diff --git a/mapmaker/sources/fc_powerpoint/components.py b/mapmaker/shapes/types.py similarity index 99% rename from mapmaker/sources/fc_powerpoint/components.py rename to mapmaker/shapes/types.py index 2d10b3f5..525aafc3 100644 --- a/mapmaker/sources/fc_powerpoint/components.py +++ b/mapmaker/shapes/types.py @@ -20,8 +20,8 @@ from mapmaker.knowledgebase.sckan import PATH_TYPE from mapmaker.knowledgebase.celldl import CD_CLASS, FC_CLASS, FC_KIND -from mapmaker.shapes import SHAPE_TYPE +from .import SHAPE_TYPE from .colours import ColourMatcher, ColourMatcherDict #=============================================================================== diff --git a/mapmaker/sources/fc_powerpoint/__init__.py b/mapmaker/sources/fc_powerpoint/__init__.py index 05188dcd..37392b96 100644 --- a/mapmaker/sources/fc_powerpoint/__init__.py +++ b/mapmaker/sources/fc_powerpoint/__init__.py @@ -37,21 +37,21 @@ from mapmaker.settings import settings from mapmaker.shapes import Shape, SHAPE_TYPE from mapmaker.shapes.shapefilter import ShapeFilter +from mapmaker.shapes.types import make_annotation, make_component, make_connection, make_connector +from mapmaker.shapes.types import is_annotation, is_component, is_connector, is_system_name +from mapmaker.shapes.types import ensure_parent_system +from mapmaker.shapes.types import HYPERLINK_KINDS, HYPERLINK_IDENTIFIERS +from mapmaker.shapes.types import NERVE_FEATURE_KINDS, NEURON_PATH_TYPES +from mapmaker.shapes.types import ORGAN_COLOUR, ORGAN_KINDS +from mapmaker.shapes.types import VASCULAR_KINDS, VASCULAR_REGION_COLOUR, VASCULAR_VESSEL_KINDS from mapmaker.utils import log -from .. import RasterSource +#=============================================================================== + +from ..import RasterSource from ..powerpoint import PowerpointSource, Slide from ..powerpoint.colour import ColourTheme -#=============================================================================== - -from .components import make_annotation, make_component, make_connection, make_connector -from .components import is_annotation, is_component, is_connector, is_system_name -from .components import ensure_parent_system -from .components import HYPERLINK_KINDS, HYPERLINK_IDENTIFIERS -from .components import NERVE_FEATURE_KINDS, NEURON_PATH_TYPES -from .components import ORGAN_COLOUR, ORGAN_KINDS -from .components import VASCULAR_KINDS, VASCULAR_REGION_COLOUR, VASCULAR_VESSEL_KINDS from .connections import ConnectionClassifier if TYPE_CHECKING: diff --git a/mapmaker/sources/fc_powerpoint/connections.py b/mapmaker/sources/fc_powerpoint/connections.py index 5cd73f4f..ae7877d8 100644 --- a/mapmaker/sources/fc_powerpoint/connections.py +++ b/mapmaker/sources/fc_powerpoint/connections.py @@ -35,12 +35,11 @@ from mapmaker.knowledgebase.sckan import PATH_TYPE from mapmaker.settings import settings from mapmaker.shapes import Shape, SHAPE_TYPE +from mapmaker.shapes.types import is_component, is_connector, make_connector, system_ids +from mapmaker.shapes.types import NEURON_PATH_TYPES, VASCULAR_KINDS +from mapmaker.shapes.types import MAX_CONNECTION_GAP from mapmaker.utils import log -from .components import is_component, is_connector, make_connector, system_ids -from .components import NEURON_PATH_TYPES, VASCULAR_KINDS -from .components import MAX_CONNECTION_GAP - #=============================================================================== def direction(coords): diff --git a/mapmaker/sources/powerpoint/powerpoint.py b/mapmaker/sources/powerpoint/powerpoint.py index dbae10b5..5d2819f5 100644 --- a/mapmaker/sources/powerpoint/powerpoint.py +++ b/mapmaker/sources/powerpoint/powerpoint.py @@ -48,12 +48,11 @@ from mapmaker.geometry import MapBounds, Transform from mapmaker.properties.markup import parse_layer_directive, parse_markup from mapmaker.shapes import Shape, SHAPE_TYPE +from mapmaker.shapes.colours import ColourMatcher +from mapmaker.shapes.types import is_system_name from mapmaker.sources import WORLD_METRES_PER_EMU from mapmaker.utils import FilePath, log, ProgressBar, TreeList -from ..fc_powerpoint.colours import ColourMatcher -from ..fc_powerpoint.components import is_system_name - from .colour import ColourMap, ColourTheme from .geometry import get_shape_geometry from .presets import CT_TextMath, DRAWINGML, PPTX_NAMESPACE, pptx_resolve, pptx_uri From fb6f600c8acc695dba976a495a3ab9d05a482c57 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 3 Mar 2025 10:31:23 +1300 Subject: [PATCH 040/111] Use a shape's `layer/name` as its ID on FC maps if a specific ID has not been set via markup. --- mapmaker/flatmap/__init__.py | 11 ++++------- mapmaker/flatmap/layers.py | 4 +--- mapmaker/shapes/classify.py | 6 +++--- mapmaker/sources/__init__.py | 3 +-- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 50acf236..0b2751c7 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -208,7 +208,6 @@ def initialise(self): self.__feature_node_map = FeatureAnatomicalNodeMap(self.__manifest.connectivity_terms) self.__features_with_id: dict[str, Feature] = {} - self.__features_with_name: dict[str, Feature] = {} self.__last_geojson_id = 0 self.__features_by_geojson_id: dict[int, Feature] = {} @@ -272,10 +271,6 @@ def get_feature(self, feature_id: str) -> Optional[Feature]: #=========================================================== return self.__features_with_id.get(feature_id) - def get_feature_by_name(self, full_name: str) -> Optional[Feature]: - #================================================================== - return self.__features_with_name.get(full_name.replace(" ", "_")) - def get_feature_by_geojson_id(self, geojson_id: int) -> Optional[Feature]: #========================================================================= return self.__features_by_geojson_id.get(geojson_id) @@ -283,11 +278,13 @@ def get_feature_by_geojson_id(self, geojson_id: int) -> Optional[Feature]: def new_feature(self, layer_id: str, geometry, properties, is_group=False) -> Feature: #===================================================================================== self.__last_geojson_id += 1 + if self.map_kind == MAP_KIND.FUNCTIONAL: + if ((name := properties.get('name', '')) != '' + and properties.get('id', '').startswith(f'{layer_id}/SHAPE_')): + properties['id'] = f'{layer_id}/{name.replace(" ", "_")}' self.properties_store.update_properties(properties) # Update from JSON properties file feature = Feature(self.__last_geojson_id, geometry, properties, is_group=is_group) feature.set_property('layer', layer_id) - if (name := properties.get('name', properties.get('label', ''))) != '': - self.__features_with_name[f'{layer_id}/{name.replace(" ", "_")}'] = feature self.__features_by_geojson_id[feature.geojson_id] = feature if feature.id: if feature.id in self.__features_with_id: diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 67179386..2838abf6 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -201,9 +201,7 @@ def add_feature(self, feature: Feature): # type: ignore def __find_feature(self, feature_id: str) -> Optional[Feature]: #============================================================== - if (feature := self.flatmap.get_feature_by_name(feature_id)) is None: - feature = self.flatmap.get_feature(feature_id) - return feature + return self.flatmap.get_feature(feature_id.replace(" ", "_")) def align_layer(self, feature_alignment: list[tuple[str, str]]): #=============================================================== diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 8d258889..9fa4c138 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -173,9 +173,9 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float # Assign text labels to components and source and target of connections for shape in self.__shapes: if shape.shape_type in [SHAPE_TYPE.ANNOTATION, SHAPE_TYPE.COMPONENT]: - if (label_and_shapes := self.__text_finder.get_text(shape)) is not None: - shape.properties['label'] = label_and_shapes[0] - shape.properties['text-shapes'] = label_and_shapes[1] + if (name_and_shapes := self.__text_finder.get_text(shape)) is not None: + shape.properties['name'] = name_and_shapes[0] + shape.properties['text-shapes'] = name_and_shapes[1] # Although we do want their text, we don't want annotations to be active features if shape.shape_type == SHAPE_TYPE.ANNOTATION: shape.properties['exclude'] = True diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index c6ad1798..de612f1c 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -140,8 +140,7 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): raise ValueError('A `detail` source must specify an existing `feature`') if source_manifest.zoom < 1: raise ValueError('A `detail` source must specify `zoom`') - if ((feature := flatmap.get_feature_by_name(source_manifest.feature)) is None - and (feature := flatmap.get_feature(source_manifest.feature)) is None): + if (feature := flatmap.get_feature(source_manifest.feature)) is None: raise ValueError(f'Unknown source feature: {source_manifest.feature}') feature.set_property('maxzoom', source_manifest.zoom-1) feature.set_property('kind', 'expandable') From 241fe802147989bf48755591e2192c3eec3b91ef Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 3 Mar 2025 10:33:06 +1300 Subject: [PATCH 041/111] Don't use math mode (for LaTex) if a FC shape's name has no super- nor sub-scripts. --- mapmaker/shapes/text_finder.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index f6845255..910b8cee 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -18,6 +18,7 @@ # #=============================================================================== +import re from typing import Optional #=============================================================================== @@ -55,6 +56,11 @@ def left_sort_shapes(self): #=============================================================================== +SUBSCRIPT_CHAR = '_' +SUPERSCRIPT_CHAR = '^' + +#=============================================================================== + class LatexMaker: def __init__(self): self.__latex = [] @@ -80,10 +86,10 @@ def __make_latex(self): #====================== if len(self.__text): if self.__state < 0: - self.__latex.append(f'_{{{''.join(self.__text)}}}') + self.__latex.append(f'{SUBSCRIPT_CHAR}{{{''.join(self.__text)}}}') self.__subscripted = True elif self.__state > 0: - superscript = f'^{{{''.join(self.__text)}}}' + superscript = f'{SUPERSCRIPT_CHAR}{{{''.join(self.__text)}}}' if self.__subscripted: self.__latex.insert(-1, superscript) else: @@ -97,6 +103,7 @@ def __make_latex(self): class TextFinder: def __init__(self, scaling: float): + self.__sub_superscript_re = re.compile(f'{SUBSCRIPT_CHAR}|\\{SUPERSCRIPT_CHAR}') self.__max_text_vertical_offset = scaling * MAX_TEXT_VERTICAL_OFFSET self.__text_baseline_offset = scaling * TEXT_BASELINE_OFFSET @@ -132,7 +139,7 @@ def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: used_text_shapes.extend(cluster.shapes) if len(clusters): latex.add_text(self.__text_clusters_to_text(clusters), state) - text = f'${latex.latex}$' + text = f'${text}$' if self.__sub_superscript_re.search(text:=latex.latex) is not None else text return (text, used_text_shapes) if text != '' else None def __text_block_to_text(self, text_block: list[Shape]) -> str: From 04a4010c338cb1cb24780c7e36f22a8c2cdabc9e Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 3 Mar 2025 11:22:07 +1300 Subject: [PATCH 042/111] Set a connection's stroke colour when generating CellDL (this will eventually become a class value). --- mapmaker/shapes/classify.py | 10 +++++++--- mapmaker/sources/celldl/__init__.py | 16 ++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 9fa4c138..7f1a1a4d 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -201,11 +201,15 @@ def __add_connection(self, shape: Shape) -> bool: shape.properties['colour'] = SHAPE_ERROR_COLOUR return False shape.geometry = line - kind = VASCULAR_KINDS.lookup(shape.properties.get('fill')) + colour = shape.properties.get('fill') else: - kind = VASCULAR_KINDS.lookup(shape.properties.get('stroke')) - if kind is not None: + colour = shape.properties.get('stroke') + if colour is not None: + shape.properties['colour'] = colour + if (kind := VASCULAR_KINDS.lookup(colour)) is not None: shape.properties['kind'] = kind + else: + print(shape.id, 'COLOUR ?', colour) shape.properties['shape-type'] = SHAPE_TYPE.CONNECTION shape.properties['tile-layer'] = PATHWAYS_TILE_LAYER shape.properties['stroke-width'] = CONNECTION_STROKE_WIDTH diff --git a/mapmaker/sources/celldl/__init__.py b/mapmaker/sources/celldl/__init__.py index b0d1014f..1f785562 100644 --- a/mapmaker/sources/celldl/__init__.py +++ b/mapmaker/sources/celldl/__init__.py @@ -140,6 +140,10 @@ def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element if shape.shape_type == SHAPE_TYPE.CONNECTION: geometry = self.__world_to_pixels.transform_geometry(shape.geometry) coords = [f'{coord[0]} {coord[1]}' for coord in geometry.coords] + attributes = { + 'id': element_id, + 'd': f'M{coords[0]}L{"L".join(coords[1:])}' + } classes = [svg_class] connection_style = 'rectilinear' for (c1, c2) in itertools.pairwise(geometry.coords): @@ -148,15 +152,15 @@ def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element connection_style = 'linear' break classes.append(connection_style) - + if shape.properties.get('directional', False): + classes.append('arrow') if self.__export_type == EXPORT_TYPE.BONDGRAPH: classes.append('bondgraph') + if (colour := shape.properties.get('colour')) is not None: + attributes['style'] = f'stroke: {colour}' + attributes['class'] = ' '.join(classes) svg_element.getparent().remove(svg_element) - svg_element = etree.SubElement(self.__connection_group, SVG_NS.path, { - 'id': element_id, - 'class': ' '.join(classes), - }) - svg_element.attrib['d'] = f'M{coords[0]}L{"L".join(coords[1:])}' + svg_element = etree.SubElement(self.__connection_group, SVG_NS.path, attributes) elif (text_shapes := shape.get_property('text-shapes')) is not None: shape_element = copy.deepcopy(svg_element) svg_element.tag = SVG_NS.g From 787c7d79eaf06910230a221f5a7d3b42e3ce4d08 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 4 Mar 2025 17:31:04 +1300 Subject: [PATCH 043/111] CellDL: use better names for variables. --- mapmaker/sources/celldl/__init__.py | 35 +++++++++++++++-------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/mapmaker/sources/celldl/__init__.py b/mapmaker/sources/celldl/__init__.py index 1f785562..17c3fbcf 100644 --- a/mapmaker/sources/celldl/__init__.py +++ b/mapmaker/sources/celldl/__init__.py @@ -63,20 +63,21 @@ class EXPORT_TYPE: #=============================================================================== class CellDLExporter: - def __init__(self, svg_element: etree.Element, source_href: str, world_to_pixels: Transform, + def __init__(self, svg_root: etree.Element, source_href: str, world_to_pixels: Transform, export_type: Optional[str]=EXPORT_TYPE.BONDGRAPH): self.__celldl = CellDLGraph(BG_NS.Model if export_type == EXPORT_TYPE.BONDGRAPH else None) self.__celldl.set_property(DCT_NS.source, rdflib.URIRef(source_href)) - self.__svg_element = svg_element + self.__svg_root = svg_root self.__world_to_pixels = world_to_pixels self.__export_type = export_type - celldl_defs = svg_element.find(f'.//{SVG_NS.defs}[@id="{CELLDL_DEFINITIONS_ID}"]') + + celldl_defs = svg_root.find(f'.//{SVG_NS.defs}[@id="{CELLDL_DEFINITIONS_ID}"]') if celldl_defs is None: - celldl_defs = svg_element.find(f'.//{SVG_NS.defs}') + celldl_defs = svg_root.find(f'.//{SVG_NS.defs}') if celldl_defs is not None: celldl_defs.attrib['id'] = CELLDL_DEFINITIONS_ID else: - celldl_defs = etree.SubElement(svg_element, SVG_NS.defs, { + celldl_defs = etree.SubElement(svg_root, SVG_NS.defs, { 'id': CELLDL_DEFINITIONS_ID }) if export_type == EXPORT_TYPE.BONDGRAPH: @@ -92,21 +93,21 @@ def __init__(self, svg_element: etree.Element, source_href: str, world_to_pixels stylesheets.append(BondgraphStylesheet) celldl_style.text = '\n'.join(stylesheets) - diagram = etree.Element(SVG_NS.g, { + celldl_diagram = etree.Element(SVG_NS.g, { 'id': DIAGRAM_LAYER, 'class': CELLDL_LAYER_CLASS }) viewbox = self.__check_viewbox() - for child in svg_element: + for child in svg_root: if child.tag != SVG_NS.defs: - diagram.append(child) - svg_element.append(diagram) # Need to append after above copy/move - self.__connection_group = etree.SubElement(diagram, SVG_NS.g) + celldl_diagram.append(child) + svg_root.append(celldl_diagram) # Need to append after above copy/move + self.__connection_group = etree.SubElement(celldl_diagram, SVG_NS.g) self.__metadata_element = etree.Element(SVG_NS.metadata, { 'id': CELLDL_METADATA_ID, 'data-content-type': 'text/turtle' }) - svg_element.insert(0, self.__metadata_element) + svg_root.insert(0, self.__metadata_element) def process(self, shapes: TreeList[Shape]): #========================================== @@ -115,18 +116,18 @@ def process(self, shapes: TreeList[Shape]): def save(self, path: Path): #========================== self.__metadata_element.text = etree.CDATA(self.__celldl.as_turtle()) - svg_tree = etree.ElementTree(self.__svg_element) + svg_tree = etree.ElementTree(self.__svg_root) svg_tree.write(path, encoding='utf-8', #inclusive_ns_prefixes=['svg'], pretty_print=True, xml_declaration=True) def __check_viewbox(self) -> tuple[float, float, float, float]: #============================================================== - width = self.__svg_element.attrib.pop('width', None) - height = self.__svg_element.attrib.pop('height', None) - if 'viewBox' not in self.__svg_element.attrib: - self.__svg_element.attrib['viewBox'] = f'0 0 {length_as_pixels(width):.1f} {length_as_pixels(height):.1f}' - return tuple(float(i) for i in self.__svg_element.attrib['viewBox'].split()) # type: ignore + width = self.__svg_root.attrib.pop('width', None) + height = self.__svg_root.attrib.pop('height', None) + if 'viewBox' not in self.__svg_root.attrib: + self.__svg_root.attrib['viewBox'] = f'0 0 {length_as_pixels(width):.1f} {length_as_pixels(height):.1f}' + return tuple(float(i) for i in self.__svg_root.attrib['viewBox'].split()) # type: ignore def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element): #======================================================= From 54e10ca23c1ab97f7e9eb0e4da61620ceb63c879 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 4 Mar 2025 17:33:18 +1300 Subject: [PATCH 044/111] CellDL: keep ANNOTATION shapes in their own group when exporting. --- mapmaker/sources/celldl/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapmaker/sources/celldl/__init__.py b/mapmaker/sources/celldl/__init__.py index 17c3fbcf..9891cd95 100644 --- a/mapmaker/sources/celldl/__init__.py +++ b/mapmaker/sources/celldl/__init__.py @@ -103,6 +103,7 @@ def __init__(self, svg_root: etree.Element, source_href: str, world_to_pixels: T celldl_diagram.append(child) svg_root.append(celldl_diagram) # Need to append after above copy/move self.__connection_group = etree.SubElement(celldl_diagram, SVG_NS.g) + self.__annotation_group = etree.SubElement(celldl_diagram, SVG_NS.g) self.__metadata_element = etree.Element(SVG_NS.metadata, { 'id': CELLDL_METADATA_ID, 'data-content-type': 'text/turtle' @@ -172,6 +173,8 @@ def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element svg_element.extend([element for text_shape in text_shapes if (element := text_shape.get_property('svg-element')) is not None]) shape.set_property('svg-element', svg_element) + if shape.shape_type == SHAPE_TYPE.ANNOTATION: + self.__annotation_group.append(svg_element) else: svg_element.attrib['id'] = element_id svg_element.attrib['class'] = ' '.join(svg_element.attrib.get('class', '').split() + [svg_class]) From fbb760c44a1ce1b90d47e6703b8efe2939e02075 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 6 Mar 2025 10:59:03 +1300 Subject: [PATCH 045/111] Rework feature lookup by name for functional maps so that we keep the separate `id` of a feature. --- mapmaker/flatmap/__init__.py | 16 ++++++++++------ mapmaker/properties/__init__.py | 4 ++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 0b2751c7..a2a67fd5 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -208,6 +208,7 @@ def initialise(self): self.__feature_node_map = FeatureAnatomicalNodeMap(self.__manifest.connectivity_terms) self.__features_with_id: dict[str, Feature] = {} + self.__features_with_name: dict[str, Feature] = {} self.__last_geojson_id = 0 self.__features_by_geojson_id: dict[int, Feature] = {} @@ -269,7 +270,11 @@ def has_feature(self, feature_id: str) -> bool: def get_feature(self, feature_id: str) -> Optional[Feature]: #=========================================================== - return self.__features_with_id.get(feature_id) + if self.map_kind == MAP_KIND.FUNCTIONAL: + return self.__features_with_name.get(feature_id.replace(" ", "_"), + self.__features_with_id.get(feature_id)) + else: + return self.__features_with_id.get(feature_id) def get_feature_by_geojson_id(self, geojson_id: int) -> Optional[Feature]: #========================================================================= @@ -278,19 +283,18 @@ def get_feature_by_geojson_id(self, geojson_id: int) -> Optional[Feature]: def new_feature(self, layer_id: str, geometry, properties, is_group=False) -> Feature: #===================================================================================== self.__last_geojson_id += 1 - if self.map_kind == MAP_KIND.FUNCTIONAL: - if ((name := properties.get('name', '')) != '' - and properties.get('id', '').startswith(f'{layer_id}/SHAPE_')): - properties['id'] = f'{layer_id}/{name.replace(" ", "_")}' + properties['layer'] = layer_id self.properties_store.update_properties(properties) # Update from JSON properties file feature = Feature(self.__last_geojson_id, geometry, properties, is_group=is_group) - feature.set_property('layer', layer_id) self.__features_by_geojson_id[feature.geojson_id] = feature if feature.id: if feature.id in self.__features_with_id: pass else: self.__features_with_id[feature.id] = feature + if self.map_kind == MAP_KIND.FUNCTIONAL: + if (name := properties.get('name', '')) != '': + self.__features_with_name[f'{layer_id}/{name.replace(" ", "_")}'] = feature return feature def network_feature(self, feature: Feature) -> bool: diff --git a/mapmaker/properties/__init__.py b/mapmaker/properties/__init__.py index 7d510184..de36abe9 100644 --- a/mapmaker/properties/__init__.py +++ b/mapmaker/properties/__init__.py @@ -164,6 +164,7 @@ def __set_feature_properties(self, features): #============================================ if isinstance(features, dict): for id, properties in features.items(): + id = id.replace(" ", "_") self.__properties_by_id[id].update(properties) if (properties.get('type') == 'nerve' and (entity := properties.get('models')) is not None): @@ -212,6 +213,9 @@ def update_properties(self, feature_properties): #=============================================== classes = feature_properties.get('class', '').split() id = feature_properties.get('id') + if self.__flatmap.map_kind == MAP_KIND.FUNCTIONAL: + if (name := feature_properties.get('name', '').replace(" ", "_")) != '': + id = f'{feature_properties.get("layer", "")}/{name}' if id is not None: classes.extend(self.__properties_by_id.get(id, {}).get('class', '').split()) for cls in classes: From 62fd0809f689ee987a5bff5db2c6c79d5e989849 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 6 Mar 2025 11:01:25 +1300 Subject: [PATCH 046/111] Don't look for super- and sub-scripts in a text block with just a single baseline. --- mapmaker/shapes/text_finder.py | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 910b8cee..56716a4d 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -117,26 +117,30 @@ def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: clusters = [] latex = LatexMaker() used_text_shapes = [] - for cluster in text_clusters: - if cluster.baseline < (baseline - offset): - if state > 0 and len(clusters): - latex.add_text(self.__text_clusters_to_text(clusters), state) - clusters = [] - clusters.append(cluster) - state = -1 - elif cluster.baseline > (baseline + offset): - if state < 0 and len(clusters): - latex.add_text(self.__text_clusters_to_text(clusters), state) - clusters = [] - clusters.append(cluster) - state = 1 - else: - if state != 0 and len(clusters): - latex.add_text(self.__text_clusters_to_text(clusters), state) - clusters = [] - latex.add_text(self.__text_block_to_text(cluster.shapes), 0) - state = 0 - used_text_shapes.extend(cluster.shapes) + if len(text_clusters) == 1: + latex.add_text(self.__text_block_to_text(text_clusters[0].shapes), 0) + used_text_shapes.extend(text_clusters[0].shapes) + else: + for cluster in text_clusters: + if cluster.baseline < (baseline - offset): + if state > 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + clusters.append(cluster) + state = -1 + elif cluster.baseline > (baseline + offset): + if state < 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + clusters.append(cluster) + state = 1 + else: + if state != 0 and len(clusters): + latex.add_text(self.__text_clusters_to_text(clusters), state) + clusters = [] + latex.add_text(self.__text_block_to_text(cluster.shapes), 0) + state = 0 + used_text_shapes.extend(cluster.shapes) if len(clusters): latex.add_text(self.__text_clusters_to_text(clusters), state) text = f'${text}$' if self.__sub_superscript_re.search(text:=latex.latex) is not None else text From 79bd44a010c35abe85a16050628d30e25f5af4f2 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 6 Mar 2025 11:02:11 +1300 Subject: [PATCH 047/111] Minor tidy. --- mapmaker/sources/svg/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index a583d45b..9e343058 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -116,13 +116,13 @@ def __init__(self, flatmap: FlatMap, source_manifest: SourceManifest): # maker scale = max(scale_x, scale_y) self.__transform = (Transform([[1, 0, bounds[0]+(bounds[2]-bounds[0])/2], [0, 1, bounds[1]+(bounds[3]-bounds[1])/2], - [0, 0, 1]]) + [0, 0, 1]]) @np.array([[scale, 0, 0], [ 0, scale, 0], [ 0, 0, 1]]) @np.array([[1.0, 0.0, -left-width/2], - [0.0, -1.0, top+height/2], - [0.0, 0.0, 1.0]])) + [0.0, -1.0, top+height/2], + [0.0, 0.0, 1.0]])) self.__metres_per_pixel = scale else: # Add a margin around the base layer of a functional map From 9ccdd0c20596386c5e29736f31756ee48088ca67 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 6 Mar 2025 11:04:18 +1300 Subject: [PATCH 048/111] TitleCase method names that are class methods. --- mapmaker/geometry/__init__.py | 4 ++-- mapmaker/sources/svg/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapmaker/geometry/__init__.py b/mapmaker/geometry/__init__.py index 5ad1dfc8..fc1b1cbf 100644 --- a/mapmaker/geometry/__init__.py +++ b/mapmaker/geometry/__init__.py @@ -131,11 +131,11 @@ def Identity(cls) -> Self: return cls(np.identity(3)) @classmethod - def scale(cls, scale: float) -> Self: + def Scale(cls, scale: float) -> Self: return cls([[scale, 0, 0], [0, scale, 0], [0, 0, 1]]) @classmethod - def translate(cls, translate: tuple[float, float]) -> Self: + def Translate(cls, translate: tuple[float, float]) -> Self: return cls([[1, 0, translate[0]], [0, 1, translate[1]], [0, 0, 1]]) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 9e343058..7aa6425a 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -186,7 +186,7 @@ def get_raster_sources(self) -> list[RasterSource]: background_path = FilePath(background.href) raster_sources.append(RasterSource(f'{self.id}_background', 'svg', background_path.get_data, self, source_path=background_path, background_layer=True, - transform=Transform.translate(background.translate)@Transform.scale(background.scale))) + transform=Transform.Translate(background.translate)@Transform.Scale(background.scale))) raster_sources.append(RasterSource(f'{self.id}_image', 'svg', self.__get_raster_data, self, source_path=self.__source_file)) return raster_sources From d3e11e095d295a04bfb05e62c4437c7d4d3da0cf Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 11 Mar 2025 08:22:14 +1300 Subject: [PATCH 049/111] GeoJSON: make sure `properties` aren't JSON encoded. --- mapmaker/output/geojson.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mapmaker/output/geojson.py b/mapmaker/output/geojson.py index fb4d0c1c..ea1d8c15 100644 --- a/mapmaker/output/geojson.py +++ b/mapmaker/output/geojson.py @@ -90,10 +90,6 @@ def __save_features(self, features): if (value := feature.get_property(name)) is not None and value != '' } - properties.update({ - name: json.dumps(value) for (name, value) in properties.items() - if name in ENCODED_FEATURE_PROPERTIES - }) geometry = feature.geometry area = geometry.area mercator_geometry = mercator_transform(geometry) @@ -141,6 +137,13 @@ def __save_features(self, features): if self.__flatmap.map_kind == MAP_KIND.CENTRELINE and feature.properties.get('kind') == 'centreline': properties['coordinates'] = geojson['geometry']['coordinates'] + # We don't want encoded JSON in ``geojson['properties']`` so ``properties`` encoding has to be + # after GeoJSON has been updated + properties.update({ + name: json.dumps(value) for (name, value) in properties.items() + if name in ENCODED_FEATURE_PROPERTIES + }) + # Output the anatomical nodes associated with the feature if len(feature.anatomical_nodes): properties['anatomical-nodes'] = feature.anatomical_nodes From cd91ccd3a0174080c331f5c6c6ed3ca59a326bf0 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 11 Mar 2025 08:30:38 +1300 Subject: [PATCH 050/111] CellDL: copy across `.background` elements from the source SVG; put export code into it's own file. --- mapmaker/sources/celldl/__init__.py | 162 +------------------- mapmaker/sources/celldl/definitions.py | 1 + mapmaker/sources/celldl/exporter.py | 202 +++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 160 deletions(-) create mode 100644 mapmaker/sources/celldl/exporter.py diff --git a/mapmaker/sources/celldl/__init__.py b/mapmaker/sources/celldl/__init__.py index 9891cd95..08cdadb6 100644 --- a/mapmaker/sources/celldl/__init__.py +++ b/mapmaker/sources/celldl/__init__.py @@ -18,165 +18,7 @@ # #=============================================================================== -import copy -import itertools -from pathlib import Path -from typing import Optional - -#=============================================================================== - -import lxml.etree as etree -import rdflib - -#=============================================================================== - -from mapmaker.geometry import Transform -from mapmaker.knowledgebase.celldl import BG_NS, CellDLGraph, DCT_NS -from mapmaker.shapes import Shape, SHAPE_TYPE -from mapmaker.sources.svg.utils import length_as_pixels -from mapmaker.utils import TreeList, SVG_NS -from mapmaker.utils.svg import svg_id - -from .definitions import CELLDL_DEFINITIONS_ID, CELLDL_LAYER_CLASS, CELLDL_METADATA_ID, CELLDL_STYLESHEET_ID, DIAGRAM_LAYER -from .definitions import BondgraphStylesheet, BondgraphSvgDefinitions, CellDLStylesheet - -#=============================================================================== - -class EXPORT_TYPE: - BONDGRAPH = 'bondgraph' - -#=============================================================================== - -MAXIMIMUM_SMALLEST_SVG_DIMENSION = 1200 - -#=============================================================================== - -# If successive x-coordinates (y-coordinates) of a path are within this distance then the -# connecting line segment is considered to be horizontal (vertical) - -PATH_EPSILON = 0.5 # pixels - -#=============================================================================== - -etree.register_namespace('svg', str(SVG_NS)) - -#=============================================================================== - -class CellDLExporter: - def __init__(self, svg_root: etree.Element, source_href: str, world_to_pixels: Transform, - export_type: Optional[str]=EXPORT_TYPE.BONDGRAPH): - self.__celldl = CellDLGraph(BG_NS.Model if export_type == EXPORT_TYPE.BONDGRAPH else None) - self.__celldl.set_property(DCT_NS.source, rdflib.URIRef(source_href)) - self.__svg_root = svg_root - self.__world_to_pixels = world_to_pixels - self.__export_type = export_type - - celldl_defs = svg_root.find(f'.//{SVG_NS.defs}[@id="{CELLDL_DEFINITIONS_ID}"]') - if celldl_defs is None: - celldl_defs = svg_root.find(f'.//{SVG_NS.defs}') - if celldl_defs is not None: - celldl_defs.attrib['id'] = CELLDL_DEFINITIONS_ID - else: - celldl_defs = etree.SubElement(svg_root, SVG_NS.defs, { - 'id': CELLDL_DEFINITIONS_ID - }) - if export_type == EXPORT_TYPE.BONDGRAPH: - celldl_defs.extend(BondgraphSvgDefinitions) - - celldl_style = celldl_defs.find(f'.//{SVG_NS.style}[@id="{CELLDL_STYLESHEET_ID}"]') - if celldl_style is None: - celldl_style = etree.SubElement(celldl_defs, SVG_NS.style, { - 'id': CELLDL_STYLESHEET_ID - }) - stylesheets = [CellDLStylesheet] - if export_type == EXPORT_TYPE.BONDGRAPH: - stylesheets.append(BondgraphStylesheet) - celldl_style.text = '\n'.join(stylesheets) - - celldl_diagram = etree.Element(SVG_NS.g, { - 'id': DIAGRAM_LAYER, - 'class': CELLDL_LAYER_CLASS - }) - viewbox = self.__check_viewbox() - for child in svg_root: - if child.tag != SVG_NS.defs: - celldl_diagram.append(child) - svg_root.append(celldl_diagram) # Need to append after above copy/move - self.__connection_group = etree.SubElement(celldl_diagram, SVG_NS.g) - self.__annotation_group = etree.SubElement(celldl_diagram, SVG_NS.g) - self.__metadata_element = etree.Element(SVG_NS.metadata, { - 'id': CELLDL_METADATA_ID, - 'data-content-type': 'text/turtle' - }) - svg_root.insert(0, self.__metadata_element) - - def process(self, shapes: TreeList[Shape]): - #========================================== - self.__process_shape_list(shapes) ##, self.__diagram) - - def save(self, path: Path): - #========================== - self.__metadata_element.text = etree.CDATA(self.__celldl.as_turtle()) - svg_tree = etree.ElementTree(self.__svg_root) - svg_tree.write(path, - encoding='utf-8', #inclusive_ns_prefixes=['svg'], - pretty_print=True, xml_declaration=True) - - def __check_viewbox(self) -> tuple[float, float, float, float]: - #============================================================== - width = self.__svg_root.attrib.pop('width', None) - height = self.__svg_root.attrib.pop('height', None) - if 'viewBox' not in self.__svg_root.attrib: - self.__svg_root.attrib['viewBox'] = f'0 0 {length_as_pixels(width):.1f} {length_as_pixels(height):.1f}' - return tuple(float(i) for i in self.__svg_root.attrib['viewBox'].split()) # type: ignore - - def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element): - #======================================================= - for shape in shapes[0:]: - if isinstance(shape, TreeList): - self.__process_shape_list(shape) - elif not shape.properties.get('exclude', False): - if (svg_element := shape.get_property('svg-element')) is not None: - element_id = svg_id(shape.id) - if (svg_class := self.__celldl.add_shape(shape)) is not None: - if shape.shape_type == SHAPE_TYPE.CONNECTION: - geometry = self.__world_to_pixels.transform_geometry(shape.geometry) - coords = [f'{coord[0]} {coord[1]}' for coord in geometry.coords] - attributes = { - 'id': element_id, - 'd': f'M{coords[0]}L{"L".join(coords[1:])}' - } - classes = [svg_class] - connection_style = 'rectilinear' - for (c1, c2) in itertools.pairwise(geometry.coords): - if (abs(c1[0] - c2[0]) > PATH_EPSILON - and abs(c1[1] - c2[1]) > PATH_EPSILON): - connection_style = 'linear' - break - classes.append(connection_style) - if shape.properties.get('directional', False): - classes.append('arrow') - if self.__export_type == EXPORT_TYPE.BONDGRAPH: - classes.append('bondgraph') - if (colour := shape.properties.get('colour')) is not None: - attributes['style'] = f'stroke: {colour}' - attributes['class'] = ' '.join(classes) - svg_element.getparent().remove(svg_element) - svg_element = etree.SubElement(self.__connection_group, SVG_NS.path, attributes) - elif (text_shapes := shape.get_property('text-shapes')) is not None: - shape_element = copy.deepcopy(svg_element) - svg_element.tag = SVG_NS.g - svg_element.attrib.clear() - svg_element.attrib['id'] = element_id - svg_element.attrib['class'] = svg_class - svg_element.append(shape_element) - svg_element.extend([element for text_shape in text_shapes - if (element := text_shape.get_property('svg-element')) is not None]) - shape.set_property('svg-element', svg_element) - if shape.shape_type == SHAPE_TYPE.ANNOTATION: - self.__annotation_group.append(svg_element) - else: - svg_element.attrib['id'] = element_id - svg_element.attrib['class'] = ' '.join(svg_element.attrib.get('class', '').split() + [svg_class]) +# Exports +from .exporter import CellDLExporter #=============================================================================== diff --git a/mapmaker/sources/celldl/definitions.py b/mapmaker/sources/celldl/definitions.py index b09085ed..b6f7d018 100644 --- a/mapmaker/sources/celldl/definitions.py +++ b/mapmaker/sources/celldl/definitions.py @@ -37,6 +37,7 @@ DIAGRAM_LAYER = 'diagram-layer' +CELLDL_BACKGROUND_CLASS = 'celldl-background' CELLDL_LAYER_CLASS = 'celldl-Layer' CELLDL_DEFINITIONS_ID = "celldl-svg-definitions" diff --git a/mapmaker/sources/celldl/exporter.py b/mapmaker/sources/celldl/exporter.py new file mode 100644 index 00000000..89a92b8b --- /dev/null +++ b/mapmaker/sources/celldl/exporter.py @@ -0,0 +1,202 @@ +#=============================================================================== +# +# Flatmap viewer and annotation tools +# +# Copyright (c) 2020 - 2025 David Brooks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#=============================================================================== + +import copy +import itertools +from pathlib import Path +from typing import Optional + +#=============================================================================== + +import lxml.etree as etree +import rdflib + +#=============================================================================== + +from mapmaker.geometry import Transform +from mapmaker.knowledgebase.celldl import BG_NS, CellDLGraph, DCT_NS +from mapmaker.properties.markup import parse_markup +from mapmaker.shapes import Shape, SHAPE_TYPE +from mapmaker.sources.svg.utils import length_as_pixels, svg_markup +from mapmaker.utils import TreeList, SVG_NS +from mapmaker.utils.svg import svg_id + +from .definitions import CELLDL_BACKGROUND_CLASS, CELLDL_DEFINITIONS_ID, CELLDL_LAYER_CLASS +from .definitions import CELLDL_METADATA_ID, CELLDL_STYLESHEET_ID, DIAGRAM_LAYER +from .definitions import BondgraphStylesheet, BondgraphSvgDefinitions, CellDLStylesheet + +#=============================================================================== + +class EXPORT_TYPE: + BONDGRAPH = 'bondgraph' + +KIND_TO_SVG_CLASS = { + 'arterial' + 'venous' +} + +#=============================================================================== + +MAXIMIMUM_SMALLEST_SVG_DIMENSION = 1200 + +#=============================================================================== + +# If successive x-coordinates (y-coordinates) of a path are within this distance then the +# connecting line segment is considered to be horizontal (vertical) + +PATH_EPSILON = 0.5 # pixels + +#=============================================================================== + +etree.register_namespace('svg', str(SVG_NS)) + +#=============================================================================== + +class CellDLExporter: + def __init__(self, svg_root: etree.Element, source_href: str, world_to_pixels: Transform, + export_type: Optional[str]=EXPORT_TYPE.BONDGRAPH): + self.__celldl = CellDLGraph(BG_NS.Model if export_type == EXPORT_TYPE.BONDGRAPH else None) + self.__celldl.set_property(DCT_NS.source, rdflib.URIRef(source_href)) + self.__svg_root = svg_root + self.__world_to_pixels = world_to_pixels + self.__export_type = export_type + + celldl_defs = svg_root.find(f'.//{SVG_NS.defs}[@id="{CELLDL_DEFINITIONS_ID}"]') + if celldl_defs is None: + celldl_defs = svg_root.find(f'.//{SVG_NS.defs}') + if celldl_defs is not None: + celldl_defs.attrib['id'] = CELLDL_DEFINITIONS_ID + else: + celldl_defs = etree.SubElement(svg_root, SVG_NS.defs, { + 'id': CELLDL_DEFINITIONS_ID + }) + if export_type == EXPORT_TYPE.BONDGRAPH: + celldl_defs.extend(BondgraphSvgDefinitions) + + celldl_style = celldl_defs.find(f'.//{SVG_NS.style}[@id="{CELLDL_STYLESHEET_ID}"]') + if celldl_style is None: + celldl_style = etree.SubElement(celldl_defs, SVG_NS.style, { + 'id': CELLDL_STYLESHEET_ID + }) + stylesheets = [CellDLStylesheet] + if export_type == EXPORT_TYPE.BONDGRAPH: + stylesheets.append(BondgraphStylesheet) + celldl_style.text = '\n'.join(stylesheets) + + celldl_diagram = etree.Element(SVG_NS.g, { + 'id': DIAGRAM_LAYER, + 'class': CELLDL_LAYER_CLASS + }) + + for child in svg_root: + if child.tag != SVG_NS.defs: + markup = svg_markup(child) + if markup.startswith('.'): + properties = parse_markup(markup) + if properties.get('background', False): + # ``.background`` elements stay where they are in the SVG + classes = child.attrib.get('class', []) + classes.append(CELLDL_BACKGROUND_CLASS) + child.attrib['class'] = ' '.join(classes) + continue + # Other elements are moved to the ``celldl_diagram`` group + celldl_diagram.append(child) + + svg_root.append(celldl_diagram) # Need to append after above copy/move + self.__connection_group = etree.SubElement(celldl_diagram, SVG_NS.g) + self.__annotation_group = etree.SubElement(celldl_diagram, SVG_NS.g) + self.__metadata_element = etree.Element(SVG_NS.metadata, { + 'id': CELLDL_METADATA_ID, + 'data-content-type': 'text/turtle' + }) + svg_root.insert(0, self.__metadata_element) + + def process(self, shapes: TreeList[Shape]): + #========================================== + self.__process_shape_list(shapes) ##, self.__diagram) + + def save(self, path: Path): + #========================== + self.__metadata_element.text = etree.CDATA(self.__celldl.as_turtle()) + svg_tree = etree.ElementTree(self.__svg_root) + svg_tree.write(path, + encoding='utf-8', #inclusive_ns_prefixes=['svg'], + pretty_print=True, xml_declaration=True) + + def __check_viewbox(self) -> tuple[float, float, float, float]: + #============================================================== + width = self.__svg_root.attrib.pop('width', None) + height = self.__svg_root.attrib.pop('height', None) + if 'viewBox' not in self.__svg_root.attrib: + self.__svg_root.attrib['viewBox'] = f'0 0 {length_as_pixels(width):.1f} {length_as_pixels(height):.1f}' + return tuple(float(i) for i in self.__svg_root.attrib['viewBox'].split()) # type: ignore + + def __process_shape_list(self, shapes: TreeList[Shape]): #, group: etree.Element): + #======================================================= + for shape in shapes[0:]: + if isinstance(shape, TreeList): + self.__process_shape_list(shape) + elif not shape.properties.get('exclude', False): + if (svg_element := shape.get_property('svg-element')) is not None: + if (T := shape.get_property('svg-transform')) is not None and not T.is_identity: + transform = T.svg_matrix + element_id = svg_id(shape.id) + if (svg_class := self.__celldl.add_shape(shape)) is not None: + if shape.shape_type == SHAPE_TYPE.CONNECTION: + geometry = self.__world_to_pixels.transform_geometry(shape.geometry) + coords = [f'{coord[0]} {coord[1]}' for coord in geometry.coords] + attributes = { + 'id': element_id, + 'd': f'M{coords[0]}L{"L".join(coords[1:])}' + } + classes = [svg_class] + connection_style = 'rectilinear' + for (c1, c2) in itertools.pairwise(geometry.coords): + if (abs(c1[0] - c2[0]) > PATH_EPSILON + and abs(c1[1] - c2[1]) > PATH_EPSILON): + connection_style = 'linear' + break + classes.append(connection_style) + if shape.properties.get('directional', False): + classes.append('arrow') + if self.__export_type == EXPORT_TYPE.BONDGRAPH: + classes.append('bondgraph') + if (colour := shape.properties.get('colour')) is not None: + attributes['style'] = f'stroke: {colour}' + attributes['class'] = ' '.join(classes) + svg_element.getparent().remove(svg_element) + svg_element = etree.SubElement(self.__connection_group, SVG_NS.path, attributes) + elif (text_shapes := shape.get_property('text-shapes')) is not None: + shape_element = copy.deepcopy(svg_element) + svg_element.tag = SVG_NS.g + svg_element.attrib.clear() + svg_element.attrib['id'] = element_id + svg_element.attrib['class'] = svg_class + svg_element.append(shape_element) + svg_element.extend([element for text_shape in text_shapes + if (element := text_shape.get_property('svg-element')) is not None]) + shape.set_property('svg-element', svg_element) + if shape.shape_type == SHAPE_TYPE.ANNOTATION: + self.__annotation_group.append(svg_element) + else: + svg_element.attrib['id'] = element_id + svg_element.attrib['class'] = ' '.join(svg_element.attrib.get('class', '').split() + [svg_class]) + +#=============================================================================== From 13dc3f1285939d6312024feeb452901cac832103 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 11 Mar 2025 08:32:42 +1300 Subject: [PATCH 051/111] Extend `PropertyMixin` with `append_property(key, value)`. --- mapmaker/utils/property_mixin.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mapmaker/utils/property_mixin.py b/mapmaker/utils/property_mixin.py index 2d6edddd..8947216b 100644 --- a/mapmaker/utils/property_mixin.py +++ b/mapmaker/utils/property_mixin.py @@ -32,8 +32,15 @@ def __init__(self, properties: Optional[dict[str, Any]]=None): def properties(self): return self.__properties - def pop_property(self, key: str, default: Any=None) -> Any: - return self.__properties.pop(key, default) + def append_property(self, key: str, value: Any) -> None: + if value is None: + return + elif key not in self.__properties: + self.__properties[key] = [value] + elif isinstance(self.__properties[key], list): + self.__properties[key].append(value) + else: + self.__properties[key] = [self.__properties[key], value] def get_property(self, key: str, default: Any=None) -> Any: return self.__properties.get(key, default) @@ -41,6 +48,9 @@ def get_property(self, key: str, default: Any=None) -> Any: def has_property(self, key: str) -> bool: return self.__properties.get(key, '') != '' + def pop_property(self, key: str, default: Any=None) -> Any: + return self.__properties.pop(key, default) + def set_property(self, key: str, value: Any) -> None: if value is None: self.pop_property(key) From 73b10c78c7a5f9e74dab262d27bbaba1dd6b15ec Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 13:31:54 +1300 Subject: [PATCH 052/111] Remove historical deployment note. --- DEPLOY.rst | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 DEPLOY.rst diff --git a/DEPLOY.rst b/DEPLOY.rst deleted file mode 100644 index 7d583877..00000000 --- a/DEPLOY.rst +++ /dev/null @@ -1,14 +0,0 @@ -========== -Deployment -========== - -Put map directory into `~/flatmap-server/flatmaps` on `ubuntu@34.209.7.109` - -:: - - $ scp MAP.tar.gz ubuntu@34.209.7.109: - $ ssh ubuntu@34.209.7.109 - $ cd ~/flatmap-server/flatmaps - $ tar xzf ~/MAP.tar.gz - $ rm ~/MAP.tar.gz - $ ^D \ No newline at end of file From ff1616c11e03a63d4db42d37000e76293bd5c0f5 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:18:19 +1300 Subject: [PATCH 053/111] Update the bounds of a details' layer when it is aligned with features in the base layer. --- mapmaker/flatmap/layers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 2838abf6..1c1d8c05 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -228,6 +228,8 @@ def align_layer(self, feature_alignment: list[tuple[str, str]]): if self.__offset != (0.0, 0.0): for feature in self.features: feature.geometry = shapely.affinity.translate(feature.geometry, xoff=self.__offset[0], yoff=self.__offset[1]) + self.__bounds = (self.__bounds[0] + self.__offset[0], self.__bounds[1] + self.__offset[1], + self.__bounds[2] + self.__offset[0], self.__bounds[3] + self.__offset[1]) def create_feature_groups(self): #=============================== From 34ff1326821097a0a6c14bb695ec891d11f38ecf Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:21:31 +1300 Subject: [PATCH 054/111] Export the `extent` (LngLat coordinates) of a MapLayer in its metadata. --- mapmaker/flatmap/__init__.py | 1 + mapmaker/flatmap/layers.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index a2a67fd5..fb6eeddb 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -385,6 +385,7 @@ def layer_metadata(self): 'id': layer.id, 'description': layer.description, 'detail-layer': layer.detail_layer, + 'extent': layer.extent, 'image-layers': [ { 'id': raster_layer.id, 'options': { diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 1c1d8c05..e7ea5b5b 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -31,8 +31,8 @@ from mapmaker import ZOOM_OFFSET_FROM_BASE from mapmaker.exceptions import GroupValueError -from mapmaker.geometry import connect_dividers, extend_line, make_boundary -from mapmaker.geometry import bounds_centroid, MapBounds, merge_bounds, translate_extent +from mapmaker.geometry import bounds_to_extent, connect_dividers, extend_line, make_boundary +from mapmaker.geometry import bounds_centroid, MapBounds, MapExtent, merge_bounds, translate_extent from mapmaker.geometry import save_geometry, Transform from mapmaker.settings import settings from mapmaker.utils import FilePath, log @@ -168,6 +168,10 @@ def detail_features(self) -> list[Feature]: def detail_layer(self) -> bool: return (self.__source.base_feature is not None) + @property + def extent(self) -> MapExtent: + return bounds_to_extent(self.__bounds) + @property def max_zoom(self) -> int: return self.__max_zoom From 4b10ec448050f9a088030280768bbe6feb2773e2 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:35:46 +1300 Subject: [PATCH 055/111] Add `zoom points` to functional maps when a feature has a detailed view or a link to another flatmap. --- mapmaker/flatmap/__init__.py | 23 +++++++++++++++++++++++ mapmaker/flatmap/layers.py | 6 +++++- mapmaker/flatmap/manifest.py | 10 ++++++++++ mapmaker/output/__init__.py | 3 ++- mapmaker/sources/__init__.py | 6 +++++- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index fb6eeddb..cc33444f 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -376,6 +376,29 @@ def add_source_layers(self, layer_number: int, source: 'MapSource'): and source.kind not in SOURCE_DETAIL_KINDS): raise ValueError('Can only have a single base map') + """ + The ``feature`` has details that appear when zoomed. + """ + def add_details_layer(self, feature: Feature, details_layer: str, description: Optional[str]=None): + #================================================================================================== + if feature.layer is not None: + feature.set_property('details-layer', details_layer) + zoom_point = self.add_zoom_point(feature, description) + if zoom_point is not None: + zoom_point.set_property('details-layer', details_layer) + + def add_zoom_point(self, feature: Feature, description: Optional[str]=None) -> Optional[Feature]: + #================================================================================================ + if feature.layer is not None: + zoom_point = self.new_feature(feature.properties['layer'], feature.geometry.centroid, { + 'kind': 'zoom-point', + 'tile-layer': feature.properties['tile-layer'] + }) + if description is not None: + zoom_point.set_property('label', description) + feature.layer.add_feature(zoom_point) + return zoom_point + def layer_metadata(self): #======================== metadata = [] diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index e7ea5b5b..2a0c190d 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -34,7 +34,7 @@ from mapmaker.geometry import bounds_to_extent, connect_dividers, extend_line, make_boundary from mapmaker.geometry import bounds_centroid, MapBounds, MapExtent, merge_bounds, translate_extent from mapmaker.geometry import save_geometry, Transform -from mapmaker.settings import settings +from mapmaker.settings import MAP_KIND, settings from mapmaker.utils import FilePath, log if TYPE_CHECKING: @@ -202,6 +202,10 @@ def add_feature(self, feature: Feature): # type: ignore super().add_feature(feature, map_layer=self) if feature.has_property('details'): self.__detail_features.append(feature) + if self.flatmap.map_kind == MAP_KIND.FUNCTIONAL: + if (hyperlinks := feature.get_property('hyperlinks')) is not None: + if 'flatmap' in hyperlinks and (zoom_point := self.flatmap.add_zoom_point(feature)) is not None: + zoom_point.set_property('hyperlinks', hyperlinks) def __find_feature(self, feature_id: str) -> Optional[Feature]: #============================================================== diff --git a/mapmaker/flatmap/manifest.py b/mapmaker/flatmap/manifest.py index 018c097b..7cb51541 100644 --- a/mapmaker/flatmap/manifest.py +++ b/mapmaker/flatmap/manifest.py @@ -183,7 +183,9 @@ def __init__(self, description: dict, manifest: 'Manifest'): self.__href = href self.__kind = description.get('kind', '') self.__boundary = description.get('boundary') + self.__description = description.get('description') self.__detail_fit = description.get('detail-fit') + self.__details = description.get('details') self.__feature = description.get('feature') self.__alignment = [(features[0], features[1]) for features in description.get('alignment', [])] self.__source_range = (([int(n) for n in source_range] if isinstance(source_range, list) @@ -212,10 +214,18 @@ def background_source(self) -> Optional[SourceBackground]: def boundary(self) -> Optional[str]: return self.__boundary + @property + def description(self) -> Optional[str]: + return self.__description + @property def detail_fit(self) -> Optional[str]: return self.__detail_fit + @property + def details(self) -> Optional[str]: + return self.__details + @property def feature(self) -> Optional[str]: return self.__feature diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index 1e359efb..8be659e6 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -28,7 +28,8 @@ 'children', # list[int] 'class', 'colour', # str - 'description', + 'description', # str + 'details-layer', # str The identifier of the layer with details about the feature 'error', 'fc-class', 'fc-kind', diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index de612f1c..18a1101d 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -143,7 +143,11 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): if (feature := flatmap.get_feature(source_manifest.feature)) is None: raise ValueError(f'Unknown source feature: {source_manifest.feature}') feature.set_property('maxzoom', source_manifest.zoom-1) - feature.set_property('kind', 'expandable') + if source_manifest.kind == 'functional': + details_for = source_manifest.details if source_manifest.details is not None else source_manifest.feature + if (detail_feature := flatmap.get_feature(details_for)) is None: + raise ValueError(f'Unknown source feature: {details_for}') + flatmap.add_details_layer(detail_feature, self.id, source_manifest.description) self.__min_zoom = source_manifest.zoom self.__base_feature = feature else: From 6a19d129199d0f587f8ddd5054b21c2e3f402abf Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:37:18 +1300 Subject: [PATCH 056/111] For functional maps, export the association between a feature and its connection to a detailed layer. --- mapmaker/flatmap/__init__.py | 12 +++++++++++- mapmaker/output/__init__.py | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index cc33444f..e9cb0d88 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -18,7 +18,7 @@ # #=============================================================================== -from collections import OrderedDict +from collections import defaultdict, OrderedDict from datetime import datetime, timezone import os from typing import Optional, TYPE_CHECKING @@ -211,6 +211,7 @@ def initialise(self): self.__features_with_name: dict[str, Feature] = {} self.__last_geojson_id = 0 self.__features_by_geojson_id: dict[int, Feature] = {} + self.__associated_layers: defaultdict[str, list[int]] = defaultdict(list) # Used to find annotated features containing a region self.__feature_search = None @@ -295,6 +296,9 @@ def new_feature(self, layer_id: str, geometry, properties, is_group=False) -> Fe if self.map_kind == MAP_KIND.FUNCTIONAL: if (name := properties.get('name', '')) != '': self.__features_with_name[f'{layer_id}/{name.replace(" ", "_")}'] = feature + if (associated_layers := properties.get('associated-details')) is not None: + for layer in associated_layers: + self.__associated_layers[layer].append(feature.geojson_id) return feature def network_feature(self, feature: Feature) -> bool: @@ -386,6 +390,12 @@ def add_details_layer(self, feature: Feature, details_layer: str, description: O zoom_point = self.add_zoom_point(feature, description) if zoom_point is not None: zoom_point.set_property('details-layer', details_layer) + # Set the ``associated-details`` property for connections to features associated with the details layer + for geojson_id in self.__associated_layers.get(details_layer, []): + if (associated_feature := self.get_feature_by_geojson_id(geojson_id)) is not None: + for connection_id in associated_feature.get_property('connections', []): + if (connection := self.get_feature(connection_id)) is not None: + connection.append_property('associated-details', details_layer) def add_zoom_point(self, feature: Feature, description: Optional[str]=None) -> Optional[Feature]: #================================================================================================ diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index 8be659e6..e81780fd 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -24,6 +24,7 @@ EXPORTED_FEATURE_PROPERTIES = [ 'cd-class', # str + 'associated-details', # Optional[str|list[str]] Identifiers of detailed layers associated with the feature 'centreline', # bool 'children', # list[int] 'class', @@ -72,6 +73,7 @@ ENCODED_FEATURE_PROPERTIES = [ 'hyperlinks', # Optional[list[dict[str, str]]] # id, url + 'associated-details', # Optional[str|list[str]] ] #=============================================================================== From 3bb4adc8d2dd550d9b2702dea9a0a667dd7d31d5 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:40:28 +1300 Subject: [PATCH 057/111] A feature can have a `hyperlink` (a single URL) as well as `hyperlinks` (a list of identifiers). --- mapmaker/output/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index e81780fd..4bc217ee 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -36,6 +36,7 @@ 'fc-kind', 'featureId', # int 'group', + 'hyperlink', # Optional[str] 'hyperlinks', # Optional[list[dict[str, str]]] # id, url 'id', # Optional[str] 'invisible', # bool From 25e8d916f7d643e7bd4274f8b19c4564591fe5b8 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:41:46 +1300 Subject: [PATCH 058/111] Export the `target` property (of a Connection on a functional map), as well as the `source`. --- mapmaker/output/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index 4bc217ee..8e3b8d94 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -59,6 +59,7 @@ 'stroke', # str 'stroke-width', # float 'path-ids', # list[str] + 'target', # str 'taxons', # list[str] 'tile-layer', 'type', From 1365a7e8bb7f81b2fb98653b41062a55d93abcb0 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:42:12 +1300 Subject: [PATCH 059/111] Tidying -- remove unused exported properties. --- mapmaker/output/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mapmaker/output/__init__.py b/mapmaker/output/__init__.py index 8e3b8d94..5cfe7e54 100644 --- a/mapmaker/output/__init__.py +++ b/mapmaker/output/__init__.py @@ -23,7 +23,6 @@ # is saved as GeoJSON. EXPORTED_FEATURE_PROPERTIES = [ - 'cd-class', # str 'associated-details', # Optional[str|list[str]] Identifiers of detailed layers associated with the feature 'centreline', # bool 'children', # list[int] @@ -32,8 +31,6 @@ 'description', # str 'details-layer', # str The identifier of the layer with details about the feature 'error', - 'fc-class', - 'fc-kind', 'featureId', # int 'group', 'hyperlink', # Optional[str] @@ -53,12 +50,12 @@ 'nodeId', 'opacity', # float 'parents', # list[int] + 'path-ids', # list[str] 'scale', 'sckan', - 'source', + 'source', # str 'stroke', # str 'stroke-width', # float - 'path-ids', # list[str] 'target', # str 'taxons', # list[str] 'tile-layer', @@ -74,8 +71,8 @@ #=============================================================================== ENCODED_FEATURE_PROPERTIES = [ - 'hyperlinks', # Optional[list[dict[str, str]]] # id, url 'associated-details', # Optional[str|list[str]] + 'hyperlinks', # Optional[list[dict[str, str]]] # id, url ] #=============================================================================== From 7175cc759c7b8f435a640ad77247ce68a21e47a0 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 12 Mar 2025 14:44:19 +1300 Subject: [PATCH 060/111] Make sure exported `associated-details` is an array. --- mapmaker/properties/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mapmaker/properties/__init__.py b/mapmaker/properties/__init__.py index de36abe9..9a275154 100644 --- a/mapmaker/properties/__init__.py +++ b/mapmaker/properties/__init__.py @@ -165,6 +165,9 @@ def __set_feature_properties(self, features): if isinstance(features, dict): for id, properties in features.items(): id = id.replace(" ", "_") + if (associated_details := properties.get('associated-details')) is not None: + if isinstance(associated_details, str): + properties['associated-details'] = [associated_details] self.__properties_by_id[id].update(properties) if (properties.get('type') == 'nerve' and (entity := properties.get('models')) is not None): From 91d6d07cd259826d828dd1a1baff9c853434318a Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 20 Mar 2025 21:14:29 +1300 Subject: [PATCH 061/111] Include the map's style in its index -- this makes it easily available from a map server. --- mapmaker/flatmap/__init__.py | 6 ++++++ mapmaker/maker.py | 9 ++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index e9cb0d88..c63bd870 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -195,6 +195,12 @@ def initialise(self): knowledge = get_knowledge(self.__models) if 'label' in knowledge: self.__metadata['describes'] = knowledge['label'] + if self.map_kind == MAP_KIND.FUNCTIONAL: + self.__metadata['style'] = 'functional' + elif self.map_kind == MAP_KIND.CENTRELINE: + self.__metadata['style'] = 'centreline' + else: + self.__metadata['style'] = 'anatomical' self.__entities = set() diff --git a/mapmaker/maker.py b/mapmaker/maker.py index d1a92356..729a1a74 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -550,7 +550,8 @@ def __save_metadata(self): 'max-zoom': self.__zoom[1], 'bounds': self.__flatmap.extent, 'version': FLATMAP_VERSION, - 'image-layers': len(self.__raster_layers) > 0 + 'image-layers': len(self.__raster_layers) > 0, + 'style': metadata['style'] } if self.__uuid is not None: map_index['uuid'] = self.__uuid @@ -559,12 +560,6 @@ def __save_metadata(self): if self.__manifest.biological_sex is not None: map_index['biologicalSex'] = self.__manifest.biological_sex map_index['authoring'] = settings.get('authoring', False) - if self.__flatmap.map_kind == MAP_KIND.FUNCTIONAL: - map_index['style'] = 'functional' - elif self.__flatmap.map_kind == MAP_KIND.CENTRELINE: - map_index['style'] = 'centreline' - else: - map_index['style'] = 'anatomical' if git_status is not None: map_index['git-status'] = git_status if len(self.__sckan_provenance): From 08a08e0fd1f1ed980b33bf097d7298a7d6ba01fd Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 21 Mar 2025 15:01:29 +1300 Subject: [PATCH 062/111] SVG: improve how missing fonts are handled. --- mapmaker/sources/svg/__init__.py | 13 +++++++++++-- mapmaker/sources/svg/rasteriser.py | 10 ++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 7aa6425a..7f042168 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -566,12 +566,21 @@ def __process_text(self, element, properties, transform: Transform) -> Optional[ if element_text == '': element_text = ' ' font_manager = skia.FontMgr() - for font_family in style_rules.get('font-family', 'Calibri').split(','): + font_families = style_rules.get('font-family', 'Calibri') + for font_family in font_families.split(','): type_face = font_manager.matchFamilyStyle(font_family, font_style) if type_face is not None: break if type_face is None: - type_face = font_manager.matchFamilyStyle(None, font_style) + if 'Calibri' not in font_families: + log.warning('Cannot get font information (missing fonts?), trying Calibri', font=font_families, text=element_text) + type_face = font_manager.matchFamilyStyle('Calibri', font_style) + if type_face is None: + log.warning('Cannot get font information for Calibri', font='Calibri') + type_face = font_manager.matchFamilyStyle(None, font_style) + else: + log.warning('Cannot get font information (missing fonts?)', font=font_families) + type_face = font_manager.matchFamilyStyle(None, font_style) font = skia.Font(type_face, length_as_points(style_rules.get('font-size', 10))) bounds = skia.Rect() width = font.measureText(element_text, skia.TextEncoding.kUTF8, bounds) diff --git a/mapmaker/sources/svg/rasteriser.py b/mapmaker/sources/svg/rasteriser.py index 1e8475db..25a0c234 100644 --- a/mapmaker/sources/svg/rasteriser.py +++ b/mapmaker/sources/svg/rasteriser.py @@ -237,12 +237,18 @@ def __init__(self, text, attribs, parent_transform: Transform, skia.FontStyle.kUpright_Slant) type_face = None font_manager = skia.FontMgr() - for font_family in style_rules.get('font-family', 'Calibri').split(','): + font_families = style_rules.get('font-family', 'Calibri') + for font_family in font_families.split(','): type_face = font_manager.matchFamilyStyle(font_family, font_style) if type_face is not None: break if type_face is None: - type_face = font_manager.matchFamilyStyle(None, font_style) + if 'Calibri' not in font_families: + type_face = font_manager.matchFamilyStyle('Calibri', font_style) + if type_face is None: + type_face = font_manager.matchFamilyStyle(None, font_style) + else: + type_face = font_manager.matchFamilyStyle(None, font_style) self.__font = skia.Font(type_face, length_as_points(style_rules.get('font-size', 10))) self.__pos = [float(attribs.get('x', 0)), float(attribs.get('y', 0))] From b90a3fd37db64a998f2ff33d26729e680021be08 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sun, 23 Mar 2025 11:44:31 +1300 Subject: [PATCH 063/111] feat: instead of having the map's `style` in its index, allow a map's `kind` (in the manifest) to be a space separated list and export it as `map-kinds`. --- mapmaker/flatmap/__init__.py | 5 ++--- mapmaker/flatmap/manifest.py | 14 ++++++++++++-- mapmaker/maker.py | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index c63bd870..5a7af70e 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -63,9 +63,7 @@ def __init__(self, manifest: Manifest, maker: 'MapMaker', annotator: Optional['A self.__id = maker.id self.__uuid = maker.uuid self.__map_dir = maker.map_dir - self.__map_kind = (MAP_KIND.FUNCTIONAL if manifest.kind == 'functional' - else MAP_KIND.CENTRELINE if manifest.kind == 'centreline' - else MAP_KIND.ANATOMICAL) + self.__map_kind = manifest.map_kind self.__manifest = manifest self.__local_id = manifest.id self.__models = manifest.models @@ -201,6 +199,7 @@ def initialise(self): self.__metadata['style'] = 'centreline' else: self.__metadata['style'] = 'anatomical' + self.__metadata['map-kinds'] = self.__manifest.map_kinds self.__entities = set() diff --git a/mapmaker/flatmap/manifest.py b/mapmaker/flatmap/manifest.py index 7cb51541..9bb15104 100644 --- a/mapmaker/flatmap/manifest.py +++ b/mapmaker/flatmap/manifest.py @@ -60,6 +60,7 @@ def clone_from_with_timeout(repo_path: str, working_directory: str) -> git.Repo: #=============================================================================== +from mapmaker.settings import MAP_KIND from mapmaker.utils import log, FilePath #=============================================================================== @@ -336,6 +337,11 @@ def __init__(self, manifest_path, single_file=None, id=None, ignore_git=False, m if not ignore_git and self.__uncommitted: raise ValueError("Not all sources are commited into git -- was the '--authoring' or '--ignore-git' option intended?") + self.__map_kinds = self.__manifest.get('kind', 'anatomical').split() + self.__map_kind = (MAP_KIND.FUNCTIONAL if 'functional' in self.__map_kinds + else MAP_KIND.CENTRELINE if 'centreline' in self.__map_kinds + else MAP_KIND.ANATOMICAL) + @property def anatomical_map(self): return self.__manifest.get('anatomicalMap') @@ -393,8 +399,12 @@ def id(self): return self.__manifest['id'] @property - def kind(self): #! Either ``anatomical`` or ``functional`` - return self.__manifest.get('kind', 'anatomical') + def map_kinds(self) -> list[str]: + return self.__map_kinds + + @property + def map_kind(self) -> MAP_KIND: + return self.__map_kind @property def raw_manifest(self): diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 729a1a74..d81ef939 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -551,7 +551,7 @@ def __save_metadata(self): 'bounds': self.__flatmap.extent, 'version': FLATMAP_VERSION, 'image-layers': len(self.__raster_layers) > 0, - 'style': metadata['style'] + 'map-kinds': metadata['map-kinds'] } if self.__uuid is not None: map_index['uuid'] = self.__uuid From ba3d3f0798cc38e5dc04facd1a63b5ba9938790b Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sun, 23 Mar 2025 12:08:21 +1300 Subject: [PATCH 064/111] fix: at least 90% of a text shape's area has to be inside its parent container. --- mapmaker/shapes/classify.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 7f1a1a4d..65dee236 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -292,8 +292,10 @@ def __set_parent_relationships(self): for shape in bbox_intersecting_shapes: if parent.shape_type != SHAPE_TYPE.TEXT and parent.id != shape.id: # A text shape is always a child even when not properly contained - if (shape.shape_type == SHAPE_TYPE.TEXT - or shapely.contains_properly(parent.geometry, shape.geometry)): + if (shapely.contains_properly(parent.geometry, shape.geometry) + # Text shapes need say at 90% containment... + or (shape.shape_type == SHAPE_TYPE.TEXT + and parent.geometry.intersection(shape.geometry).area/shape.geometry.area > 0.9)): parent_child.append((parent, shape)) last_child_id = None for (parent, child) in sorted(parent_child, key=lambda s: (s[1].id, s[0].geometry.area)): From 2e96d338d56019b2607474386ea8403298a252b4 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 11 Apr 2025 11:19:40 +1200 Subject: [PATCH 065/111] Update Python packages. --- poetry.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 10566960..d6174c58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3571,14 +3571,14 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.51.1" +version = "4.51.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.9.0" groups = ["alignments"] files = [ - {file = "transformers-4.51.1-py3-none-any.whl", hash = "sha256:c7038e216afb2a3e9b00dd12d87ad5e3af4c30895f70b28e92f65459eded0161"}, - {file = "transformers-4.51.1.tar.gz", hash = "sha256:206ea0b75dfde142ed7495b911da76579dce6ea249cc3695fdd29a544a9e007b"}, + {file = "transformers-4.51.2-py3-none-any.whl", hash = "sha256:5cb8259098b75ff4b5dd04533a318f7c4750d5307d9617e6d0593526432c404d"}, + {file = "transformers-4.51.2.tar.gz", hash = "sha256:ed221c31581e97127cff5de775b05f05d19698b439d7d638ff445502a7f37331"}, ] [package.dependencies] @@ -3715,14 +3715,14 @@ urllib3 = ">=2" [[package]] name = "typing-extensions" -version = "4.13.1" +version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["alignments", "dev"] files = [ - {file = "typing_extensions-4.13.1-py3-none-any.whl", hash = "sha256:4b6cf02909eb5495cfbc3f6e8fd49217e6cc7944e145cdda8caa3734777f9e69"}, - {file = "typing_extensions-4.13.1.tar.gz", hash = "sha256:98795af00fb9640edec5b8e31fc647597b4691f099ad75f469a2616be1a76dff"}, + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, ] [[package]] @@ -3751,14 +3751,14 @@ files = [ [[package]] name = "urllib3" -version = "2.3.0" +version = "2.4.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main", "alignments", "docs"] files = [ - {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, - {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, + {file = "urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"}, + {file = "urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466"}, ] [package.extras] From 7bdb3aa454a2936946cfaa2678f692d1d7147eea Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 21 Apr 2025 15:48:30 +1200 Subject: [PATCH 066/111] Always set the map's `style` (as well as `map-kinds`) in its `index.json` file, as the viewer uses it. --- mapmaker/maker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 94cec75f..8fbccfd1 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -553,6 +553,7 @@ def __save_metadata(self): 'bounds': self.__flatmap.extent, 'version': FLATMAP_VERSION, 'image-layers': len(self.__raster_layers) > 0, + 'style': metadata['style'], 'map-kinds': metadata['map-kinds'] } if self.__uuid is not None: From 4acafa0874710eeafc55b9448d227840806b8f09 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 21 Apr 2025 15:54:01 +1200 Subject: [PATCH 067/111] shapes: text can be slightly outside of its parent container. --- mapmaker/shapes/classify.py | 3 +-- mapmaker/shapes/constants.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 65dee236..5d92bfc3 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -293,9 +293,8 @@ def __set_parent_relationships(self): if parent.shape_type != SHAPE_TYPE.TEXT and parent.id != shape.id: # A text shape is always a child even when not properly contained if (shapely.contains_properly(parent.geometry, shape.geometry) - # Text shapes need say at 90% containment... or (shape.shape_type == SHAPE_TYPE.TEXT - and parent.geometry.intersection(shape.geometry).area/shape.geometry.area > 0.9)): + and parent.geometry.intersection(shape.geometry).area/shape.geometry.area > MIN_TEXT_INSIDE)): parent_child.append((parent, shape)) last_child_id = None for (parent, child) in sorted(parent_child, key=lambda s: (s[1].id, s[0].geometry.area)): diff --git a/mapmaker/shapes/constants.py b/mapmaker/shapes/constants.py index f01562a3..e7a01685 100644 --- a/mapmaker/shapes/constants.py +++ b/mapmaker/shapes/constants.py @@ -45,6 +45,9 @@ MAX_TEXT_VERTICAL_OFFSET = 3 # Between cluster baseline and baselines of text in the cluster TEXT_BASELINE_OFFSET = -14.5 # From vertical centre of a component +# Text shapes need at least 80% containment in their parent +MIN_TEXT_INSIDE = 0.8 + #=============================================================================== # Scaling factors for styling components and connections in map viewers From 19ccd5eb981e382d8ae7e02dc2552942677052c3 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 22 Apr 2025 11:51:31 +1200 Subject: [PATCH 068/111] Add map's created timestamp to its `index.json`. --- mapmaker/maker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 8fbccfd1..9cd4adee 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -554,7 +554,8 @@ def __save_metadata(self): 'version': FLATMAP_VERSION, 'image-layers': len(self.__raster_layers) > 0, 'style': metadata['style'], - 'map-kinds': metadata['map-kinds'] + 'map-kinds': metadata['map-kinds'], + 'created': metadata['created'] } if self.__uuid is not None: map_index['uuid'] = self.__uuid From c207ee285306486c6e10be5f739c55357ddcdf14 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 22 Apr 2025 21:46:48 +1200 Subject: [PATCH 069/111] A zoom point marker must have the same model as its underlying feature. --- mapmaker/flatmap/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 5a7af70e..2c190133 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -411,6 +411,8 @@ def add_zoom_point(self, feature: Feature, description: Optional[str]=None) -> O }) if description is not None: zoom_point.set_property('label', description) + if (models := feature.models) is not None: + zoom_point.set_property('models', models) feature.layer.add_feature(zoom_point) return zoom_point From 8fdb408236ef91c8887617578b77b3ba7481aa0b Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 22 Apr 2025 22:16:34 +1200 Subject: [PATCH 070/111] functional: increase ZOOM_OFFSET_FROM_BASE, for showing rasterised image of detailed map on base map before details show. --- mapmaker/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapmaker/__init__.py b/mapmaker/__init__.py index 0bd92708..f8e04d5d 100644 --- a/mapmaker/__init__.py +++ b/mapmaker/__init__.py @@ -31,8 +31,8 @@ MIN_ZOOM = 2 #: Default minimum zoom level for generated flatmaps MAX_ZOOM = 10 #: Default maximum zoom level for generated flatmaps - -ZOOM_OFFSET_FROM_BASE = 1 +# For showing rasterised image of detailed map on its base map before the details show +ZOOM_OFFSET_FROM_BASE = 2 #=============================================================================== From 26d62a6a26182c7d121a740f8d1c7c2df9ce1036 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Tue, 22 Apr 2025 22:18:15 +1200 Subject: [PATCH 071/111] options: default to have `--no-path-layout` enabled and use new `--path-layout` option to attempt transit map layout. --- mapmaker/__main__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mapmaker/__main__.py b/mapmaker/__main__.py index 22deaa93..c4c9667d 100644 --- a/mapmaker/__main__.py +++ b/mapmaker/__main__.py @@ -57,10 +57,12 @@ def arg_parser(): help="Don't check if functional connectivity neurons are known in SCKAN. Sets `--invalid-neurons` option") generation_options.add_argument('--invalid-neurons', dest='invalidNeurons', action='store_true', help="Include functional connectivity neurons that aren't known in SCKAN") - generation_options.add_argument('--no-path-layout', dest='noPathLayout', action='store_true', - help="Don't do `TransitMap` optimisation of paths") generation_options.add_argument('--path-arrows', dest='pathArrows', action='store_true', help="Render arrows at the terminal nodes of paths") + generation_options.add_argument('--path-layout', dest='pathLayout', action='store_true', + help="Do `TransitMap` optimisation of paths") + generation_options.add_argument('--no-path-layout', dest='noPathLayout', action='store_true', + help="Don't do `TransitMap` optimisation of paths") generation_options.add_argument('--publish', metavar='SPARC_DATASET', help="Create a SPARC Dataset containing the map's sources and the generated map") generation_options.add_argument('--sckan-version', dest='sckanVersion', choices=['production', 'staging'], @@ -119,6 +121,8 @@ def main(): import sys parser = arg_parser() args = parser.parse_args() + if not args.pathLayout: + args.noPathLayout = True try: mapmaker = MapMaker({k:v for k, v in vars(args).items() if not (v is None or isinstance(v, bool) and v == False)}) mapmaker.make() From dd6100700b92da83d99cf957ce22deb97afcedc9 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 24 Apr 2025 20:37:18 +1200 Subject: [PATCH 072/111] Include the parent layer and the zoom point feature id with a detail layer's metadata. --- mapmaker/flatmap/__init__.py | 7 +++++-- mapmaker/flatmap/layers.py | 11 +++++++++++ mapmaker/sources/__init__.py | 7 ++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 2c190133..b6b46e0a 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -388,8 +388,8 @@ def add_source_layers(self, layer_number: int, source: 'MapSource'): """ The ``feature`` has details that appear when zoomed. """ - def add_details_layer(self, feature: Feature, details_layer: str, description: Optional[str]=None): - #================================================================================================== + def add_details_layer(self, feature: Feature, details_layer: str, description: Optional[str]=None) -> Optional[int]: + #=================================================================================================================== if feature.layer is not None: feature.set_property('details-layer', details_layer) zoom_point = self.add_zoom_point(feature, description) @@ -401,6 +401,7 @@ def add_details_layer(self, feature: Feature, details_layer: str, description: O for connection_id in associated_feature.get_property('connections', []): if (connection := self.get_feature(connection_id)) is not None: connection.append_property('associated-details', details_layer) + return zoom_point.geojson_id if zoom_point else None def add_zoom_point(self, feature: Feature, description: Optional[str]=None) -> Optional[Feature]: #================================================================================================ @@ -425,6 +426,8 @@ def layer_metadata(self): 'id': layer.id, 'description': layer.description, 'detail-layer': layer.detail_layer, + 'parent-layer': layer.parent_layer, + 'zoom-point': layer.zoom_point_id, 'extent': layer.extent, 'image-layers': [ { 'id': raster_layer.id, diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 2a0c190d..1be65727 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -148,6 +148,7 @@ def __init__(self, id: str, source: 'MapSource', exported=False, min_zoom: Optio self.__min_zoom = min_zoom if min_zoom is not None else source.min_zoom self.__max_zoom = source.max_zoom self.__offset = (0.0, 0.0) + self.__zoom_point_id = None @property def boundary_feature(self) -> Optional[Feature]: @@ -187,6 +188,12 @@ def min_zoom(self) -> int: def outer_geometry(self) -> BaseGeometry: return self.__outer_geometry + @property + def parent_layer(self) -> Optional[str]: + if (base_feature := self.__source.base_feature) is not None: + return base_feature.get_property('layer') + return None + @property def raster_layers(self) -> list['RasterLayer']: return self.__raster_layers @@ -195,6 +202,10 @@ def raster_layers(self) -> list['RasterLayer']: def source(self) -> 'MapSource': return self.__source + @property + def zoom_point_id(self) -> Optional[int]: + return self.source.zoom_point_id + def add_feature(self, feature: Feature): # type: ignore #======================================= if self.__min_zoom is not None and not feature.has_property('minzoom'): diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index 18a1101d..edcb5b25 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -135,6 +135,7 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): self.__bounds: MapBounds = (0, 0, 0, 0) self.__raster_sources = None self.__background_raster_source = source_manifest.background_source + self.__zoom_point_id = None if self.__kind in SOURCE_DETAIL_KINDS: if source_manifest.feature is None: raise ValueError('A `detail` source must specify an existing `feature`') @@ -147,7 +148,7 @@ def __init__(self, flatmap: 'FlatMap', source_manifest: SourceManifest): details_for = source_manifest.details if source_manifest.details is not None else source_manifest.feature if (detail_feature := flatmap.get_feature(details_for)) is None: raise ValueError(f'Unknown source feature: {details_for}') - flatmap.add_details_layer(detail_feature, self.id, source_manifest.description) + self.__zoom_point_id = flatmap.add_details_layer(detail_feature, self.id, source_manifest.description) self.__min_zoom = source_manifest.zoom self.__base_feature = feature else: @@ -233,6 +234,10 @@ def source_range(self) -> Optional[list[int]]: def transform(self) -> Optional[Transform]: return None + @property + def zoom_point_id(self): + return self.__zoom_point_id + def add_layer(self, layer: MapLayer): #==================================== layer.create_feature_groups() From 5d1b2099204cc54ef4d6c26999f1ba5d52e76085 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 21 May 2025 10:26:12 +1200 Subject: [PATCH 073/111] functional: `zoom-point` and related layer metadata depends on the layer having a parent. --- mapmaker/flatmap/__init__.py | 7 ++++--- mapmaker/flatmap/layers.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mapmaker/flatmap/__init__.py b/mapmaker/flatmap/__init__.py index 9b84b666..43ee59e0 100644 --- a/mapmaker/flatmap/__init__.py +++ b/mapmaker/flatmap/__init__.py @@ -426,9 +426,6 @@ def layer_metadata(self): 'id': layer.id, 'description': layer.description, 'detail-layer': layer.detail_layer, - 'parent-layer': layer.parent_layer, - 'zoom-point': layer.zoom_point_id, - 'extent': layer.extent, 'image-layers': [ { 'id': raster_layer.id, 'options': { @@ -444,6 +441,10 @@ def layer_metadata(self): map_layer['min-zoom'] = layer.min_zoom if layer.max_zoom is not None: map_layer['max-zoom'] = layer.max_zoom + if layer.parent_layer is not None: + map_layer['extent'] = layer.extent + map_layer['parent-layer'] = layer.parent_layer + map_layer['zoom-point'] = layer.zoom_point_id metadata.append(map_layer) return metadata diff --git a/mapmaker/flatmap/layers.py b/mapmaker/flatmap/layers.py index 1be65727..93960bf3 100644 --- a/mapmaker/flatmap/layers.py +++ b/mapmaker/flatmap/layers.py @@ -103,6 +103,10 @@ def min_zoom(self) -> Optional[int]: def offset(self) -> tuple[float, float]: return (0.0, 0.0) + @property + def parent_layer(self) -> Optional[str]: + return None + @property def raster_layers(self) -> list['RasterLayer']: return [] From 6688bfb31ca38eac59c7f0b688cf5a840b68e560 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 6 Jun 2025 14:31:14 +1200 Subject: [PATCH 074/111] geometry: add `is_identity()`, `scale()`, and `translate()` to our Transform class. --- mapmaker/geometry/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mapmaker/geometry/__init__.py b/mapmaker/geometry/__init__.py index fc1b1cbf..d77e8034 100644 --- a/mapmaker/geometry/__init__.py +++ b/mapmaker/geometry/__init__.py @@ -140,6 +140,10 @@ def Translate(cls, translate: tuple[float, float]) -> Self: [0, 1, translate[1]], [0, 0, 1]]) + @property + def is_identity(self) -> bool: + return np.allclose(self.__matrix, np.identity(3)) + @property def matrix(self) -> np.ndarray: return self.__matrix @@ -169,6 +173,12 @@ def rotate_angle(self, angle): angle -= 2*PI return angle + def scale(self, scale: float) -> 'Transform': + #================================ + return Transform([[scale*self.__matrix[0, 0], self.__matrix[0, 1], self.__matrix[0, 2]], + [ self.__matrix[1, 0], scale*self.__matrix[1, 1], self.__matrix[1, 2]], + [ self.__matrix[2, 0], self.__matrix[2, 1], self.__matrix[2, 2]]]) + def scale_length(self, length): #============================== scaling = transforms3d.affines.decompose(self.__matrix)[2] @@ -189,6 +199,12 @@ def transform_point(self, point) -> tuple[float, float]: #======================================================= return tuple(self.__matrix@[point[0], point[1], 1.0])[:2] + def translate(self, translation: tuple[float, float]) -> 'Transform': + #==================================================================== + return Transform([[self.__matrix[0, 0], self.__matrix[0, 1], translation[0] + self.__matrix[0, 2]], + [self.__matrix[1, 0], self.__matrix[1, 1], translation[1] + self.__matrix[1, 2]], + [self.__matrix[2, 0], self.__matrix[2, 1], self.__matrix[2, 2]]]) + #=============================================================================== def ellipse_point(a, b, theta): From 04e51a6bdb206e2b929b69c489ad1b154c2aee10 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 6 Jun 2025 14:35:09 +1200 Subject: [PATCH 075/111] Code tidying. --- mapmaker/sources/__init__.py | 2 +- mapmaker/sources/svg/__init__.py | 2 +- mapmaker/sources/svg/utils.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mapmaker/sources/__init__.py b/mapmaker/sources/__init__.py index edcb5b25..12ab7c5e 100644 --- a/mapmaker/sources/__init__.py +++ b/mapmaker/sources/__init__.py @@ -113,7 +113,7 @@ def mask_image(image, mask_polygon): if image.shape[2] == 4: mask[:, :, 3] = 0 mask_color = (0,)*image.shape[2] - cv2.fillPoly(mask, np.array([mask_polygon.exterior.coords], dtype=np.int32), + cv2.fillPoly(mask, np.array([mask_polygon.exterior.coords], dtype=np.int32), # type: ignore color=mask_color, lineType=cv2.LINE_AA) return cv2.bitwise_or(image, mask) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 7f042168..8d5ce05b 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -170,7 +170,7 @@ def process(self): def create_preview(self): #======================== # Save a cleaned copy of the SVG in the map's output directory. Call after - # connectivity has been generated otherwise thno paths will be in the saved SVG + # connectivity has been generated otherwise no paths will be in the saved SVG cleaner = SVGCleaner(self.__source_file, self.flatmap.properties_store, all_layers=True) cleaner.clean() cleaner.add_connectivity_group(self.flatmap, self.__transform) diff --git a/mapmaker/sources/svg/utils.py b/mapmaker/sources/svg/utils.py index 06a8b43b..d786ffcc 100644 --- a/mapmaker/sources/svg/utils.py +++ b/mapmaker/sources/svg/utils.py @@ -33,7 +33,7 @@ from beziers.quadraticbezier import QuadraticBezier from beziers.segment import Segment as BezierSegment -import shapely.geometry +import shapely from shapely.geometry.base import BaseGeometry import lxml.etree as etree @@ -177,8 +177,8 @@ def svg_markup(element): #=============================================================================== def circle_from_bounds(bounds): - centre = shapely.geometry.Point((bounds[0] + bounds[2])/2.0, - (bounds[1] + bounds[3])/2.0) + centre = shapely.Point((bounds[0] + bounds[2])/2.0, + (bounds[1] + bounds[3])/2.0) return centre.buffer(math.sqrt(abs((bounds[2] - bounds[0])*(bounds[3] - bounds[1])))/2.0) #=============================================================================== @@ -241,15 +241,15 @@ def __geometry_from_coordinates(coordinates: list[Coordinate], closed: bool, mus raise ValueError("Shape must have closed geometry") if closed and len(coordinates) >= 3: - geometry = shapely.geometry.Polygon(coordinates).buffer(0) + geometry = shapely.Polygon(coordinates).buffer(0) elif must_close == True and len(coordinates) >= 3: # Return a polygon if flagged as `closed` coordinates.append(coordinates[0]) - geometry = shapely.geometry.Polygon(coordinates).buffer(0) + geometry = shapely.Polygon(coordinates).buffer(0) elif len(coordinates) >= 2: ## Warn if start and end point are ``close`` wrt to the length of the line as shape ## may be intended to be closed... (test with ``cardio_8-1``) - geometry = shapely.geometry.LineString(coordinates) + geometry = shapely.LineString(coordinates) else: geometry = None From eb4a421d2a3a4361a5d361ff8aa0e3d5b9971d82 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Wed, 11 Jun 2025 11:44:51 +1200 Subject: [PATCH 076/111] Update Python project file to remove Poetry specific configuration. --- pyproject.toml | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2321e3e4..3001d828 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,38 +49,37 @@ dependencies = [ "tippecanoe>=2.72.0", ] -[tool.poetry.group.dev.dependencies] -attribution = "^1.7.1" -mypy = "^0.982" -sphinx = "^8.1" -sphinx-rtd-theme = "^3.0.1" -sphinx-argparse = "^0.5.2" - -[tool.poetry.group.alignments] -optional = true - -[tool.poetry.group.alignments.dependencies] -pandas = "^2.2.2" -sentence-transformers = "^2.2.2" -torch = "^2.1.1" - -[tool.poetry.group.tools] -optional = true - -[tool.poetry.group.tools.dependencies] -pandas = "^2.2.2" +[dependency-groups] +dev = [ + "attribution<=1.7.1", + "mypy<=0.982", + "sphinx<=8.1", + "sphinx-rtd-theme<=3.0.1", + "sphinx-argparse<=0.5.2", +] +alignments = [ + "pandas<=2.2.2", + "sentence-transformers<=2.2.2", + "torch<=2.1.1", +] +tools = [ + "pandas<=2.2.2", +] [tool.attribution] name = "mapmaker" package = "mapmaker" version_file = true -[tool.poetry.scripts] +[project.scripts] mapmaker = 'mapmaker.__main__:main' [build-system] -requires = ["poetry_core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true [tool.pyright] pythonVersion = "3.12" From 21b91a8d131a848822df1350726bdb91a2d84ce4 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Thu, 12 Jun 2025 12:50:32 +1200 Subject: [PATCH 077/111] Work on bondgraph RDF export. --- mapmaker/output/bondgraph/__init__.py | 212 ++++++++++++++++++++++++++ mapmaker/sources/svg/__init__.py | 11 +- 2 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 mapmaker/output/bondgraph/__init__.py diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py new file mode 100644 index 00000000..b67d8b5b --- /dev/null +++ b/mapmaker/output/bondgraph/__init__.py @@ -0,0 +1,212 @@ +#=============================================================================== +# +# Flatmap maker and annotation tools +# +# Copyright (c) 2019 - 2025 David Brooks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#=============================================================================== + +from datetime import datetime, UTC + +#=============================================================================== + +import lark +from lark import Lark, UnexpectedInput +import rdflib + +#=============================================================================== + +from mapmaker.shapes import Shape, SHAPE_TYPE +from mapmaker.shapes.colours import ColourMatcherDict +from mapmaker.utils.svg import svg_id + +#=============================================================================== + +BG_FRAMEWORK_VERSION = "1.0" + +#=============================================================================== + +NODE_BORDER_TYPES = ColourMatcherDict({ + '#042433': 'bgf:OneNode', # dark green + '#00B050': 'bgf:OneResistance', # green + '#FF0000': 'bgf:ZeroStorage', # red +}) + +#=============================================================================== + +LATEX_GRAMMAR = """ + start: _name + + _name: TEXT super? sub? + | TEXT sub? super? + + super: "^" _block + sub: "_" _block + _block: _brackets | TEXT + + _brackets: "{" [ _name+ | _chem ] "}" + _chem: "\\ce{" _formula "}" + _formula: [ SYMBOL _exponent? ]+ + SYMBOL: /[A-Za-z]+/ + _exponent: NUMBER? _sign? + _sign: "+" | "-" + + NUMBER: /[0-9]+/ + TEXT: /[A-Za-z0-9\\-\\+\\.]+/ +""" + +latex_parser = Lark(LATEX_GRAMMAR) + +#=============================================================================== + +def get_text(tokens: list) -> str: +#================================= + text = [] + for token in tokens: + if isinstance(token, lark.Token): + if token.type == 'TEXT': + text.append(token.value.replace('+', '').replace('-', '')) + elif token.type in ['NUMBER', 'SYMBOL']: + text.append(token.value) + elif isinstance(token, lark.Tree): + text.append(get_text(token.children)) + return ''.join(text) + +def latex_to_symbol(name: str) -> str: +#==================================== + try: + children = latex_parser.parse(name).children + except UnexpectedInput: + raise ValueError(f'Cannot parse LaTeX name of shape: {name}') + base = None + sub_text = None + super_text = None + for n, child in enumerate(children): + if n == 0: + if isinstance(child, lark.Token) and child.type == 'TEXT': + base = child.value + elif isinstance(child, lark.Tree): + if child.data == 'super': + super_text = get_text(child.children) + elif child.data == 'sub': + sub_text = get_text(child.children) + + if base is None: + return name + else: + text = [base] + if super_text is not None: + text.append(super_text) + if sub_text is not None: + text.append(sub_text) + return '_'.join(text) + +def name_to_symbol(name: str) -> str: +#==================================== + if name.startswith('$') and name.endswith('$'): + return ', '.join([latex_to_symbol(latex) for latex in name[1:-1].split(',')]) + return name + +#=============================================================================== + +DCT_NS = rdflib.Namespace('http://purl.org/dc/terms/') +CDT_NS = rdflib.Namespace('https://w3id.org/cdt/') +RDF_NS = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +RDFS_NS = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') + +#=============================================================================== + +BG_NS = rdflib.Namespace('http://celldl.org/ontologies/bondgraph#') +BGF_NS = rdflib.Namespace('http://celldl.org/ontologies/bondgraph-framework#') +MODEL_NS = rdflib.Namespace('#') + +#=============================================================================== + +NAMESPACES = { + 'bg': BG_NS, + 'bgf': BGF_NS, + 'cdt': CDT_NS, + 'dct': DCT_NS, + 'rdfs': RDFS_NS, + '': MODEL_NS +} + +#=============================================================================== + +class BondgraphModel: + def __init__(self, shapes: list[Shape]): + self.__graph = rdflib.Graph() + + ## This could be embedded into a CellDL diagram, separate to its + ## CellDL structure. + + self.__uri = MODEL_NS[''] + for (prefix, ns) in NAMESPACES.items(): + self.__graph.bind(prefix, str(ns)) + self.__graph.add((self.__uri, RDF_NS.type, BG_NS.BondGraph)) + self.__graph.add((self.__uri, BGF_NS.hasSchema, rdflib.Literal(BG_FRAMEWORK_VERSION))) + self.__graph.add((self.__uri, DCT_NS.created, rdflib.Literal(datetime.now(UTC).isoformat()))) + self.__process_shape_list(shapes) + + def __add_shape(self, shape: Shape): + #=================================== + pass + + def as_turtle(self) -> bytes: + #============================ + return self.__graph.serialize(format='turtle', encoding='utf-8') + + def as_xml(self) -> bytes: + #========================= + return self.__graph.serialize(format='xml', encoding='utf-8') + + def set_property(self, property: rdflib.URIRef, value: rdflib.Literal|rdflib.URIRef): + #==================================================================================== + self.__graph.add((self.__uri, property, value)) + + def __process_shape_list(self, shapes: list[Shape]): + #=================================================== + nodes: dict[str, tuple[str, str]] = {} + bonds: dict[str, tuple[str, str]] = {} + + for shape in shapes: + if not shape.properties.get('exclude', False): + uri = svg_id(shape.id) + if shape.shape_type == SHAPE_TYPE.COMPONENT: + if shape.has_property('stroke'): + stroke = shape.get_property('stroke') + component_type = str(NODE_BORDER_TYPES.lookup(stroke, 'unknown')) + elif shape.name.startswith('$q^'): + component_type = 'bgf:StorageNode' + elif shape.name.startswith('$u^'): + component_type = 'bgf:ZeroNode' + elif shape.name.startswith('$v^'): + component_type = 'bgf:OneNode' + else: + component_type = 'unknown' + if component_type != 'unknown': + nodes[shape.id] = (component_type, name_to_symbol(shape.name)) + + elif shape.shape_type == SHAPE_TYPE.CONNECTION: + bonds[shape.id] = (shape.source, shape.target) + + elif shape.shape_type == SHAPE_TYPE.ANNOTATION: + pass + + for shape_id, bond in bonds.items(): + if bond[0] not in nodes or bond[1] not in nodes: + raise ValueError(f'Bad bondgraph connection ({shape_id}) -- source ({bond[0]}) and/or target ({bond[1]}) missing') + +#=============================================================================== diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index 8d5ce05b..b01f1fcb 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -42,6 +42,7 @@ from mapmaker.flatmap import Feature, FlatMap, SourceManifest, SOURCE_DETAIL_KINDS from mapmaker.flatmap.layers import FEATURES_TILE_LAYER, MapLayer from mapmaker.geometry import Transform +from mapmaker.output.bondgraph import BondgraphModel from mapmaker.settings import MAP_KIND, settings from mapmaker.shapes import Shape, SHAPE_TYPE from mapmaker.shapes.classify import ShapeClassifier @@ -235,11 +236,11 @@ def __process_shapes(self, shapes: TreeList[Shape]) -> list[Feature]: shape_classifier = ShapeClassifier(shapes.flatten(), self.source.map_area(), self.source.metres_per_pixel) shapes = TreeList(shape_classifier.shapes) if settings.get('exportBondgraphs', False): - celldl_file = pathlib_path(self.source.href).with_suffix('.celldl.svg') - log.info(f'Exporting layer `{self.id}` to `{str(celldl_file)}`...') - celldl_export = CellDLExporter(self.__svg, self.source.href, self.source.transform.inverse()) - celldl_export.process(shapes) - celldl_export.save(celldl_file) + bondgraph_file = pathlib_path(self.source.href).with_suffix('.bondgraph.ttl') + log.info(f'Exporting layer `{self.id}` to `{str(bondgraph_file)}`...') + bondgraph = BondgraphModel(shapes) + with open(bondgraph_file, 'wb') as fp: + fp.write(bondgraph.as_turtle()) # Add a background shape behind a detailed functional map if (self.flatmap.map_kind == MAP_KIND.FUNCTIONAL and self.source.kind == 'functional'): From 96b76a2e3dec3e221cdff63d3b7f5ac90668761a Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 12:13:08 +1200 Subject: [PATCH 078/111] celldl: define `ANNOTATION` and `PORT` shape types and CellDL classes. --- mapmaker/knowledgebase/celldl.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mapmaker/knowledgebase/celldl.py b/mapmaker/knowledgebase/celldl.py index e55b66d1..ca4bbaeb 100644 --- a/mapmaker/knowledgebase/celldl.py +++ b/mapmaker/knowledgebase/celldl.py @@ -1,6 +1,6 @@ #=============================================================================== # -# Flatmap viewer and annotation tools +# Flatmap maker and annotation tools # # Copyright (c) 2019 - 2023 David Brooks # @@ -42,12 +42,16 @@ class CD_CLASS: UNKNOWN = 'celldl:Unknown' LAYER = 'celldl:Layer' + ANNOTATION = 'celldl:Annotation' + COMPONENT = 'celldl:Component' # What has CONNECTORs CONNECTOR = 'celldl:Connector' # What a CONNECTION connects to CONNECTION = 'celldl:Connection' # The path between CONNECTORS CONDUIT = 'celldl:Conduit' # A container for CONNECTIONs + PORT = 'celldl:UnconnectedPort' + MEMBRANE = 'celldl:Membrane' # A boundary around a collection of COMPONENTS ANNOTATION = 'celldl:Annotation' # Additional information about something @@ -132,10 +136,12 @@ class FC_KIND: #=============================================================================== CELLDL_TYPE_FROM_CLASS = { + CD_CLASS.ANNOTATION: CELLDL_NS.Annotation, CD_CLASS.CONDUIT: CELLDL_NS.Conduit, CD_CLASS.CONNECTION: CELLDL_NS.Connection, CD_CLASS.CONNECTOR: CELLDL_NS.Connector, CD_CLASS.COMPONENT: CELLDL_NS.Component, + CD_CLASS.PORT: CELLDL_NS.UnconnectedPort, } #=============================================================================== @@ -145,9 +151,11 @@ class FC_KIND: #=============================================================================== CELLDL_CLASS_FROM_SHAPE_TYPE = { + SHAPE_TYPE.ANNOTATION: CD_CLASS.ANNOTATION, SHAPE_TYPE.COMPONENT: CD_CLASS.COMPONENT, SHAPE_TYPE.CONNECTION: CD_CLASS.CONNECTION, - SHAPE_TYPE.CONTAINER: CD_CLASS.COMPONENT + SHAPE_TYPE.CONTAINER: CD_CLASS.COMPONENT, + SHAPE_TYPE.PORT: CD_CLASS.PORT, } #=============================================================================== From d53b3af30cd8d95a5dbbfcbb28e279b12bcb52a7 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 12:45:54 +1200 Subject: [PATCH 079/111] svg: the default `id` of a shape is its ID attribute, which then might be overridden by `.id()` markup. --- mapmaker/properties/__init__.py | 3 ++- mapmaker/sources/svg/__init__.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mapmaker/properties/__init__.py b/mapmaker/properties/__init__.py index 42669900..e88c9a5d 100644 --- a/mapmaker/properties/__init__.py +++ b/mapmaker/properties/__init__.py @@ -220,7 +220,8 @@ def update_properties(self, feature_properties): #=============================================== classes = feature_properties.get('class', '').split() id = feature_properties.get('id') - if self.__flatmap.map_kind == MAP_KIND.FUNCTIONAL: + if self.__flatmap.map_kind == MAP_KIND.FUNCTIONAL and id not in self.__properties_by_id: + # Use the feature's name to lookup properties when the feature has no ID if (name := feature_properties.get('name', '').replace(" ", "_")) != '': id = f'{feature_properties.get("layer", "")}/{name}' if id is not None: diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index b01f1fcb..e7636a71 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -405,6 +405,8 @@ def __process_element(self, wrapped_element: ElementWrapper, transform, parent_p properties = parent_properties.copy() for name in NON_INHERITED_PROPERTIES: properties.pop(name, None) + if 'id' in element.attrib: + properties['id'] = element.attrib.get('id') properties.update(properties_from_markup) shape_id = properties.get('id') ## versus element.attrib.get('id') if 'path' in properties: From 60f61f1a41462e44c8ac020510faba36ab209b0d Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 12:46:17 +1200 Subject: [PATCH 080/111] Minor tidying. --- mapmaker/properties/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapmaker/properties/__init__.py b/mapmaker/properties/__init__.py index e88c9a5d..f04f965c 100644 --- a/mapmaker/properties/__init__.py +++ b/mapmaker/properties/__init__.py @@ -168,7 +168,7 @@ def __set_feature_properties(self, features): #============================================ if isinstance(features, dict): for id, properties in features.items(): - id = id.replace(" ", "_") + id = id.replace(' ', '_') if (associated_details := properties.get('associated-details')) is not None: if isinstance(associated_details, str): properties['associated-details'] = [associated_details] @@ -222,8 +222,8 @@ def update_properties(self, feature_properties): id = feature_properties.get('id') if self.__flatmap.map_kind == MAP_KIND.FUNCTIONAL and id not in self.__properties_by_id: # Use the feature's name to lookup properties when the feature has no ID - if (name := feature_properties.get('name', '').replace(" ", "_")) != '': - id = f'{feature_properties.get("layer", "")}/{name}' + if (name := feature_properties.get('name', '').replace(' ', '_')) != '': + id = f'{feature_properties.get('layer', '')}/{name}' if id is not None: classes.extend(self.__properties_by_id.get(id, {}).get('class', '').split()) for cls in classes: From aa1a4ae8348bba3ab240e16a2538c6bd8075fe99 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 12:55:29 +1200 Subject: [PATCH 081/111] bondgraphs: use a separate source file to specify namespaces. --- mapmaker/output/bondgraph/__init__.py | 26 ++------------ mapmaker/output/bondgraph/namespaces.py | 47 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 mapmaker/output/bondgraph/namespaces.py diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index b67d8b5b..c498f0f1 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -32,6 +32,9 @@ from mapmaker.shapes.colours import ColourMatcherDict from mapmaker.utils.svg import svg_id +from .namespaces import NAMESPACES +from .namespaces import BG, BGF, DCT, RDF, MODEL + #=============================================================================== BG_FRAMEWORK_VERSION = "1.0" @@ -120,29 +123,6 @@ def name_to_symbol(name: str) -> str: return name #=============================================================================== - -DCT_NS = rdflib.Namespace('http://purl.org/dc/terms/') -CDT_NS = rdflib.Namespace('https://w3id.org/cdt/') -RDF_NS = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') -RDFS_NS = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') - -#=============================================================================== - -BG_NS = rdflib.Namespace('http://celldl.org/ontologies/bondgraph#') -BGF_NS = rdflib.Namespace('http://celldl.org/ontologies/bondgraph-framework#') -MODEL_NS = rdflib.Namespace('#') - -#=============================================================================== - -NAMESPACES = { - 'bg': BG_NS, - 'bgf': BGF_NS, - 'cdt': CDT_NS, - 'dct': DCT_NS, - 'rdfs': RDFS_NS, - '': MODEL_NS -} - #=============================================================================== class BondgraphModel: diff --git a/mapmaker/output/bondgraph/namespaces.py b/mapmaker/output/bondgraph/namespaces.py new file mode 100644 index 00000000..a26198c3 --- /dev/null +++ b/mapmaker/output/bondgraph/namespaces.py @@ -0,0 +1,47 @@ +#=============================================================================== +# +# Flatmap maker and annotation tools +# +# Copyright (c) 2019 - 2025 David Brooks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#=============================================================================== + +import rdflib + +#=============================================================================== + +DCT = rdflib.Namespace('http://purl.org/dc/terms/') +CDT = rdflib.Namespace('https://w3id.org/cdt/') +RDF = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') +RDFS = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') + +#=============================================================================== + +BG = rdflib.Namespace('http://celldl.org/ontologies/bondgraph#') +BGF = rdflib.Namespace('http://celldl.org/ontologies/bondgraph-framework#') +MODEL = rdflib.Namespace('#') + +#=============================================================================== + +NAMESPACES = { + 'bg': BG, + 'bgf': BGF, + 'cdt': CDT, + 'dct': DCT, + 'rdfs': RDFS, + '': MODEL +} + +#=============================================================================== From 18550031da9ecc8dc20eb044ea9e331f33c8d843 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 12:58:04 +1200 Subject: [PATCH 082/111] bondgraphs: change format of names to `BASE_SUBSCRIPT_SUPERSCRIPT`. --- mapmaker/output/bondgraph/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index c498f0f1..911041aa 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -110,10 +110,10 @@ def latex_to_symbol(name: str) -> str: return name else: text = [base] - if super_text is not None: - text.append(super_text) if sub_text is not None: text.append(sub_text) + if super_text is not None: + text.append(super_text) return '_'.join(text) def name_to_symbol(name: str) -> str: From c643a339e41029225523e9af71bfa7fa5ff74065 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 13:31:22 +1200 Subject: [PATCH 083/111] bondgraphs: use the map's `id` as the id of the generated bondgraph model. --- mapmaker/output/bondgraph/__init__.py | 4 ++-- mapmaker/sources/svg/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 911041aa..f5e29068 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -126,13 +126,13 @@ def name_to_symbol(name: str) -> str: #=============================================================================== class BondgraphModel: - def __init__(self, shapes: list[Shape]): + def __init__(self, id: str, shapes: TreeList[Shape]): self.__graph = rdflib.Graph() ## This could be embedded into a CellDL diagram, separate to its ## CellDL structure. - self.__uri = MODEL_NS[''] + self.__uri = MODEL[id] for (prefix, ns) in NAMESPACES.items(): self.__graph.bind(prefix, str(ns)) self.__graph.add((self.__uri, RDF_NS.type, BG_NS.BondGraph)) diff --git a/mapmaker/sources/svg/__init__.py b/mapmaker/sources/svg/__init__.py index e7636a71..4e43ebf8 100644 --- a/mapmaker/sources/svg/__init__.py +++ b/mapmaker/sources/svg/__init__.py @@ -238,7 +238,7 @@ def __process_shapes(self, shapes: TreeList[Shape]) -> list[Feature]: if settings.get('exportBondgraphs', False): bondgraph_file = pathlib_path(self.source.href).with_suffix('.bondgraph.ttl') log.info(f'Exporting layer `{self.id}` to `{str(bondgraph_file)}`...') - bondgraph = BondgraphModel(shapes) + bondgraph = BondgraphModel(self.id, shapes) with open(bondgraph_file, 'wb') as fp: fp.write(bondgraph.as_turtle()) # Add a background shape behind a detailed functional map From 94e1964a97655c385fa7b9c75f52b00d4694a245 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:24:39 +1200 Subject: [PATCH 084/111] bondgraphs: return Unicode bytes when serialising RDF. --- mapmaker/output/bondgraph/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index f5e29068..3eff52e3 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -146,11 +146,12 @@ def __add_shape(self, shape: Shape): def as_turtle(self) -> bytes: #============================ - return self.__graph.serialize(format='turtle', encoding='utf-8') + ttl = self.__graph.serialize(format='turtle', encoding='unicode') + return ttl def as_xml(self) -> bytes: #========================= - return self.__graph.serialize(format='xml', encoding='utf-8') + return self.__graph.serialize(format='xml', encoding='unicode') def set_property(self, property: rdflib.URIRef, value: rdflib.Literal|rdflib.URIRef): #==================================================================================== From 5d2e055b0dc987ea3f41beba5c8f19f82d075cc7 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:26:09 +1200 Subject: [PATCH 085/111] bondgraphs: remove unused code. --- mapmaker/output/bondgraph/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 3eff52e3..03e7ffbb 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -140,10 +140,6 @@ def __init__(self, id: str, shapes: TreeList[Shape]): self.__graph.add((self.__uri, DCT_NS.created, rdflib.Literal(datetime.now(UTC).isoformat()))) self.__process_shape_list(shapes) - def __add_shape(self, shape: Shape): - #=================================== - pass - def as_turtle(self) -> bytes: #============================ ttl = self.__graph.serialize(format='turtle', encoding='unicode') From 45cd440157ee907caa55ef659b6e3557b27a9cd0 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:36:40 +1200 Subject: [PATCH 086/111] bondgraphs: improve checks and construction of RDF graph. --- mapmaker/output/bondgraph/__init__.py | 39 +++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 03e7ffbb..1765e35f 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -30,6 +30,7 @@ from mapmaker.shapes import Shape, SHAPE_TYPE from mapmaker.shapes.colours import ColourMatcherDict +from mapmaker.utils import log, TreeList from mapmaker.utils.svg import svg_id from .namespaces import NAMESPACES @@ -135,10 +136,10 @@ def __init__(self, id: str, shapes: TreeList[Shape]): self.__uri = MODEL[id] for (prefix, ns) in NAMESPACES.items(): self.__graph.bind(prefix, str(ns)) - self.__graph.add((self.__uri, RDF_NS.type, BG_NS.BondGraph)) - self.__graph.add((self.__uri, BGF_NS.hasSchema, rdflib.Literal(BG_FRAMEWORK_VERSION))) - self.__graph.add((self.__uri, DCT_NS.created, rdflib.Literal(datetime.now(UTC).isoformat()))) - self.__process_shape_list(shapes) + self.__graph.add((self.__uri, RDF.type, BG.BondGraph)) + self.__graph.add((self.__uri, BGF.hasSchema, rdflib.Literal(BG_FRAMEWORK_VERSION))) + self.__graph.add((self.__uri, DCT.created, rdflib.Literal(datetime.now(UTC).isoformat()))) + self.__process_shape_list(shapes.flatten()) def as_turtle(self) -> bytes: #============================ @@ -156,11 +157,9 @@ def set_property(self, property: rdflib.URIRef, value: rdflib.Literal|rdflib.URI def __process_shape_list(self, shapes: list[Shape]): #=================================================== nodes: dict[str, tuple[str, str]] = {} - bonds: dict[str, tuple[str, str]] = {} - + connections: dict[str, tuple[str, str]] = {} for shape in shapes: if not shape.properties.get('exclude', False): - uri = svg_id(shape.id) if shape.shape_type == SHAPE_TYPE.COMPONENT: if shape.has_property('stroke'): stroke = shape.get_property('stroke') @@ -173,17 +172,29 @@ def __process_shape_list(self, shapes: list[Shape]): component_type = 'bgf:OneNode' else: component_type = 'unknown' + # if component_type != 'unknown': nodes[shape.id] = (component_type, name_to_symbol(shape.name)) - elif shape.shape_type == SHAPE_TYPE.CONNECTION: - bonds[shape.id] = (shape.source, shape.target) - + connections[shape.id] = (shape.source, shape.target) elif shape.shape_type == SHAPE_TYPE.ANNOTATION: pass - - for shape_id, bond in bonds.items(): - if bond[0] not in nodes or bond[1] not in nodes: - raise ValueError(f'Bad bondgraph connection ({shape_id}) -- source ({bond[0]}) and/or target ({bond[1]}) missing') + for type, name in nodes.values(): + if type.startswith('bgf:'): + self.__graph.add((MODEL[name], RDF.type, BGF[type[4:]])) + missing_node = MODEL.MISSING + for shape_id, connection in connections.items(): + if connection[0] in nodes: + source = MODEL[nodes[connection[0]][1]] + else: + log.warning(f'Missing source node shape `{connection[0]}/` for connection') + source = missing_node + if connection[1] in nodes: + target = MODEL[nodes[connection[1]][1]] + else: + log.warning(f'Missing target node shape `{connection[1]}` for connection') + target = missing_node + self.__graph.add((MODEL[svg_id(shape_id)], BGF.hasSource, source)) + self.__graph.add((MODEL[svg_id(shape_id)], BGF.hasSource, target)) #=============================================================================== From e1fb58998b64e0994dbc970f75a383a8151f0e04 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:39:48 +1200 Subject: [PATCH 087/111] bondgraphs: an element's name could in fact be a comma separated list of names. --- mapmaker/output/bondgraph/__init__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 1765e35f..36406941 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -117,11 +117,11 @@ def latex_to_symbol(name: str) -> str: text.append(super_text) return '_'.join(text) -def name_to_symbol(name: str) -> str: -#==================================== +def name_to_symbols(name: str) -> tuple[str]: +#============================================ if name.startswith('$') and name.endswith('$'): - return ', '.join([latex_to_symbol(latex) for latex in name[1:-1].split(',')]) - return name + (latex_to_symbol(latex) for latex in name[1:-1].split(',')) + return (name,) #=============================================================================== #=============================================================================== @@ -156,7 +156,7 @@ def set_property(self, property: rdflib.URIRef, value: rdflib.Literal|rdflib.URI def __process_shape_list(self, shapes: list[Shape]): #=================================================== - nodes: dict[str, tuple[str, str]] = {} + nodes: dict[str, tuple[str, tuple[str]]] = {} connections: dict[str, tuple[str, str]] = {} for shape in shapes: if not shape.properties.get('exclude', False): @@ -174,23 +174,26 @@ def __process_shape_list(self, shapes: list[Shape]): component_type = 'unknown' # if component_type != 'unknown': - nodes[shape.id] = (component_type, name_to_symbol(shape.name)) + names = name_to_symbols(shape.name) + if component_type in ['bgf:OneNode', 'bgf:ZeroNode']: + self.__graph.add((MODEL[names[0]], RDF.type, BGF[component_type[4:]])) + nodes[shape.id] = (component_type, name_to_symbols(shape.name)) elif shape.shape_type == SHAPE_TYPE.CONNECTION: connections[shape.id] = (shape.source, shape.target) elif shape.shape_type == SHAPE_TYPE.ANNOTATION: pass - for type, name in nodes.values(): + for type, names in nodes.values(): if type.startswith('bgf:'): - self.__graph.add((MODEL[name], RDF.type, BGF[type[4:]])) + self.__graph.add((MODEL[names[0]], RDF.type, BGF[type[4:]])) missing_node = MODEL.MISSING for shape_id, connection in connections.items(): if connection[0] in nodes: - source = MODEL[nodes[connection[0]][1]] + source = MODEL[nodes[connection[0]][1][0]] else: log.warning(f'Missing source node shape `{connection[0]}/` for connection') source = missing_node if connection[1] in nodes: - target = MODEL[nodes[connection[1]][1]] + target = MODEL[nodes[connection[1]][1][0]] else: log.warning(f'Missing target node shape `{connection[1]}` for connection') target = missing_node From 8216aaf4d75cd2f00d4e7fc367ec2656cc5a7f4f Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:40:33 +1200 Subject: [PATCH 088/111] bondgraphs: reaction nodes have a dark blue border. --- mapmaker/output/bondgraph/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 36406941..294315fd 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -45,9 +45,11 @@ NODE_BORDER_TYPES = ColourMatcherDict({ '#042433': 'bgf:OneNode', # dark green '#00B050': 'bgf:OneResistance', # green + '#061B27': 'bgf:OneReaction', # navy blue '#FF0000': 'bgf:ZeroStorage', # red }) +#=============================================================================== #=============================================================================== LATEX_GRAMMAR = """ From a00630fa19f108725ad991eaecdaeb93b0c02b21 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:41:54 +1200 Subject: [PATCH 089/111] bondgraphs: `q` nodes are `bgf:ZeroStorage` nodes. --- mapmaker/output/bondgraph/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/output/bondgraph/__init__.py b/mapmaker/output/bondgraph/__init__.py index 294315fd..f3da9a06 100644 --- a/mapmaker/output/bondgraph/__init__.py +++ b/mapmaker/output/bondgraph/__init__.py @@ -167,7 +167,7 @@ def __process_shape_list(self, shapes: list[Shape]): stroke = shape.get_property('stroke') component_type = str(NODE_BORDER_TYPES.lookup(stroke, 'unknown')) elif shape.name.startswith('$q^'): - component_type = 'bgf:StorageNode' + component_type = 'bgf:ZeroStorage' elif shape.name.startswith('$u^'): component_type = 'bgf:ZeroNode' elif shape.name.startswith('$v^'): From f57d4ff2baf7850ab3dfd5d373e7fb7fc2bb1e40 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Fri, 15 Aug 2025 14:45:36 +1200 Subject: [PATCH 090/111] celldl: use `\ce{}` MathJax markup for known chemical symbols. --- mapmaker/shapes/text_finder.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 56716a4d..dfb4f190 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -61,6 +61,19 @@ def left_sort_shapes(self): #=============================================================================== +CHEMICAL_SYMBOLS = { + 'Ca^{2+}': 'Ca2+', + 'CO_{2}': 'CO2', + 'Glc': 'Glc', + 'H_{2}O': 'H2O', + 'K^{+}': 'K+', + 'Na': 'Na', + 'Na^{+}': 'Na+', + 'O_{2}': 'O2', +} + +#=============================================================================== + class LatexMaker: def __init__(self): self.__latex = [] @@ -72,7 +85,10 @@ def __init__(self): def latex(self) -> str: #====================== self.__make_latex() - return ''.join(self.__latex) + latex = ''.join(self.__latex) + if (chem := CHEMICAL_SYMBOLS.get(latex)) is not None: + latex = f'\\ce{{{chem}}}' + return latex def add_text(self, text: str, state: int): #========================================= From e5495d4d451206bd032d463c653c0408f859b555 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:08:45 +1200 Subject: [PATCH 091/111] celldl: take the size of the containing shape when locating text within the shape. --- mapmaker/shapes/constants.py | 2 ++ mapmaker/shapes/text_finder.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mapmaker/shapes/constants.py b/mapmaker/shapes/constants.py index e7a01685..ca40f973 100644 --- a/mapmaker/shapes/constants.py +++ b/mapmaker/shapes/constants.py @@ -45,6 +45,8 @@ MAX_TEXT_VERTICAL_OFFSET = 3 # Between cluster baseline and baselines of text in the cluster TEXT_BASELINE_OFFSET = -14.5 # From vertical centre of a component +TEXT_COMPONENT_HEIGHT = 75000 # World metres height of an "average" component + # Text shapes need at least 80% containment in their parent MIN_TEXT_INSIDE = 0.8 diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index dfb4f190..806ad81c 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -24,7 +24,7 @@ #=============================================================================== from . import Shape, SHAPE_TYPE -from .constants import MAX_TEXT_VERTICAL_OFFSET, TEXT_BASELINE_OFFSET +from .constants import MAX_TEXT_VERTICAL_OFFSET, TEXT_BASELINE_OFFSET, TEXT_COMPONENT_HEIGHT #=============================================================================== @@ -122,13 +122,15 @@ def __init__(self, scaling: float): self.__sub_superscript_re = re.compile(f'{SUBSCRIPT_CHAR}|\\{SUPERSCRIPT_CHAR}') self.__max_text_vertical_offset = scaling * MAX_TEXT_VERTICAL_OFFSET self.__text_baseline_offset = scaling * TEXT_BASELINE_OFFSET + self.__shape_scaling = 1.0 def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: #===================================================================== + self.__shape_scaling = shape.height/TEXT_COMPONENT_HEIGHT text_shapes = [s for s in shape.children if s.shape_type == SHAPE_TYPE.TEXT] text_clusters = self.__cluster_text(text_shapes) - offset = self.__max_text_vertical_offset baseline = (shape.geometry.bounds[1] + shape.geometry.bounds[3])/2 + self.__text_baseline_offset + offset = self.__shape_scaling*self.__max_text_vertical_offset state = 0 clusters = [] latex = LatexMaker() @@ -169,7 +171,7 @@ def __text_block_to_text(self, text_block: list[Shape]) -> str: def __text_clusters_to_text(self, text_clusters: list[TextShapeCluster]) -> str: #=============================================================================== baseline = text_clusters[0].baseline - offset = 0.9*self.__max_text_vertical_offset + offset = 0.9*self.__shape_scaling*self.__max_text_vertical_offset latex = LatexMaker() for cluster in text_clusters: if cluster.baseline < (baseline - offset): @@ -182,7 +184,7 @@ def __text_clusters_to_text(self, text_clusters: list[TextShapeCluster]) -> str: def __cluster_text(self, text_shapes: list[Shape]) -> list[TextShapeCluster]: #============================================================================ - offset = self.__max_text_vertical_offset + offset = self.__shape_scaling*self.__max_text_vertical_offset shapes_seen_order = sorted(text_shapes, key=lambda s: s.number) clusters: list[TextShapeCluster] = [] current_cluster = None From 4dd808e349ec3fbf96a760fc28c84054c4549de8 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:10:19 +1200 Subject: [PATCH 092/111] celldl: allow for a default value when finding items by colour using a ColourMatcherDict. --- mapmaker/shapes/colours.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/colours.py b/mapmaker/shapes/colours.py index b1ae9c93..b4e0ac3d 100644 --- a/mapmaker/shapes/colours.py +++ b/mapmaker/shapes/colours.py @@ -67,11 +67,12 @@ def __init__(self, lookup_table: dict[str, Any]): for key, value in lookup_table.items() } - def lookup(self, colour: Optional[str]) -> Optional[Any]: + def lookup(self, colour: Optional[str], default: Optional[Any]=None) -> Optional[Any]: if colour is not None and colour != 'none': lab_colour = convert_color(sRGBColor.new_from_rgb_hex(colour), LabColor) for key, value in self.__lookup_table.items(): if delta_e_cie2000(lab_colour, key) < CLOSE_COLOUR_DISTANCE: return value + return default #=============================================================================== From 88d8b6f8d51c17da9018dce71b4253b059434ed2 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:25:59 +1200 Subject: [PATCH 093/111] shapes: expose `width` and `height` of a shape. --- mapmaker/shapes/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index eac075e9..d11d1bb3 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -88,6 +88,7 @@ def __init__(self, id: Optional[str], geometry: BaseGeometry, properties=None, * self.__geometry = geometry if geometry is not None: self.set_property('geometry', geometry.geom_type) + self.__bounds = geometry.bounds self.__children: list[Shape] = [] self.__parents: list[Shape] = [] self.__metadata: dict[str, str] = {} # kw_only=True field for Python 3.10 @@ -153,6 +154,10 @@ def geojson_id(self) -> int: def global_shape(self) -> 'Shape': # The shape that excluded this one via a filter return self.get_property('global-shape', self) + @property + def height(self) -> float: + return abs(self.__bounds[3] - self.__bounds[1]) + @property def id(self) -> Optional[str]: return self.__id @@ -193,6 +198,10 @@ def shape_name(self) -> str: # The name of the shape in the s def shape_type(self) -> SHAPE_TYPE: return self.get_property('shape-type', SHAPE_TYPE.UNKNOWN) + @property + def width(self) -> float: + return abs(self.__bounds[2] - self.__bounds[0]) + def add_parent(self, parent): self.parents.append(parent) parent.children.append(self) From 297f6af719fd1d15d086817029c26e31666663f6 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:26:15 +1200 Subject: [PATCH 094/111] Remove unused code. --- mapmaker/shapes/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index d11d1bb3..ff79d402 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -25,7 +25,6 @@ #=============================================================================== -from mapmaker.settings import settings from mapmaker.utils import log, PropertyMixin #=============================================================================== @@ -51,7 +50,6 @@ class SHAPE_TYPE(str, Enum): ## Or IntEnum ?? 'coverage', 'fill', 'geometry', - 'stroke', 'stroke-width', 'svg-element', 'tile-layer', From f9343a951950049153b18b35021f881c1a312361 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:28:33 +1200 Subject: [PATCH 095/111] shapes: `shape.id` will in fact be set... --- mapmaker/shapes/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index ff79d402..9d2350c1 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -70,7 +70,7 @@ def __init__(self, id: Optional[str], geometry: BaseGeometry, properties=None, * Shape.__last_shape_number += 1 self.__number: int = Shape.__last_shape_number if self.has_property('id'): - id = self.get_property('id') + id = self.get_property('id', '') if Shape.__shape_id_prefix == '': self.__id = id else: @@ -157,8 +157,8 @@ def height(self) -> float: return abs(self.__bounds[3] - self.__bounds[1]) @property - def id(self) -> Optional[str]: - return self.__id + def id(self) -> str: + return self.__id # pyright: ignore[reportReturnType] @property def kind(self) -> Optional[str]: # The geometric name of the shape or, for an image, From 1e69fa75fb347986a28501fc08802d02ff13e800 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 18:30:14 +1200 Subject: [PATCH 096/111] shapes: don't try to make a connection if we can only find end. --- mapmaker/shapes/classify.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 5d92bfc3..026f4cf4 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -258,7 +258,9 @@ def __join_connections(self, connection_joiners): joined_connection_graph = nx.Graph() for joiner in connection_joiners: ends = connection_index.query_nearest(joiner.geometry) - if len(ends) == 2: + if len(ends) == 1: + continue + elif len(ends) == 2: joiner.properties['exclude'] = True (connection_0, connection_1) = self.__extend_joined_connections(ends) joined_connection_graph.add_edge(connection_0, connection_1) From 8d28093a3af53e94a8561fbd5d180bde79522e01 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:02:38 +1200 Subject: [PATCH 097/111] shapes: distinguish COMPONENTS as PORTS as shapes that a connection might connect to. --- mapmaker/shapes/__init__.py | 1 + mapmaker/shapes/classify.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mapmaker/shapes/__init__.py b/mapmaker/shapes/__init__.py index 9d2350c1..aa1e4e9d 100644 --- a/mapmaker/shapes/__init__.py +++ b/mapmaker/shapes/__init__.py @@ -37,6 +37,7 @@ class SHAPE_TYPE(str, Enum): ## Or IntEnum ?? CONTAINER = 'container' GROUP = 'group' IMAGE = 'image' + PORT = 'port' TEXT = 'text' UNKNOWN = 'unknown' diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 026f4cf4..f7dfd3e9 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -137,14 +137,15 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float elif 'LineString' in geometry.geom_type or coverage < 0.4 and 'Multi' not in geometry.geom_type: if not self.__add_connection(shape): log.warning('Cannot extract line from polygon', shape=shape.id) + elif 'Multi' not in geometry.boundary.geom_type and len(shape.geometry.boundary.coords) == 4: # A triangle + connection_joiners.append(shape) + shape.properties['shape-type'] = SHAPE_TYPE.PORT elif bbox_coverage > 0.001 and coverage > 0.9: shape.properties['shape-type'] = SHAPE_TYPE.CONTAINER if bbox_coverage > 0.2 else SHAPE_TYPE.COMPONENT elif bbox_coverage < 0.0003 and 0.7 < coverage <= 0.8: shape.properties['shape-type'] = SHAPE_TYPE.ANNOTATION elif bbox_coverage < 0.001 and coverage > 0.75: shape.properties['shape-type'] = SHAPE_TYPE.COMPONENT - elif 'Multi' not in geometry.boundary.geom_type and len(shape.geometry.boundary.coords) == 4: # A triangle - connection_joiners.append(shape) elif not self.__add_connection(shape): log.warning('Unclassifiable shape', shape=shape.id) if settings.get('authoring', False): @@ -155,9 +156,12 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float self.__shapes_by_type[shape.shape_type].append(shape) if shape.shape_type in [SHAPE_TYPE.ANNOTATION, SHAPE_TYPE.COMPONENT, + SHAPE_TYPE.PORT, SHAPE_TYPE.TEXT]: self.__geometry_to_shape[id(shape.geometry)] = shape component_geometries.append(shape.geometry) + if shape.shape_type in [SHAPE_TYPE.COMPONENT, + SHAPE_TYPE.PORT]: shape.properties['stroke-width'] = COMPONENT_BORDER_WIDTH # An index for component geometries @@ -231,7 +235,8 @@ def __connect_line_end(self, shape: Shape, end: shapely.Point, property: str): for child in [self.__geometry_to_shape[id(self.__component_geometries[c])] for c in self.__component_index.query(end.buffer(self.__max_line_width), predicate='intersects') if self.__component_geometries[c].area > 0]: - if not child.exclude: + child.append_property('connections', shape.id) + if not child.exclude and child.shape_type in [SHAPE_TYPE.COMPONENT, SHAPE_TYPE.PORT]: shape.set_property(property, child.id) return From c2b8a640ae25f88b930f96bf47849b9d81dcac5b Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:03:27 +1200 Subject: [PATCH 098/111] Resolve linting errors. --- mapmaker/shapes/classify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index f7dfd3e9..bfc6b57b 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -88,9 +88,9 @@ def line_string(self): def end_line(self, end: int) -> Line: if end == 0: - return Line.from_coords((self.__coords[0], self.__coords[1])) + return Line.from_coords((self.__coords[0], self.__coords[1])) # pyright: ignore[reportArgumentType] else: - return Line.from_coords((self.__coords[-2], self.__coords[-1])) + return Line.from_coords((self.__coords[-2], self.__coords[-1])) # pyright: ignore[reportArgumentType] #=============================================================================== From 86242efc40a816030a8c2d3e3f3ea098f2c3f3a0 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:03:41 +1200 Subject: [PATCH 099/111] shapes: add missing import. --- mapmaker/shapes/classify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index bfc6b57b..1ca30f1b 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -38,7 +38,7 @@ from mapmaker.utils import log from .constants import COMPONENT_BORDER_WIDTH, CONNECTION_STROKE_WIDTH, MAX_LINE_WIDTH -from .constants import SHAPE_ERROR_COLOUR, SHAPE_ERROR_BORDER +from .constants import MIN_TEXT_INSIDE, SHAPE_ERROR_COLOUR, SHAPE_ERROR_BORDER from .line_finder import Line, LineFinder, XYPair from .text_finder import TextFinder From 3f0fb8e597ad99f92bed039ae4669e74df9acb6f Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:06:17 +1200 Subject: [PATCH 100/111] functional: we don't want annotation nor container shapes to be active features. --- mapmaker/shapes/classify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index 1ca30f1b..ffc00309 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -181,12 +181,12 @@ def __init__(self, shapes: list[Shape], map_area: float, metres_per_pixel: float shape.properties['name'] = name_and_shapes[0] shape.properties['text-shapes'] = name_and_shapes[1] # Although we do want their text, we don't want annotations to be active features - if shape.shape_type == SHAPE_TYPE.ANNOTATION: - shape.properties['exclude'] = True elif shape.shape_type == SHAPE_TYPE.CONNECTION: line_ends: shapely.geometry.base.GeometrySequence[shapely.MultiPoint] = shape.geometry.boundary.geoms # type: ignore self.__connect_line_end(shape, line_ends[0], 'source') self.__connect_line_end(shape, line_ends[1], 'target') + elif shape.shape_type == SHAPE_TYPE.CONTAINER: + shape.properties['exclude'] = True @property def shapes(self) -> list[Shape]: From 23ffc08f23d07edcefea7a9a76aae008e2c906ef Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:06:32 +1200 Subject: [PATCH 101/111] shapes: remove debugging. --- mapmaker/shapes/classify.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mapmaker/shapes/classify.py b/mapmaker/shapes/classify.py index ffc00309..8c8c832a 100644 --- a/mapmaker/shapes/classify.py +++ b/mapmaker/shapes/classify.py @@ -212,8 +212,6 @@ def __add_connection(self, shape: Shape) -> bool: shape.properties['colour'] = colour if (kind := VASCULAR_KINDS.lookup(colour)) is not None: shape.properties['kind'] = kind - else: - print(shape.id, 'COLOUR ?', colour) shape.properties['shape-type'] = SHAPE_TYPE.CONNECTION shape.properties['tile-layer'] = PATHWAYS_TILE_LAYER shape.properties['stroke-width'] = CONNECTION_STROKE_WIDTH From c6b83f95f94618954c2be712eef6c70d697f30d8 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:11:01 +1200 Subject: [PATCH 102/111] shapes: code tidying. --- mapmaker/shapes/text_finder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 806ad81c..0cd2a9af 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -119,7 +119,7 @@ def __make_latex(self): class TextFinder: def __init__(self, scaling: float): - self.__sub_superscript_re = re.compile(f'{SUBSCRIPT_CHAR}|\\{SUPERSCRIPT_CHAR}') + self.__latex_re = re.compile(f'{SUBSCRIPT_CHAR}|\\{SUPERSCRIPT_CHAR}|{{') self.__max_text_vertical_offset = scaling * MAX_TEXT_VERTICAL_OFFSET self.__text_baseline_offset = scaling * TEXT_BASELINE_OFFSET self.__shape_scaling = 1.0 @@ -161,7 +161,7 @@ def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: used_text_shapes.extend(cluster.shapes) if len(clusters): latex.add_text(self.__text_clusters_to_text(clusters), state) - text = f'${text}$' if self.__sub_superscript_re.search(text:=latex.latex) is not None else text + text = f'${text}$' if self.__latex_re.search(text:=latex.latex) else text.replace('\\ ', ' ') return (text, used_text_shapes) if text != '' else None def __text_block_to_text(self, text_block: list[Shape]) -> str: From 46836bcf6f13f7db4879f3cdd8245871527ad299 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 19:13:57 +1200 Subject: [PATCH 103/111] bondgraphs: we use `lark` to parse text. --- pyproject.toml | 1 + uv.lock | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d33abccb..65477713 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "structlog>=24.4.0", "rich>=13.9.4", "tippecanoe>=2.72.0", + "lark>=1.2.2", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index caeec067..bc1f1182 100644 --- a/uv.lock +++ b/uv.lock @@ -510,6 +510,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl", hash = "sha256:b9e329dc1ceaae7cf5d5b214684a0b100e0dad0be6d5bbabac281ec35ddeca65", size = 4388 }, ] +[[package]] +name = "lark" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036 }, +] + [[package]] name = "lxml" version = "5.4.0" @@ -578,6 +587,7 @@ dependencies = [ { name = "flatmapknowledge" }, { name = "gitpython" }, { name = "giturlparse" }, + { name = "lark" }, { name = "lxml" }, { name = "mapknowledge" }, { name = "mbutil" }, @@ -630,6 +640,7 @@ requires-dist = [ { name = "flatmapknowledge", url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.2/flatmapknowledge-2.5.2-py3-none-any.whl" }, { name = "gitpython", specifier = ">=3.1.41" }, { name = "giturlparse", specifier = ">=0.12.0" }, + { name = "lark", specifier = ">=1.2.2" }, { name = "lxml", specifier = ">=5.2.2" }, { name = "mapknowledge", url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl" }, { name = "mbutil", specifier = ">=0.3.0" }, From 868424c6c05f43e286cfdf74fd721898f03ae006 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 20:03:12 +1200 Subject: [PATCH 104/111] shapes: the baseline of a block of text is that of the leftmost character in the block. --- mapmaker/shapes/text_finder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mapmaker/shapes/text_finder.py b/mapmaker/shapes/text_finder.py index 0cd2a9af..2c73ccf2 100644 --- a/mapmaker/shapes/text_finder.py +++ b/mapmaker/shapes/text_finder.py @@ -37,7 +37,7 @@ def __init__(self, shape: Optional[Shape]=None): @property def baseline(self) -> float: - return self.__baselines/len(self.__shapes) if len(self.__shapes) else 0 + return self.__shapes[0].baseline @property def left(self) -> float: @@ -129,8 +129,10 @@ def get_text(self, shape: Shape) -> Optional[tuple[str, list[Shape]]]: self.__shape_scaling = shape.height/TEXT_COMPONENT_HEIGHT text_shapes = [s for s in shape.children if s.shape_type == SHAPE_TYPE.TEXT] text_clusters = self.__cluster_text(text_shapes) - baseline = (shape.geometry.bounds[1] + shape.geometry.bounds[3])/2 + self.__text_baseline_offset + if len(text_clusters) == 0: + return None offset = self.__shape_scaling*self.__max_text_vertical_offset + baseline = text_clusters[0].baseline state = 0 clusters = [] latex = LatexMaker() From 4a49e89a8dcad3bbf60f44d210299608fc45ba39 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 20:10:09 +1200 Subject: [PATCH 105/111] shapes: improve detect of parallel pairs of lines that might be part of a connection shape. --- mapmaker/shapes/line_finder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index a6ed43b2..56fb7d48 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -288,20 +288,24 @@ def get_line(self, shape: Shape) -> Optional[LineString]: boundary_lines = [Line.from_coords(coords) for coords in boundary_line_coord_pairs] unused_boundary_lines = set(boundary_lines) for (line0, line1) in itertools.combinations(boundary_lines, 2): + # Iterate over all pairs of line segments that make up the shape's boundary if line0.parallel(line1): + # Two line segments are parallel -- are they adjacent sides of a centreline? p0 = HorizontalLine.from_line(line0) p1 = p0.project(line1) # reject if centroid of overlapping region isn't inside the shape's polygon if ((pt := p0.mid_point(p1)) is not None - and shapely.contains_xy(shape.geometry, pt.x, pt.y)): - if ((w := p0.separation(p1)) <= self.__max_line_width - and p0.overlap(p1, True) > MIN_LINE_ASPECT_RATIO*w - and p0.overlap(p1, False)/p0.overlap(p1, True) >= LINE_OVERLAP_RATIO): + and shapely.contains_xy(shape.geometry, pt.x, pt.y) + and p0.separation(p1)) <= self.__max_line_width: + # Centroid of overlapping region is inside the shape's polygon + # and distance between lines is less than scaled MAX_LINE_WIDTH + if p0.overlap(p1, False)/p0.overlap(p1, True) >= LINE_OVERLAP_RATIO: mid_lines.append(p0.mid_line(p1)) used_lines.update([line0, line1]) unused_boundary_lines.remove(line0) unused_boundary_lines.remove(line1) elif (pt := line0.intersection(line1)) is not None: + # Non parallel line pair that intersect without extension ends_graph.add_edge(line0, line1, intersection=pt) # ``Use`` any boundary line parallel to a mid-line and within From add978a5de99e0c7fb5f2452ab8dd7e4775cb8f9 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 20:10:23 +1200 Subject: [PATCH 106/111] Remove debugging. --- mapmaker/shapes/line_finder.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index 56fb7d48..3ce72764 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -338,9 +338,6 @@ def get_line(self, shape: Shape) -> Optional[LineString]: if connecting_line: i0 = l0.intersection(connecting_line, True) i1 = l1.intersection(connecting_line, True) - if trace: - print(i0, l0.string, connecting_line.string) - print(i1, connecting_line.string, l1.string) if i0 is not None and i1 is not None: G.add_edge(l0, connecting_line, intersection=i0) G.add_edge(connecting_line, l1, intersection=i1) From 9c0a035388663ad3e8f58273e0f8f4133805ff35 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 20:10:36 +1200 Subject: [PATCH 107/111] Fix lint errors. --- mapmaker/shapes/line_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index 3ce72764..35e7a471 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -285,7 +285,7 @@ def get_line(self, shape: Shape) -> Optional[LineString]: shapely.prepare(shape.geometry) boundary_line_coord_pairs = zip(boundary_coords, boundary_coords[1:]) - boundary_lines = [Line.from_coords(coords) for coords in boundary_line_coord_pairs] + boundary_lines = [Line.from_coords(coords) for coords in boundary_line_coord_pairs] # pyright: ignore[reportArgumentType] unused_boundary_lines = set(boundary_lines) for (line0, line1) in itertools.combinations(boundary_lines, 2): # Iterate over all pairs of line segments that make up the shape's boundary From 70da77bfc85b0674eda002ed3798c8de33eb51d8 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sat, 16 Aug 2025 20:29:46 +1200 Subject: [PATCH 108/111] shapes: make sure two parallel lines actually overlap before calculating the overlap ratio. --- mapmaker/shapes/line_finder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapmaker/shapes/line_finder.py b/mapmaker/shapes/line_finder.py index 35e7a471..f4db0dc2 100644 --- a/mapmaker/shapes/line_finder.py +++ b/mapmaker/shapes/line_finder.py @@ -299,7 +299,8 @@ def get_line(self, shape: Shape) -> Optional[LineString]: and p0.separation(p1)) <= self.__max_line_width: # Centroid of overlapping region is inside the shape's polygon # and distance between lines is less than scaled MAX_LINE_WIDTH - if p0.overlap(p1, False)/p0.overlap(p1, True) >= LINE_OVERLAP_RATIO: + if (p0.overlap(p1, False) > 0 + and p0.overlap(p1, False)/p0.overlap(p1, True) >= LINE_OVERLAP_RATIO): mid_lines.append(p0.mid_line(p1)) used_lines.update([line0, line1]) unused_boundary_lines.remove(line0) From b11c3f817cb2fb981b931b7bdb15b0d65c65d658 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sun, 17 Aug 2025 17:48:32 +1200 Subject: [PATCH 109/111] logging: rework logging to support the use of a `QueueHandler` to pass log messages to a manager when we are run as a sub-process. --- mapmaker/maker.py | 45 ++++++++++++++------ mapmaker/utils/logging.py | 90 +++++++++++++++++++++++++-------------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 9cd4adee..5b55f702 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -21,10 +21,12 @@ import json import os import pathlib +import multiprocessing import multiprocessing.connection import shutil import subprocess import uuid +from typing import Any #=============================================================================== @@ -77,8 +79,8 @@ #=============================================================================== -class MapMaker(object): - def __init__(self, options): +class MapMaker: + def __init__(self, options: dict[str, Any]): # ``silent`` implies not ``verbose`` if options.get('silent', False): options['verbose'] = False @@ -86,6 +88,11 @@ def __init__(self, options): if options.get('sckanVersion') is not None and not options.get('ignoreGit', False): raise ValueError('`--ignore-git` must be set when `--sckan-version` is used') + # For when we try to make() an invalid configuration + process_log_queue = options.pop('logQueue', None) + self.__tell_manager = process_log_queue is not None + self.__valid_configuration = False + # Setup logging if (log_file := options.get('logFile')) is None: if (log_path := options.get('logPath')) is not None: @@ -93,10 +100,12 @@ def __init__(self, options): if options.get('silent', False) and log_file is None: raise ValueError('`--silent` option requires `--log LOG_FILE` to be given') - self.__file_log = configure_logging(log_file, + self.__file_log = configure_logging( + log_json_file=log_file, verbose=options.get('verbose', False), silent=options.get('silent', False), - debug=options.get('debug', False)) + debug=options.get('debug', False), + log_queue=process_log_queue) log.info('Mapmaker', version=__version__) # Default base output directory to ``./flatmaps``. @@ -214,11 +223,12 @@ def __init__(self, options): if os.path.exists(self.__map_dir): if os.path.exists(self.__maker_sentinel): self.__clean_up(remove_sentinel=False) - raise MakerException('Last making of map failed -- use `--force` to re-make') - log.info('Map already exists -- use `--force` to re-make', id=self.id, uuid=self.uuid, path=self.__map_dir) - self.__clean_up() - exit(0) + log.error('Last making of map failed -- use `--force` to re-make', id=self.id, uuid=self.uuid, path=self.__map_dir) + else: + log.info('Map already exists -- use `--force` to re-make', id=self.id, uuid=self.uuid, path=self.__map_dir) + return else: + self.__valid_configuration = True os.makedirs(self.__map_dir) # Create an empty sentinel @@ -266,6 +276,10 @@ def zoom(self): def make(self): #============== + if self.__tell_manager and not self.__valid_configuration: + log.critical('Mapmaker failed') + return + self.__begin_make() # Process flatmap's sources to create MapLayers @@ -298,6 +312,14 @@ def make(self): # Save the flatmap's metadata self.__save_metadata() + # We now have successfully generated the flatmap + generated_map = {'id': self.id, 'uuid': self.uuid, 'path': self.__map_dir} + if self.__flatmap.models is not None: + generated_map['models'] = self.__flatmap.models + log.info('Generated map', **generated_map) + if self.__tell_manager: + log.critical('Mapmaker succeeded', **generated_map) + # Write out details of FC neurons if option set if (export_file := settings.get('exportNeurons')) is not None: with open(export_file, 'w') as fp: @@ -318,12 +340,6 @@ def make(self): sparc_dataset.generate() sparc_dataset.save(sds_output) - # Show what the map is about - log_details = {'id': self.id, 'uuid': self.uuid, 'path': self.__map_dir} - if self.__flatmap.models is not None: - log_details['models'] = self.__flatmap.models - log.info('Generated map', **log_details) - # Tidy up self.__clean_up() @@ -351,6 +367,7 @@ def __clean_up(self, remove_sentinel=True): # Remove any temporary directory created for the map's sources self.__manifest.clean_up() + # Copy the log file into the generated map's directory if self.__file_log is not None: maker_log = os.path.join(self.__map_dir, MAKER_LOG) if not os.path.exists(maker_log): diff --git a/mapmaker/utils/logging.py b/mapmaker/utils/logging.py index f7846449..03821cf6 100644 --- a/mapmaker/utils/logging.py +++ b/mapmaker/utils/logging.py @@ -21,7 +21,8 @@ import json import logging import logging.config -import typing +import logging.handlers +import multiprocessing from typing import Any, Callable, Optional #=============================================================================== @@ -36,6 +37,14 @@ #=============================================================================== +class QueueHandlerJSON(logging.handlers.QueueHandler): + def prepare(self, record: logging.LogRecord) -> logging.LogRecord: + record = super().prepare(record) + record.msg = json.dumps(record.msg) + return record + +#=============================================================================== + class RenameJSONRenderer: def __init__(self, to: str, replace_by: str | None = None, @@ -48,52 +57,68 @@ def __call__(self, logger: WrappedLogger, name: str, event_dict: EventDict) -> s #=============================================================================== -def configure_logging(log_file=None, verbose=False, silent=False, debug=False) -> Optional[logging.FileHandler]: +def configure_logging(log_json_file=None, verbose=False, silent=False, debug=False, +#================================================================================== + log_queue: Optional[multiprocessing.Queue]=None) -> Optional[logging.FileHandler]: log_level = logging.DEBUG if debug else logging.INFO - logging_config = { + # Configure standard logger with a null configuration + logging.config.dictConfig({ 'version': 1, 'handlers': { - 'stream': { - 'class': 'logging.StreamHandler', - 'level': log_level, - 'formatter': 'structured' + 'null': { + 'class': 'logging.NullHandler', + 'level': log_level } }, 'formatters': { - 'json': { - '()': structlog.stdlib.ProcessorFormatter, - "processor": RenameJSONRenderer('msg'), - }, - 'structured': { - '()': structlog.stdlib.ProcessorFormatter, - 'processor': structlog.dev.ConsoleRenderer(colors=True), - }, }, 'loggers': { '': { - 'handlers': ['stream'], + 'handlers': ['null'], 'level': log_level, 'propagate': True }, } - } - - if silent: - logging_config['handlers']['stream']['level'] = logging.CRITICAL + }) + + # Get our logger + logger = logging.getLogger('mapmaker') + + # Log to the console if not sending logs to a queue + if log_queue is None: + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.CRITICAL if silent else log_level) + structured_formatter = structlog.stdlib.ProcessorFormatter( + processors=[ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + structlog.dev.ConsoleRenderer(colors=True) + ], + ) + stream_handler.setFormatter(structured_formatter) + logger.addHandler(stream_handler) + + json_formatter = structlog.stdlib.ProcessorFormatter( + processors=[ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + RenameJSONRenderer('msg') + ], + ) - if log_file is not None: - logging_config['handlers']['jsonfile'] = { - 'class': 'logging.FileHandler', - 'level': log_level, - 'formatter': 'json', - 'filename': log_file - } - logging_config['loggers']['']['handlers'].append('jsonfile') + # Log as JSON to a file if requested + json_handler = None + if log_json_file is not None: + json_handler = logging.FileHandler(log_json_file) + json_handler.setLevel(log_level) + json_handler.setFormatter(json_formatter) + logger.addHandler(json_handler) - # Configure standard logger - logging.config.dictConfig(logging_config) + # For when mapmaker is run as a process by a flatmap server. + if log_queue is not None: + queue_handler = logging.handlers.QueueHandler(log_queue) + queue_handler.setFormatter(json_formatter) + logger.addHandler(queue_handler) # Configure structlog structlog.configure( @@ -111,12 +136,11 @@ def configure_logging(log_file=None, verbose=False, silent=False, debug=False) - cache_logger_on_first_use=True, ) - if log_file is not None: - return typing.cast(logging.FileHandler, logging.getLogger().handlers[1]) + return json_handler #=============================================================================== -log: structlog.BoundLogger = structlog.get_logger() +log: structlog.BoundLogger = structlog.get_logger('mapmaker') #=============================================================================== From 8b8747c0147402d230bc691e11581c0b49ea4e03 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Sun, 17 Aug 2025 17:48:52 +1200 Subject: [PATCH 110/111] Minor tweaks. --- mapmaker/__main__.py | 2 +- mapmaker/maker.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mapmaker/__main__.py b/mapmaker/__main__.py index c4c9667d..6da9569f 100644 --- a/mapmaker/__main__.py +++ b/mapmaker/__main__.py @@ -34,7 +34,7 @@ def arg_parser(): log_options = parser.add_argument_group('Logging') log_options.add_argument('--log', dest='logFile', metavar='LOG_FILE', - help="Append messages to a log file") + help="Append messages to a log file as JSON") log_options.add_argument('--silent', action='store_true', help='Suppress all messages to screen') log_options.add_argument('--verbose', action='store_true', diff --git a/mapmaker/maker.py b/mapmaker/maker.py index 5b55f702..78d2bd16 100644 --- a/mapmaker/maker.py +++ b/mapmaker/maker.py @@ -165,6 +165,7 @@ def __init__(self, options: dict[str, Any]): # Make sure our top-level directory exists map_base = options.get('output') + assert map_base is not None if not os.path.exists(map_base): os.makedirs(map_base) From cb6977984e7deb0c3238c81253f1bb4051bbd448 Mon Sep 17 00:00:00 2001 From: David Brooks Date: Mon, 18 Aug 2025 19:12:24 +1200 Subject: [PATCH 111/111] Update `mapknowledge` to version 1.3.3 and `flatmapknowledge` to version 2.5.3. --- pyproject.toml | 5 ++--- uv.lock | 18 +++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 65477713..022a67e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,8 @@ dependencies = [ "cssselect2>=0.6.0", "webcolors>=1.12", "xmltodict>=0.12.0", - "flatmapknowledge @ https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.2/flatmapknowledge-2.5.2-py3-none-any.whl", - "mapknowledge @ https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl", + "flatmapknowledge @ https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.3/flatmapknowledge-2.5.3-py3-none-any.whl", + "mapknowledge @ https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.3/mapknowledge-1.3.3-py3-none-any.whl", "Pyomo>=6.8", "svgwrite>=1.4.3", "XlsxWriter>=3.0.3", @@ -93,4 +93,3 @@ include = ['mapmaker'] pythonVersion = "3.12" venvPath = "." venv = ".venv" - diff --git a/uv.lock b/uv.lock index bc1f1182..32d32a64 100644 --- a/uv.lock +++ b/uv.lock @@ -256,17 +256,17 @@ wheels = [ [[package]] name = "flatmapknowledge" -version = "2.5.2" -source = { url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.2/flatmapknowledge-2.5.2-py3-none-any.whl" } +version = "2.5.3" +source = { url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.3/flatmapknowledge-2.5.3-py3-none-any.whl" } dependencies = [ { name = "mapknowledge" }, ] wheels = [ - { url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.2/flatmapknowledge-2.5.2-py3-none-any.whl", hash = "sha256:3bf0693f3cfa3b79915a0cd5fae89c05d6fbf53cc9b927e4b8eab83a03beda00" }, + { url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.3/flatmapknowledge-2.5.3-py3-none-any.whl", hash = "sha256:b52164c3e4dd0510a98a32ff7177575e70c620cc40d8062d2681af0a018a1a01" }, ] [package.metadata] -requires-dist = [{ name = "mapknowledge", url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl" }] +requires-dist = [{ name = "mapknowledge", url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.3/mapknowledge-1.3.3-py3-none-any.whl" }] [[package]] name = "frozendict" @@ -546,8 +546,8 @@ wheels = [ [[package]] name = "mapknowledge" -version = "1.3.2" -source = { url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl" } +version = "1.3.3" +source = { url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.3/mapknowledge-1.3.3-py3-none-any.whl" } dependencies = [ { name = "networkx" }, { name = "neurondm" }, @@ -559,7 +559,7 @@ dependencies = [ { name = "structlog" }, ] wheels = [ - { url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl", hash = "sha256:3240fd88a5bdb7333f93910ca112affc8429bacf91f034f462241e0c2958560b" }, + { url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.3/mapknowledge-1.3.3-py3-none-any.whl", hash = "sha256:ede53f90d027030738fae8af0891374d3bf14915b89fe26c740d9e44182fcfa8" }, ] [package.metadata] @@ -637,12 +637,12 @@ requires-dist = [ { name = "beziers", specifier = ">=0.5.0" }, { name = "colormath", specifier = ">=3.0.0" }, { name = "cssselect2", specifier = ">=0.6.0" }, - { name = "flatmapknowledge", url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.2/flatmapknowledge-2.5.2-py3-none-any.whl" }, + { name = "flatmapknowledge", url = "https://github.com/AnatomicMaps/flatmap-knowledge/releases/download/v2.5.3/flatmapknowledge-2.5.3-py3-none-any.whl" }, { name = "gitpython", specifier = ">=3.1.41" }, { name = "giturlparse", specifier = ">=0.12.0" }, { name = "lark", specifier = ">=1.2.2" }, { name = "lxml", specifier = ">=5.2.2" }, - { name = "mapknowledge", url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.2/mapknowledge-1.3.2-py3-none-any.whl" }, + { name = "mapknowledge", url = "https://github.com/AnatomicMaps/map-knowledge/releases/download/v1.3.3/mapknowledge-1.3.3-py3-none-any.whl" }, { name = "mbutil", specifier = ">=0.3.0" }, { name = "mercantile", specifier = ">=1.2.1" }, { name = "multiprocess", specifier = ">=0.70.13" },