Skip to content

Commit 6d7ccd7

Browse files
CleptomaniaCleptomania
committed
Default to UTF-8 Encoding and Add Optional Encoding Argument (#81)
1 parent c8e7bbd commit 6d7ccd7

16 files changed

+118
-74
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
44

55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66

7+
## [2.2.8] - UNRELEASED
8+
9+
Converts all file loading to use UTF-8 encoding by default. In most cases, all Tiled files will be exported from Tiled in UTF-8 encoding, however the python `open()` function uses the system default locale. The only case where Tiled would not have used UTF-8 is for JSON files when Tiled was compiled against Qt 5, which is only in some builds of Tiled from older systems. All XML files exported from Tiled will always be UTF-8. If someone happens to have a JSON file which was exported from Tiled on an encoding other than UTF-8, or for some other reason is in a different encoding. This can be switched using a new optional argument named `encoding` in the various public API `parse` functions such as `parse_map()`. This value is handed down through the pipeline of file loading in pytiled-parser, and will apply to every file loaded during the chain from this. This means that every file in a chain(for example, Map, Tileset, and Template File) must share the same encoding. This new argument is a string which is ultimately passed to the Python [open()](https://docs.python.org/3/library/functions.html#open) function. This change does introduce breaking changes in the underlying API which is not intended to be public facing, but if you are going deeper than the top level parse functions, you may need to adjust for this, as many of the underlying internal functions now have a mandatory encoding argument.
10+
711
## [2.2.7] - 2024-10-03
812

913
Fixes a bug when using the TMX format, where multi-line String properties would not be correctly parsed, as they are placed differently in the XML than single line strings. (#75)

‎pytiled_parser/parser.py

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,64 +14,67 @@
1414
from pytiled_parser.world import parse_world as _parse_world
1515

1616

17-
def parse_map(file: Path) -> TiledMap:
17+
def parse_map(file: Path, encoding: str = "utf-8") -> TiledMap:
1818
"""Parse the raw Tiled map into a pytiled_parser type
1919
2020
Args:
2121
file: Path to the map file
22+
encoding: The character encoding set to use when opening the file
2223
2324
Returns:
2425
TiledMap: A parsed and typed TiledMap
2526
"""
26-
parser = check_format(file)
27+
parser = check_format(file, encoding)
2728

2829
# The type ignores are because mypy for some reason thinks those functions return Any
2930
if parser == "tmx":
30-
return tmx_map_parse(file) # type: ignore
31+
return tmx_map_parse(file, encoding) # type: ignore
3132
else:
3233
try:
33-
return json_map_parse(file) # type: ignore
34+
return json_map_parse(file, encoding) # type: ignore
3435
except ValueError:
3536
raise UnknownFormat(
3637
"Unknown Map Format, please use either the TMX or JSON format. "
3738
"This message could also mean your map file is invalid or corrupted."
3839
)
3940

4041

41-
def parse_tileset(file: Path) -> Tileset:
42+
def parse_tileset(file: Path, encoding: str = "utf-8") -> Tileset:
4243
"""Parse the raw Tiled Tileset into a pytiled_parser type
4344
4445
Args:
4546
file: Path to the map file
47+
encoding: The character encoding set to use when opening the file
4648
4749
Returns:
4850
Tileset: A parsed and typed Tileset
4951
"""
50-
parser = check_format(file)
52+
parser = check_format(file, encoding)
5153

5254
if parser == "tmx":
53-
with open(file) as map_file:
55+
with open(file, encoding=encoding) as map_file:
5456
raw_tileset = etree.parse(map_file).getroot()
55-
return tmx_tileset_parse(raw_tileset, 1)
57+
return tmx_tileset_parse(raw_tileset, 1, encoding)
5658
else:
5759
try:
58-
with open(file) as my_file:
60+
with open(file, encoding=encoding) as my_file:
5961
raw_tileset = json.load(my_file)
60-
return json_tileset_parse(raw_tileset, 1)
62+
return json_tileset_parse(raw_tileset, 1, encoding)
6163
except ValueError:
6264
raise UnknownFormat(
6365
"Unknowm Tileset Format, please use either the TSX or JSON format. "
6466
"This message could also mean your tileset file is invalid or corrupted."
6567
)
6668

6769

68-
def parse_world(file: Path) -> World:
70+
def parse_world(file: Path, encoding: str = "utf-8") -> World:
6971
"""Parse the raw world file into a pytiled_parser type
7072
7173
Args:
7274
file: Path to the world file
75+
encoding: The character encoding set to use when opening the file
7376
7477
Returns:
7578
World: A parsed and typed World
7679
"""
77-
return _parse_world(file)
80+
return _parse_world(file, encoding)

‎pytiled_parser/parsers/json/layer.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,7 @@ def _parse_tile_layer(raw_layer: RawLayer) -> TileLayer:
298298

299299
def _parse_object_layer(
300300
raw_layer: RawLayer,
301+
encoding: str,
301302
parent_dir: Optional[Path] = None,
302303
) -> ObjectLayer:
303304
"""Parse the raw_layer to an ObjectLayer.
@@ -310,7 +311,7 @@ def _parse_object_layer(
310311
"""
311312
objects = []
312313
for object_ in raw_layer["objects"]:
313-
objects.append(parse_object(object_, parent_dir))
314+
objects.append(parse_object(object_, encoding, parent_dir))
314315

315316
return ObjectLayer(
316317
tiled_objects=objects,
@@ -339,7 +340,7 @@ def _parse_image_layer(raw_layer: RawLayer) -> ImageLayer:
339340

340341

341342
def _parse_group_layer(
342-
raw_layer: RawLayer, parent_dir: Optional[Path] = None
343+
raw_layer: RawLayer, encoding, parent_dir: Optional[Path] = None
343344
) -> LayerGroup:
344345
"""Parse the raw_layer to a LayerGroup.
345346
@@ -352,13 +353,14 @@ def _parse_group_layer(
352353
layers = []
353354

354355
for layer in raw_layer["layers"]:
355-
layers.append(parse(layer, parent_dir=parent_dir))
356+
layers.append(parse(layer, encoding, parent_dir=parent_dir))
356357

357358
return LayerGroup(layers=layers, **_parse_common(raw_layer).__dict__)
358359

359360

360361
def parse(
361362
raw_layer: RawLayer,
363+
encoding: str,
362364
parent_dir: Optional[Path] = None,
363365
) -> Layer:
364366
"""Parse a raw Layer into a pytiled_parser object.
@@ -378,9 +380,9 @@ def parse(
378380
type_ = raw_layer["type"]
379381

380382
if type_ == "objectgroup":
381-
return _parse_object_layer(raw_layer, parent_dir)
383+
return _parse_object_layer(raw_layer, encoding, parent_dir)
382384
elif type_ == "group":
383-
return _parse_group_layer(raw_layer, parent_dir)
385+
return _parse_group_layer(raw_layer, encoding, parent_dir)
384386
elif type_ == "imagelayer":
385387
return _parse_image_layer(raw_layer)
386388
elif type_ == "tilelayer":

‎pytiled_parser/parsers/json/tiled_map.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
"""
5656

5757

58-
def parse(file: Path) -> TiledMap:
58+
def parse(file: Path, encoding: str) -> TiledMap:
5959
"""Parse the raw Tiled map into a pytiled_parser type.
6060
6161
Args:
@@ -64,7 +64,7 @@ def parse(file: Path) -> TiledMap:
6464
Returns:
6565
TiledMap: A parsed TiledMap.
6666
"""
67-
with open(file) as map_file:
67+
with open(file, encoding=encoding) as map_file:
6868
raw_tiled_map = json.load(map_file)
6969

7070
parent_dir = file.parent
@@ -76,20 +76,22 @@ def parse(file: Path) -> TiledMap:
7676
if raw_tileset.get("source") is not None:
7777
# Is an external Tileset
7878
tileset_path = Path(parent_dir / raw_tileset["source"])
79-
parser = check_format(tileset_path)
80-
with open(tileset_path) as raw_tileset_file:
79+
parser = check_format(tileset_path, encoding)
80+
with open(tileset_path, encoding=encoding) as raw_tileset_file:
8181
if parser == "tmx":
8282
raw_tileset_external = etree.parse(raw_tileset_file).getroot()
8383
tilesets[raw_tileset["firstgid"]] = parse_tmx_tileset(
8484
raw_tileset_external,
8585
raw_tileset["firstgid"],
86+
encoding,
8687
external_path=tileset_path.parent,
8788
)
8889
else:
8990
try:
9091
tilesets[raw_tileset["firstgid"]] = parse_json_tileset(
9192
json.load(raw_tileset_file),
9293
raw_tileset["firstgid"],
94+
encoding,
9395
external_path=tileset_path.parent,
9496
)
9597
except ValueError:
@@ -101,7 +103,7 @@ def parse(file: Path) -> TiledMap:
101103
# Is an embedded Tileset
102104
raw_tileset = cast(RawTileSet, raw_tileset)
103105
tilesets[raw_tileset["firstgid"]] = parse_json_tileset(
104-
raw_tileset, raw_tileset["firstgid"]
106+
raw_tileset, raw_tileset["firstgid"], encoding
105107
)
106108

107109
if isinstance(raw_tiled_map["version"], float): # pragma: no cover
@@ -113,7 +115,10 @@ def parse(file: Path) -> TiledMap:
113115
map_ = TiledMap(
114116
map_file=file,
115117
infinite=raw_tiled_map.get("infinite", False),
116-
layers=[parse_layer(layer_, parent_dir) for layer_ in raw_tiled_map["layers"]],
118+
layers=[
119+
parse_layer(layer_, encoding, parent_dir)
120+
for layer_ in raw_tiled_map["layers"]
121+
],
117122
map_size=Size(raw_tiled_map["width"], raw_tiled_map["height"]),
118123
next_layer_id=raw_tiled_map.get("nextlayerid"),
119124
next_object_id=raw_tiled_map["nextobjectid"],
@@ -157,6 +162,7 @@ def parse(file: Path) -> TiledMap:
157162
map_.tilesets[new_firstgid] = parse_json_tileset(
158163
tiled_object.new_tileset,
159164
new_firstgid,
165+
encoding,
160166
tiled_object.new_tileset_path,
161167
)
162168
tiled_object.gid = tiled_object.gid + (new_firstgid - 1)

‎pytiled_parser/parsers/json/tiled_object.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,7 @@ def _get_parser(raw_object: RawObject) -> Callable[[RawObject], TiledObject]:
291291

292292
def parse(
293293
raw_object: RawObject,
294+
encoding: str,
294295
parent_dir: Optional[Path] = None,
295296
) -> TiledObject:
296297
"""Parse the raw object into a pytiled_parser version
@@ -314,7 +315,9 @@ def parse(
314315
"A parent directory must be specified when using object templates."
315316
)
316317
template_path = Path(parent_dir / raw_object["template"])
317-
template, new_tileset, new_tileset_path = load_object_template(template_path)
318+
template, new_tileset, new_tileset_path = load_object_template(
319+
template_path, encoding
320+
)
318321

319322
if isinstance(template, dict):
320323
loaded_template = template["object"]

‎pytiled_parser/parsers/json/tileset.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,9 @@ def _parse_grid(raw_grid: RawGrid) -> Grid:
161161
)
162162

163163

164-
def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile:
164+
def _parse_tile(
165+
raw_tile: RawTile, encoding: str, external_path: Optional[Path] = None
166+
) -> Tile:
165167
"""Parse the raw_tile to a Tile object.
166168
167169
Args:
@@ -180,7 +182,7 @@ def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile
180182
tile.animation.append(_parse_frame(frame))
181183

182184
if raw_tile.get("objectgroup") is not None:
183-
tile.objects = parse_layer(raw_tile["objectgroup"])
185+
tile.objects = parse_layer(raw_tile["objectgroup"], encoding)
184186

185187
if raw_tile.get("properties") is not None:
186188
tile.properties = parse_properties(raw_tile["properties"])
@@ -231,6 +233,7 @@ def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile
231233
def parse(
232234
raw_tileset: RawTileSet,
233235
firstgid: int,
236+
encoding: str,
234237
external_path: Optional[Path] = None,
235238
) -> Tileset:
236239
"""Parse the raw tileset into a pytiled_parser type
@@ -309,12 +312,12 @@ def parse(
309312
assert raw_tile.get("id") is None
310313
raw_tile["id"] = int(raw_tile_id)
311314
tiles[raw_tile["id"]] = _parse_tile(
312-
raw_tile, external_path=external_path
315+
raw_tile, encoding, external_path=external_path
313316
)
314317
else:
315318
for raw_tile in raw_tileset["tiles"]:
316319
tiles[raw_tile["id"]] = _parse_tile(
317-
raw_tile, external_path=external_path
320+
raw_tile, encoding, external_path=external_path
318321
)
319322
tileset.tiles = tiles
320323

‎pytiled_parser/parsers/tmx/layer.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ def _parse_tile_layer(raw_layer: etree.Element) -> TileLayer:
262262

263263

264264
def _parse_object_layer(
265-
raw_layer: etree.Element, parent_dir: Optional[Path] = None
265+
raw_layer: etree.Element, encoding: str, parent_dir: Optional[Path] = None
266266
) -> ObjectLayer:
267267
"""Parse the raw_layer to an ObjectLayer.
268268
@@ -274,7 +274,7 @@ def _parse_object_layer(
274274
"""
275275
objects = []
276276
for object_ in raw_layer.findall("./object"):
277-
objects.append(parse_object(object_, parent_dir))
277+
objects.append(parse_object(object_, encoding, parent_dir))
278278

279279
object_layer = ObjectLayer(
280280
tiled_objects=objects,
@@ -316,7 +316,7 @@ def _parse_image_layer(raw_layer: etree.Element) -> ImageLayer:
316316

317317

318318
def _parse_group_layer(
319-
raw_layer: etree.Element, parent_dir: Optional[Path] = None
319+
raw_layer: etree.Element, encoding: str, parent_dir: Optional[Path] = None
320320
) -> LayerGroup:
321321
"""Parse the raw_layer to a LayerGroup.
322322
@@ -331,7 +331,7 @@ def _parse_group_layer(
331331
layers.append(_parse_tile_layer(layer))
332332

333333
for layer in raw_layer.findall("./objectgroup"):
334-
layers.append(_parse_object_layer(layer, parent_dir))
334+
layers.append(_parse_object_layer(layer, encoding, parent_dir))
335335

336336
for layer in raw_layer.findall("./imagelayer"):
337337
layers.append(_parse_image_layer(layer))
@@ -350,6 +350,7 @@ def _parse_group_layer(
350350

351351
def parse(
352352
raw_layer: etree.Element,
353+
encoding: str,
353354
parent_dir: Optional[Path] = None,
354355
) -> Layer:
355356
"""Parse a raw Layer into a pytiled_parser object.
@@ -369,9 +370,9 @@ def parse(
369370
type_ = raw_layer.tag
370371

371372
if type_ == "objectgroup":
372-
return _parse_object_layer(raw_layer, parent_dir)
373+
return _parse_object_layer(raw_layer, encoding, parent_dir)
373374
elif type_ == "group":
374-
return _parse_group_layer(raw_layer, parent_dir)
375+
return _parse_group_layer(raw_layer, encoding, parent_dir)
375376
elif type_ == "imagelayer":
376377
return _parse_image_layer(raw_layer)
377378
elif type_ == "layer":

0 commit comments

Comments
 (0)