diff --git a/CHANGELOG.md b/CHANGELOG.md index edf83d0..d5ddd1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **PX: values that start on the `DATA=` line** (#50). INE writes `DATA=70.0 68.0 ...`; the + first value was `"DATA=70.0"`, and a closing `;` followed by a space stayed on the last value. +- **Shapefile: DBF encodings without a `.cpg` file** (#50). The ESRI `.cst` code page file is read + (Statistics Finland ships one), then the language driver id in the DBF header; with neither, the + records are UTF-8 when they decode as UTF-8, else cp1252. Such files failed with + `dbfFileException`. + - **CSV: the encoding of compressed files is detected** (#40). A `.csv.gz`, `.csv.bz2`, `.csv.xz` or `.csv.zst` file was always read as UTF-8, so cp1252 or cp932 files failed with `UnicodeDecodeError`. The decompressed bytes are now sampled as for plain files. chardet diff --git a/iterable/datatypes/px.py b/iterable/datatypes/px.py index df11432..a381c91 100644 --- a/iterable/datatypes/px.py +++ b/iterable/datatypes/px.py @@ -159,8 +159,10 @@ def _parse_data_section(self, content: str, start_idx: int): # Combine all data lines and split by whitespace data_text = " ".join(data_lines) - # Remove trailing semicolon if present - data_text = data_text.rstrip(";").strip() + # Values may start on the DATA= line itself (INE writes "DATA=70.0 68.0 ..."); #50. + data_text = re.sub(r"^DATA\s*=", "", data_text.strip(), flags=re.IGNORECASE) + # Remove the closing semicolon, also when spaces or a line break follow it + data_text = data_text.strip().rstrip(";").strip() # Split by whitespace and filter empty strings values = [v for v in data_text.split() if v] diff --git a/iterable/datatypes/shapefile.py b/iterable/datatypes/shapefile.py index 0322437..19d35e0 100644 --- a/iterable/datatypes/shapefile.py +++ b/iterable/datatypes/shapefile.py @@ -50,21 +50,92 @@ def shapefiles_in_zip(path: str) -> list[str]: return [] -def _read_cpg(path: str, member: str | None) -> str | None: - """Text of the ``.cpg`` next to a ``.shp`` file or archive member, if there is one.""" +def _read_sidecar(path: str, member: str | None, extension: str) -> str | None: + """Text of the ``.cpg`` or ``.cst`` file next to a ``.shp`` file or archive member.""" if member is not None: stem = member[: -len(".shp")] with zipfile.ZipFile(path) as archive: names = {name.lower(): name for name in archive.namelist()} - name = names.get(stem.lower() + ".cpg") + name = names.get(stem.lower() + extension) return archive.read(name).decode("ascii", "replace") if name else None - cpg = path[: -len(".shp")] + ".cpg" if path.lower().endswith(".shp") else None - if cpg is None or not os.path.exists(cpg): + sidecar = path[: -len(".shp")] + extension if path.lower().endswith(".shp") else None + if sidecar is None or not os.path.exists(sidecar): return None - with open(cpg, encoding="ascii", errors="replace") as handle: + with open(sidecar, encoding="ascii", errors="replace") as handle: return handle.read() +def _read_cpg(path: str, member: str | None) -> str | None: + """Text of the ``.cpg`` next to a ``.shp`` file or archive member, if there is one.""" + return _read_sidecar(path, member, ".cpg") + + +# DBF language driver ids (header byte 29) and their code pages. +_LDID_CODE_PAGES = { + 0x01: "cp437", + 0x02: "cp850", + 0x03: "cp1252", + 0x57: "cp1252", + 0x58: "cp1252", + 0x59: "cp1252", + 0x64: "cp852", + 0x65: "cp866", + 0x66: "cp865", + 0x67: "cp861", + 0x6A: "cp737", + 0x6B: "cp857", + 0x78: "cp950", + 0x79: "cp949", + 0x7A: "cp936", + 0x7B: "cp932", + 0x7C: "cp874", + 0x7D: "cp1255", + 0x7E: "cp1256", + 0xC8: "cp1250", + 0xC9: "cp1251", + 0xCA: "cp1254", + 0xCB: "cp1253", + 0xCC: "cp1257", +} + + +def _read_dbf_sample(path: str, member: str | None, size: int = 65536) -> bytes: + """The first bytes of the ``.dbf`` next to a ``.shp`` file or archive member.""" + if member is not None: + stem = member[: -len(".shp")] + with zipfile.ZipFile(path) as archive: + names = {name.lower(): name for name in archive.namelist()} + name = names.get(stem.lower() + ".dbf") + if not name: + return b"" + with archive.open(name) as handle: + return handle.read(size) + dbf = path[: -len(".shp")] + ".dbf" if path.lower().endswith(".shp") else None + if dbf is None or not os.path.exists(dbf): + return b"" + with open(dbf, "rb") as handle: + return handle.read(size) + + +def guess_dbf_encoding(sample: bytes) -> str: + """DBF encoding when no ``.cpg`` or ``.cst`` declares it. + + The language driver id in header byte 29 names the code page; when it is 0, the records + are UTF-8 if they decode as UTF-8 (the end of the sample may cut a character), else cp1252. + """ + if len(sample) > 29 and sample[29] in _LDID_CODE_PAGES: + return _LDID_CODE_PAGES[sample[29]] + header_length = int.from_bytes(sample[8:10], "little") if len(sample) >= 10 else 0 + records = sample[header_length:] if 0 < header_length < len(sample) else b"" + try: + records.decode("utf-8") + return "utf-8" + except UnicodeDecodeError as error: + if error.start >= len(records) - 3: + return "utf-8" + return "cp1252" + + def _pyshp_decodes_cpg(text: str) -> bool: """Whether pyshp >= 3, which reads the ``.cpg`` itself, can use its text as a codec name.""" if not _PYSHP_READS_CPG: @@ -241,6 +312,11 @@ def reset(self): target, member = self._reader_target() cpg_text = None if self.dbf_encoding else _read_cpg(self.filename, member) cpg = cpg_encoding(cpg_text) if cpg_text else None + if not (self.dbf_encoding or cpg): + # ESRI's .cst code page file (Statistics Finland), then the DBF header. + cst_text = _read_sidecar(self.filename, member, ".cst") + cpg = cpg_encoding(cst_text) if cst_text else None + cpg = cpg or guess_dbf_encoding(_read_dbf_sample(self.filename, member)) encoding = self.dbf_encoding or cpg or "utf-8" kwargs: dict[str, Any] = {"encoding": encoding} if cpg_text is not None and _pyshp_decodes_cpg(cpg_text): diff --git a/tests/test_px.py b/tests/test_px.py index 3f0cc56..1b9c51f 100644 --- a/tests/test_px.py +++ b/tests/test_px.py @@ -142,3 +142,13 @@ def test_px_metadata_option_leaves_out_keyword_copies(tmp_path): assert next(source) == {"Gemeinde": "Zürich", "Jahr": "2000", "VALUE": 1} with open_iterable(str(path)) as source: assert "_CHARSET" in next(source) + + +def test_px_values_on_the_data_line(tmp_path): + """INE writes the first values on the DATA= line itself (#50).""" + from iterable.helpers.detect import open_iterable + + path = tmp_path / "ine.px" + path.write_bytes(PX_ANSI.replace("DATA=\n1 2\n3 4;", "DATA=1 2\n3 4;").encode("cp1252")) + with open_iterable(str(path), iterableargs={"metadata": False}) as source: + assert [row["VALUE"] for row in source] == [1, 2, 3, 4] diff --git a/tests/test_shapefile.py b/tests/test_shapefile.py index 76165a3..568fc74 100644 --- a/tests/test_shapefile.py +++ b/tests/test_shapefile.py @@ -227,3 +227,60 @@ def test_zip_with_several_shapefiles_needs_a_member(tmp_path): source = ShapefileIterable(archive, options={"member": "b/other.shp"}) assert source.read()["properties"]["plz"] == "79379" source.close() + + +def _zip_without_cpg(tmp_path, sidecar=None, encoding="cp1252", ldid=None): + """A zipped Shapefile with a .cst file or no code page file at all (#50).""" + import zipfile + + shapefile = pytest.importorskip("shapefile") + base = str(tmp_path / "kunta") + writer = shapefile.Writer(base, shapeType=shapefile.POINT, encoding=encoding) + writer.field("nimi", "C", 40) + writer.point(23.0, 63.0) + writer.record("Alajärvi") + writer.point(25.7, 62.2) + writer.record("Äänekoski") + writer.close() + if ldid is not None: + with open(base + ".dbf", "r+b") as handle: + handle.seek(29) + handle.write(bytes([ldid])) + exts = ["shp", "shx", "dbf"] + if sidecar is not None: + with open(base + ".cst", "w") as handle: + handle.write(sidecar) + exts.append("cst") + archive = str(tmp_path / "kunta.shp.zip") + with zipfile.ZipFile(archive, "w") as zipped: + for ext in exts: + zipped.write(f"{base}.{ext}", f"kunta.{ext}") + return archive + + +@pytest.mark.parametrize( + "sidecar,encoding,ldid", + [ + ("ISO-8859-1", "latin-1", 0), # ESRI .cst file (Statistics Finland) + (None, "cp1252", 0x57), # language driver id in the DBF header + (None, "cp1252", 0), # nothing declared, not UTF-8: cp1252 + (None, "utf-8", 0), # nothing declared, UTF-8 records + ], +) +def test_dbf_encoding_without_cpg(tmp_path, sidecar, encoding, ldid): + from iterable.helpers.detect import open_iterable + + archive = _zip_without_cpg(tmp_path, sidecar, encoding, ldid) + with open_iterable(archive) as source: + names = [row["properties"]["nimi"] for row in source] + assert names == ["Alajärvi", "Äänekoski"] + + +def test_guess_dbf_encoding(): + from iterable.datatypes.shapefile import guess_dbf_encoding + + header = bytearray(64) + header[8:10] = (40).to_bytes(2, "little") + assert guess_dbf_encoding(bytes(header[:29]) + bytes([0xC9]) + bytes(header[30:])) == "cp1251" + assert guess_dbf_encoding(bytes(header) + "Äänekoski".encode()) == "utf-8" + assert guess_dbf_encoding(bytes(header) + "Äänekoski".encode("cp1252")) == "cp1252"