Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

- **PX: values that start on the `DATA=` line** (#50). INE writes `DATA=70.0 68.0 ...`; the
first value was `"DATA=70.0"`, and a closing `;` followed by a space stayed on the last value.
- **Shapefile: DBF encodings without a `.cpg` file** (#50). The ESRI `.cst` code page file is read
(Statistics Finland ships one), then the language driver id in the DBF header; with neither, the
records are UTF-8 when they decode as UTF-8, else cp1252. Such files failed with
`dbfFileException`.

- **CSV: the encoding of compressed files is detected** (#40). A `.csv.gz`, `.csv.bz2`,
`.csv.xz` or `.csv.zst` file was always read as UTF-8, so cp1252 or cp932 files failed with
`UnicodeDecodeError`. The decompressed bytes are now sampled as for plain files. chardet
Expand Down
6 changes: 4 additions & 2 deletions iterable/datatypes/px.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,10 @@ def _parse_data_section(self, content: str, start_idx: int):

# Combine all data lines and split by whitespace
data_text = " ".join(data_lines)
# Remove trailing semicolon if present
data_text = data_text.rstrip(";").strip()
# Values may start on the DATA= line itself (INE writes "DATA=70.0 68.0 ..."); #50.
data_text = re.sub(r"^DATA\s*=", "", data_text.strip(), flags=re.IGNORECASE)
# Remove the closing semicolon, also when spaces or a line break follow it
data_text = data_text.strip().rstrip(";").strip()

# Split by whitespace and filter empty strings
values = [v for v in data_text.split() if v]
Expand Down
88 changes: 82 additions & 6 deletions iterable/datatypes/shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,92 @@ def shapefiles_in_zip(path: str) -> list[str]:
return []


def _read_cpg(path: str, member: str | None) -> str | None:
"""Text of the ``.cpg`` next to a ``.shp`` file or archive member, if there is one."""
def _read_sidecar(path: str, member: str | None, extension: str) -> str | None:
"""Text of the ``.cpg`` or ``.cst`` file next to a ``.shp`` file or archive member."""
if member is not None:
stem = member[: -len(".shp")]
with zipfile.ZipFile(path) as archive:
names = {name.lower(): name for name in archive.namelist()}
name = names.get(stem.lower() + ".cpg")
name = names.get(stem.lower() + extension)
return archive.read(name).decode("ascii", "replace") if name else None
cpg = path[: -len(".shp")] + ".cpg" if path.lower().endswith(".shp") else None
if cpg is None or not os.path.exists(cpg):
sidecar = path[: -len(".shp")] + extension if path.lower().endswith(".shp") else None
if sidecar is None or not os.path.exists(sidecar):
return None
with open(cpg, encoding="ascii", errors="replace") as handle:
with open(sidecar, encoding="ascii", errors="replace") as handle:
return handle.read()


def _read_cpg(path: str, member: str | None) -> str | None:
"""Text of the ``.cpg`` next to a ``.shp`` file or archive member, if there is one."""
return _read_sidecar(path, member, ".cpg")


# DBF language driver ids (header byte 29) and their code pages.
_LDID_CODE_PAGES = {
0x01: "cp437",
0x02: "cp850",
0x03: "cp1252",
0x57: "cp1252",
0x58: "cp1252",
0x59: "cp1252",
0x64: "cp852",
0x65: "cp866",
0x66: "cp865",
0x67: "cp861",
0x6A: "cp737",
0x6B: "cp857",
0x78: "cp950",
0x79: "cp949",
0x7A: "cp936",
0x7B: "cp932",
0x7C: "cp874",
0x7D: "cp1255",
0x7E: "cp1256",
0xC8: "cp1250",
0xC9: "cp1251",
0xCA: "cp1254",
0xCB: "cp1253",
0xCC: "cp1257",
}


def _read_dbf_sample(path: str, member: str | None, size: int = 65536) -> bytes:
"""The first bytes of the ``.dbf`` next to a ``.shp`` file or archive member."""
if member is not None:
stem = member[: -len(".shp")]
with zipfile.ZipFile(path) as archive:
names = {name.lower(): name for name in archive.namelist()}
name = names.get(stem.lower() + ".dbf")
if not name:
return b""
with archive.open(name) as handle:
return handle.read(size)
dbf = path[: -len(".shp")] + ".dbf" if path.lower().endswith(".shp") else None
if dbf is None or not os.path.exists(dbf):
return b""
with open(dbf, "rb") as handle:
return handle.read(size)


def guess_dbf_encoding(sample: bytes) -> str:
"""DBF encoding when no ``.cpg`` or ``.cst`` declares it.

The language driver id in header byte 29 names the code page; when it is 0, the records
are UTF-8 if they decode as UTF-8 (the end of the sample may cut a character), else cp1252.
"""
if len(sample) > 29 and sample[29] in _LDID_CODE_PAGES:
return _LDID_CODE_PAGES[sample[29]]
header_length = int.from_bytes(sample[8:10], "little") if len(sample) >= 10 else 0
records = sample[header_length:] if 0 < header_length < len(sample) else b""
try:
records.decode("utf-8")
return "utf-8"
except UnicodeDecodeError as error:
if error.start >= len(records) - 3:
return "utf-8"
return "cp1252"


def _pyshp_decodes_cpg(text: str) -> bool:
"""Whether pyshp >= 3, which reads the ``.cpg`` itself, can use its text as a codec name."""
if not _PYSHP_READS_CPG:
Expand Down Expand Up @@ -241,6 +312,11 @@ def reset(self):
target, member = self._reader_target()
cpg_text = None if self.dbf_encoding else _read_cpg(self.filename, member)
cpg = cpg_encoding(cpg_text) if cpg_text else None
if not (self.dbf_encoding or cpg):
# ESRI's .cst code page file (Statistics Finland), then the DBF header.
cst_text = _read_sidecar(self.filename, member, ".cst")
cpg = cpg_encoding(cst_text) if cst_text else None
cpg = cpg or guess_dbf_encoding(_read_dbf_sample(self.filename, member))
encoding = self.dbf_encoding or cpg or "utf-8"
kwargs: dict[str, Any] = {"encoding": encoding}
if cpg_text is not None and _pyshp_decodes_cpg(cpg_text):
Expand Down
10 changes: 10 additions & 0 deletions tests/test_px.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,13 @@ def test_px_metadata_option_leaves_out_keyword_copies(tmp_path):
assert next(source) == {"Gemeinde": "Zürich", "Jahr": "2000", "VALUE": 1}
with open_iterable(str(path)) as source:
assert "_CHARSET" in next(source)


def test_px_values_on_the_data_line(tmp_path):
"""INE writes the first values on the DATA= line itself (#50)."""
from iterable.helpers.detect import open_iterable

path = tmp_path / "ine.px"
path.write_bytes(PX_ANSI.replace("DATA=\n1 2\n3 4;", "DATA=1 2\n3 4;").encode("cp1252"))
with open_iterable(str(path), iterableargs={"metadata": False}) as source:
assert [row["VALUE"] for row in source] == [1, 2, 3, 4]
57 changes: 57 additions & 0 deletions tests/test_shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,3 +227,60 @@ def test_zip_with_several_shapefiles_needs_a_member(tmp_path):
source = ShapefileIterable(archive, options={"member": "b/other.shp"})
assert source.read()["properties"]["plz"] == "79379"
source.close()


def _zip_without_cpg(tmp_path, sidecar=None, encoding="cp1252", ldid=None):
"""A zipped Shapefile with a .cst file or no code page file at all (#50)."""
import zipfile

shapefile = pytest.importorskip("shapefile")
base = str(tmp_path / "kunta")
writer = shapefile.Writer(base, shapeType=shapefile.POINT, encoding=encoding)
writer.field("nimi", "C", 40)
writer.point(23.0, 63.0)
writer.record("Alajärvi")
writer.point(25.7, 62.2)
writer.record("Äänekoski")
writer.close()
if ldid is not None:
with open(base + ".dbf", "r+b") as handle:
handle.seek(29)
handle.write(bytes([ldid]))
exts = ["shp", "shx", "dbf"]
if sidecar is not None:
with open(base + ".cst", "w") as handle:
handle.write(sidecar)
exts.append("cst")
archive = str(tmp_path / "kunta.shp.zip")
with zipfile.ZipFile(archive, "w") as zipped:
for ext in exts:
zipped.write(f"{base}.{ext}", f"kunta.{ext}")
return archive


@pytest.mark.parametrize(
"sidecar,encoding,ldid",
[
("ISO-8859-1", "latin-1", 0), # ESRI .cst file (Statistics Finland)
(None, "cp1252", 0x57), # language driver id in the DBF header
(None, "cp1252", 0), # nothing declared, not UTF-8: cp1252
(None, "utf-8", 0), # nothing declared, UTF-8 records
],
)
def test_dbf_encoding_without_cpg(tmp_path, sidecar, encoding, ldid):
from iterable.helpers.detect import open_iterable

archive = _zip_without_cpg(tmp_path, sidecar, encoding, ldid)
with open_iterable(archive) as source:
names = [row["properties"]["nimi"] for row in source]
assert names == ["Alajärvi", "Äänekoski"]


def test_guess_dbf_encoding():
from iterable.datatypes.shapefile import guess_dbf_encoding

header = bytearray(64)
header[8:10] = (40).to_bytes(2, "little")
assert guess_dbf_encoding(bytes(header[:29]) + bytes([0xC9]) + bytes(header[30:])) == "cp1251"
assert guess_dbf_encoding(bytes(header) + "Äänekoski".encode()) == "utf-8"
assert guess_dbf_encoding(bytes(header) + "Äänekoski".encode("cp1252")) == "cp1252"
Loading