update_import_examples.py

#!/usr/bin/env python3
"""Regenerate the example source + command + output blocks on the import pages.

For each import format it reads the example source file under examples/imports/,
runs `datacontract import <format>` to generate the resulting ODCS contract,
writes that contract back next to the source (so it is committed and linted by
lint_examples.py), and injects source + command + contract into
docs/docs/imports/<format>.md between the AUTOGENERATED markers.

    python update_import_examples.py
"""

import re
from pathlib import Path

from docs_examples import DOCS, EXAMPLES_URL, REPO, fence, run_cli, upsert

# format -> (source file relative to repo, source fence language, extra run args)
IMPORTS = {
    "sql": ("examples/imports/sql/orders.sql", "sql", ["--dialect", "postgres"]),
    "csv": ("examples/imports/csv/orders.csv", "text", []),
    "json": ("examples/imports/json/orders.json", "json", []),
    "jsonschema": ("examples/imports/jsonschema/orders.schema.json", "json", []),
    "avro": ("examples/imports/avro/orders.avsc", "json", []),
    "protobuf": ("examples/imports/protobuf/orders.proto", "protobuf", []),
    "pydantic-model": ("examples/imports/pydantic-model/orders.py", "python", []),
    "dbml": ("examples/imports/dbml/orders.dbml", "text", []),
    "bigquery": ("examples/imports/bigquery/orders.json", "json", []),
    "iceberg": ("examples/imports/iceberg/orders.json", "json", []),
    "xsd": ("examples/imports/xsd/orders.xsd", "xml", []),
    "openapi": ("examples/imports/openapi/orders.yaml", "yaml", ["--operation", "listOrders"]),
}

OUT_MAX = 34  # max lines of generated ODCS shown before truncating

# The hand-written command block to replace on the first run (before markers exist).
FALLBACK = re.compile(r"```bash\n.*?\n```", re.DOTALL)


def main() -> None:
    for fmt, (source_rel, src_lang, extra) in IMPORTS.items():
        source = REPO / source_rel
        source_name = source.name

        contract = run_cli(["import", fmt, "--source", source_rel, *extra])

        # Persist the generated contract next to its source so it is version
        # controlled and validated by lint_examples.py.
        out_path = source.parent / "datacontract.odcs.yaml"
        out_path.write_text(contract + "\n")

        url = f"{EXAMPLES_URL}/imports/{fmt}/{source_name}"
        display = f"datacontract import {fmt} --source {source_name}" + ("".join(f" {a}" for a in extra))
        block = (
            f"Given this [`{source_name}`]({url}):\n\n"
            f"{fence(src_lang, source.read_text().strip(), None)}\n\n"
            f"Run:\n\n```bash\n{display}\n```\n\n"
            f"to produce the data contract:\n\n"
            f"{fence('yaml', contract, OUT_MAX)}"
        )
        upsert(DOCS / "imports" / f"{fmt}.md", block, FALLBACK)
        excerpt = len(contract.split("\n")) > OUT_MAX
        print(f"ok imports/{fmt}.md{' (excerpt)' if excerpt else ''}  -> {Path(out_path).relative_to(REPO)}")


if __name__ == "__main__":
    main()