aigrowthtech/scripts/convert-docs.py

"""
Convert the copy decks (.docx) into structured page JSON.

Usage:
    python3 scripts/convert-docs.py "<folder with the .docx files>"

Paragraphs that start with "> " were bold sentences in the deck and render as
emphasised lines. Each document follows the same conventions: bold all-caps lines are eyebrows
(when followed by a heading) or CTAs; Heading 1/2/3 give the structure.
The output lands in src/content/pages/en/<slug>.json. The homepage is hand-built
and is skipped.
"""

import html
import json
import pathlib
import re
import sys
import zipfile

OUT = pathlib.Path(__file__).resolve().parent.parent / "src" / "content" / "pages" / "en"

# Lines in the decks that are notes to the team, not site copy.
DROP = [
    re.compile(r"^<a id="),
    re.compile(r"should remain only if .* substantiate", re.I),
]


def read_blocks(path):
    xml = zipfile.ZipFile(path).read("word/document.xml").decode("utf8")
    out = []
    for p in re.findall(r"<w:p[ >].*?</w:p>", xml, re.S):
        parts, bolds = [], []
        for r in re.findall(r"<w:r[ >].*?</w:r>", p, re.S):
            text = ""
            for tok in re.findall(r"<w:t[^>]*>[^<]*</w:t>|<w:br[^>]*/>", r):
                text += "\n" if tok.startswith("<w:br") else re.sub(r"<[^>]+>", "", tok)
            if not text:
                continue
            parts.append(text)
            if text.strip():
                bolds.append(bool(re.search(r'<w:b(/>| w:val="(1|true)")', r)))
        text = html.unescape("".join(parts)).strip()
        text = re.sub(r"[ \t]*\n[ \t]*", "\n", text)
        if not text or any(d.search(text) for d in DROP):
            continue
        st = re.search(r'<w:pStyle w:val="([^"]+)"', p)
        style = st.group(1) if st else ""
        m = re.match(r"Heading([123])", style)
        if text.startswith(("Title Tag:", "Meta Description:", "Suggested URL:")):
            k, v = text.split(":", 1)
            out.append({"t": "meta", "k": k.strip(), "v": v.strip()})
        elif m:
            out.append({"t": "h" + m.group(1), "v": text})
        elif "<w:numPr>" in p:
            out.append({"t": "li", "v": text})
        elif bolds and all(bolds):
            out.append({"t": "b", "v": text})
        else:
            out.append({"t": "p", "v": text})
    return out


def is_caps(s):
    letters = [c for c in s if c.isalpha()]
    return bool(letters) and all(c.isupper() for c in letters if c not in "me")  # "meSEO"


def new_section(eyebrow=None, title=None):
    return {"eyebrow": eyebrow, "title": title, "intro": [], "items": [], "outro": [], "ctas": [], "form": False}


def new_item(title, kicker=None):
    return {"kicker": kicker, "title": title, "subtitle": None, "body": [], "list": [], "cta": None}


def structure(blocks):
    meta = {}
    sections = []
    sec = None
    item = None
    pending_kicker = None

    for i, b in enumerate(blocks):
        t, v = b["t"], b["v"]
        nxt = blocks[i + 1]["t"] if i + 1 < len(blocks) else None
        if t == "meta":
            meta[b["k"]] = v
            continue

        # Eyebrows: bold (or an all-caps plain line) right before an h1/h2.
        if t in ("b", "p") and nxt in ("h1", "h2") and is_caps(v):
            sec = new_section(eyebrow=v)
            sections.append(sec)
            item = None
            continue
        if t == "b" and nxt == "h3" and re.search(r"FREQUENTLY ASKED|FAQ", v):
            sec = new_section(eyebrow=v)
            sections.append(sec)
            item = None
            continue
        if t == "b" and nxt == "h3" and "|" in v:
            pending_kicker = v
            continue

        if t in ("h1", "h2"):
            if sec is None or sec["title"] is not None:
                sec = new_section()
                sections.append(sec)
            sec["title"] = v
            sec["level"] = t
            item = None
            continue
        if sec is None:
            sec = new_section()
            sections.append(sec)

        if t == "h3":
            item = new_item(v, kicker=pending_kicker)
            pending_kicker = None
            sec["items"].append(item)
            continue

        if t == "b" and not is_caps(v) and not (item is not None and not item["body"] and item["subtitle"] is None):
            # A bold sentence inside running copy: keep it as an emphasised paragraph.
            t, v = "p", "> " + v

        if t == "b":
            if v.strip("()").upper() == "FORM":
                sec["form"] = True
            elif item is not None and not item["body"] and item["subtitle"] is None and not item["list"]:
                item["subtitle"] = v
            elif item is not None and item["cta"] is None and re.match(r"(EXPLORE|VIEW|LEARN|SEE|EXAMINE|DISCOVER)\b", v) and not re.search(r"\bALL\b", v):
                item["cta"] = v
            else:
                sec["ctas"].append(v)
                item = None
            continue

        if t == "li":
            (item["list"] if item else sec["intro"]).append(v)
            continue

        # paragraph
        if item is not None and sec["ctas"] == []:
            item["body"].append(v)
        elif sec["items"]:
            sec["outro"].append(v)
        else:
            sec["intro"].append(v)

    for s in sections:
        tidy(s)
    return meta, sections


def tidy(sec):
    items = sec["items"]
    # A CTA held by only the last item belongs to the section.
    if len(items) > 1 and items[-1]["cta"] and not any(it["cta"] for it in items[:-1]):
        sec["ctas"].insert(0, items[-1]["cta"])
        items[-1]["cta"] = None
    # Trailing paragraphs beyond the other items' length are the section outro.
    if len(items) > 1:
        lens = [len(it["body"]) for it in items[:-1]]
        keep = max(lens)
        last = items[-1]
        if len(last["body"]) > keep and not last["list"]:
            sec["outro"] = last["body"][keep:] + sec["outro"]
            last["body"] = last["body"][:keep]
    sec["kind"] = kind_of(sec)
    if sec["kind"] == "steps":
        for it in items:
            m = re.match(r"^(\d+)\.\s*(.*)$", it["title"])
            it["num"], it["title"] = m.group(1), m.group(2)


def kind_of(sec):
    items = sec["items"]
    if not items:
        return "text"
    titles = [it["title"] for it in items]
    if all(re.match(r"^\d+\.\s", t) for t in titles):
        return "steps"
    if len(items) >= 3 and all(t.endswith("?") for t in titles):
        return "faq"
    if all(re.match(r"^[\d.,]+[%+x]?$", t) for t in titles):
        return "stats"
    if any("Phone:" in " ".join(it["body"]) for it in items):
        return "offices"
    if any(it["kicker"] for it in items):
        return "showcase"
    if all(it["cta"] for it in items):
        return "links"
    return "cards"


def slug_for(name):
    name = name.strip().replace("https_www.aigrowthtechnologies.com_", "")
    return name


def main(folder):
    OUT.mkdir(parents=True, exist_ok=True)
    for f in sorted(pathlib.Path(folder).rglob("*.docx")):
        slug = slug_for(f.stem)
        if not slug:
            continue  # homepage — hand-built
        meta, sections = structure(read_blocks(f))
        hero = sections[0]
        page = {
            "slug": slug,
            "path": "/" + slug.replace("_", "/"),
            "meta": {"title": meta.get("Title Tag"), "description": meta.get("Meta Description")},
            "hero": hero,
            "sections": sections[1:],
        }
        (OUT / f"{slug}.json").write_text(json.dumps(page, ensure_ascii=False, indent=2) + "\n")
        kinds = [s["kind"] for s in sections[1:]]
        print(f"{slug:32} {len(sections)-1:2} sections  {' '.join(kinds)}")


if __name__ == "__main__":
    main(sys.argv[1])