diff --git a/assets/shared/symbols/img_022.png b/assets/shared/symbols/img_022.png deleted file mode 100644 index 7d3605c..0000000 Binary files a/assets/shared/symbols/img_022.png and /dev/null differ diff --git a/assets/shared/symbols/img_023.png b/assets/shared/symbols/img_023.png deleted file mode 100644 index 0a05b28..0000000 Binary files a/assets/shared/symbols/img_023.png and /dev/null differ diff --git a/assets/shared/symbols/img_024.png b/assets/shared/symbols/img_024.png deleted file mode 100644 index a9f2359..0000000 Binary files a/assets/shared/symbols/img_024.png and /dev/null differ diff --git a/assets/shared/symbols/img_025.png b/assets/shared/symbols/img_025.png deleted file mode 100644 index 4bfa3bc..0000000 Binary files a/assets/shared/symbols/img_025.png and /dev/null differ diff --git a/assets/shared/symbols/img_026.png b/assets/shared/symbols/img_026.png deleted file mode 100644 index 9a992f9..0000000 Binary files a/assets/shared/symbols/img_026.png and /dev/null differ diff --git a/assets/shared/symbols/img_027.png b/assets/shared/symbols/img_027.png deleted file mode 100644 index a1e2ec6..0000000 Binary files a/assets/shared/symbols/img_027.png and /dev/null differ diff --git a/assets/shared/symbols/img_032.png b/assets/shared/symbols/img_032.png deleted file mode 100644 index 2b5a238..0000000 Binary files a/assets/shared/symbols/img_032.png and /dev/null differ diff --git a/assets/shared/symbols/img_039.png b/assets/shared/symbols/img_039.png deleted file mode 100644 index 775de91..0000000 Binary files a/assets/shared/symbols/img_039.png and /dev/null differ diff --git a/assets/shared/symbols/img_040.png b/assets/shared/symbols/img_040.png deleted file mode 100644 index e757a53..0000000 Binary files a/assets/shared/symbols/img_040.png and /dev/null differ diff --git a/assets/shared/symbols/img_065.png b/assets/shared/symbols/img_065.png deleted file mode 100644 index 0ec6dc2..0000000 Binary files a/assets/shared/symbols/img_065.png and /dev/null differ diff --git a/assets/shared/symbols/img_127.png b/assets/shared/symbols/img_127.png deleted file mode 100644 index 580ae0c..0000000 Binary files a/assets/shared/symbols/img_127.png and /dev/null differ diff --git a/assets/shared/symbols/img_164.png b/assets/shared/symbols/img_164.png deleted file mode 100644 index b88069a..0000000 Binary files a/assets/shared/symbols/img_164.png and /dev/null differ diff --git a/assets/shared/symbols/img_302.png b/assets/shared/symbols/img_302.png deleted file mode 100644 index 6e248cc..0000000 Binary files a/assets/shared/symbols/img_302.png and /dev/null differ diff --git a/assets/shared/symbols/img_308.png b/assets/shared/symbols/img_308.png deleted file mode 100644 index 9c26fb4..0000000 Binary files a/assets/shared/symbols/img_308.png and /dev/null differ diff --git a/assets/shared/symbols/symbol_0c0099c3.jpeg b/assets/shared/symbols/symbol_0c0099c3.jpeg new file mode 100644 index 0000000..b237b8e Binary files /dev/null and b/assets/shared/symbols/symbol_0c0099c3.jpeg differ diff --git a/assets/shared/symbols/symbol_2c99c3e7.jpeg b/assets/shared/symbols/symbol_2c99c3e7.jpeg new file mode 100644 index 0000000..fa0d1d2 Binary files /dev/null and b/assets/shared/symbols/symbol_2c99c3e7.jpeg differ diff --git a/assets/shared/symbols/symbol_2cae247c.png b/assets/shared/symbols/symbol_2cae247c.png new file mode 100644 index 0000000..212aa89 Binary files /dev/null and b/assets/shared/symbols/symbol_2cae247c.png differ diff --git a/assets/shared/symbols/symbol_36161676.png b/assets/shared/symbols/symbol_36161676.png new file mode 100644 index 0000000..498dd30 Binary files /dev/null and b/assets/shared/symbols/symbol_36161676.png differ diff --git a/assets/shared/symbols/symbol_4262b5c3.jpeg b/assets/shared/symbols/symbol_4262b5c3.jpeg new file mode 100644 index 0000000..153dbe3 Binary files /dev/null and b/assets/shared/symbols/symbol_4262b5c3.jpeg differ diff --git a/assets/shared/symbols/symbol_464abdc3.png b/assets/shared/symbols/symbol_464abdc3.png new file mode 100644 index 0000000..ac6afde Binary files /dev/null and b/assets/shared/symbols/symbol_464abdc3.png differ diff --git a/assets/shared/symbols/symbol_6d6d2466.png b/assets/shared/symbols/symbol_6d6d2466.png new file mode 100644 index 0000000..1a72b5d Binary files /dev/null and b/assets/shared/symbols/symbol_6d6d2466.png differ diff --git a/assets/shared/symbols/symbol_71c70f3f.jpeg b/assets/shared/symbols/symbol_71c70f3f.jpeg new file mode 100644 index 0000000..5b38396 Binary files /dev/null and b/assets/shared/symbols/symbol_71c70f3f.jpeg differ diff --git a/assets/shared/symbols/symbol_818183ff.png b/assets/shared/symbols/symbol_818183ff.png new file mode 100644 index 0000000..d8307d9 Binary files /dev/null and b/assets/shared/symbols/symbol_818183ff.png differ diff --git a/assets/shared/symbols/symbol_c3c3e7e7.png b/assets/shared/symbols/symbol_c3c3e7e7.png new file mode 100644 index 0000000..ed4357a Binary files /dev/null and b/assets/shared/symbols/symbol_c3c3e7e7.png differ diff --git a/assets/shared/symbols/symbol_c3e7e7ff.png b/assets/shared/symbols/symbol_c3e7e7ff.png new file mode 100644 index 0000000..02fe3b0 Binary files /dev/null and b/assets/shared/symbols/symbol_c3e7e7ff.png differ diff --git a/assets/shared/symbols/symbol_dbc3e7e7.png b/assets/shared/symbols/symbol_dbc3e7e7.png new file mode 100644 index 0000000..2b6fe80 Binary files /dev/null and b/assets/shared/symbols/symbol_dbc3e7e7.png differ diff --git a/assets/shared/symbols/symbol_e0e0f0f3.jpeg b/assets/shared/symbols/symbol_e0e0f0f3.jpeg new file mode 100644 index 0000000..96bed63 Binary files /dev/null and b/assets/shared/symbols/symbol_e0e0f0f3.jpeg differ diff --git a/assets/shared/symbols/symbol_e7ffffe7.png b/assets/shared/symbols/symbol_e7ffffe7.png new file mode 100644 index 0000000..4a37e99 Binary files /dev/null and b/assets/shared/symbols/symbol_e7ffffe7.png differ diff --git a/doc/plan_fortna_extraktion.md b/doc/plan_fortna_extraktion.md index 3d50d1a..6ebaca0 100644 --- a/doc/plan_fortna_extraktion.md +++ b/doc/plan_fortna_extraktion.md @@ -50,6 +50,15 @@ sauberere `.rst` erzeugt: vorhanden – der folgenden „Abbildung N: …"-Zeile als Caption eingebaut. (625 figure-Refs, 0 fehlende Bilddateien.) Duplikate werden per MD5 entfernt, Titelseiten-Deko und Vollseiten-Hintergründe herausgefiltert. +- **Symbole/Warnzeichen zentral:** Kleine (≤ 260 px), mehrfach vorkommende Bilder (Warndreiecke, + Piktogramme) werden per Perceptual-Hash (aHash + Hamming-Distanz ≤ 6) erkannt, **einmal** zentral + nach `assets/shared/symbols/symbol_.png` abgelegt und im RST als `.. image::` (12 mm, + zentriert) auf die eine Quelle verwiesen – dokumentübergreifend. Vorher lag dasselbe Warndreieck + z. B. 8× als `img_003/018/020/025/026/031/035…` in einem einzigen Kapitel. Auch großformatige + Inhaltsbild-Duplikate werden über den aHash zusammengeführt (nur einmal abgelegt, mehrfach + referenziert). Wirkung: omniflo-`images/` von 83 → 62, Warndreieck 1× statt 58× gespeichert. +- **Fragment-„Überschriften" unterdrückt:** Bildlegenden/Tabellenzellen mit führender Nummer + („POS", „li.", „re.", „bar, max. 10 bar") werden nicht mehr als Überschrift gesetzt. - **Fließtext zu Absätzen zusammengeführt:** PDF-Zeilenumbrüche werden nicht mehr zu je einem Absatz; zusammenhängende Zeilen bilden einen Absatz, Satzende beendet ihn. - **Aufzählungen als RST-Listen:** einzeln stehende Bullet-Zeichen werden mit der Folgezeile diff --git a/lib/fortna_convert_chapters.py b/lib/fortna_convert_chapters.py index cf116c6..d20d0f3 100644 --- a/lib/fortna_convert_chapters.py +++ b/lib/fortna_convert_chapters.py @@ -87,7 +87,8 @@ CHAPTER_DOC_TITLE = { # --------------------------------------------------------------------------- def convert_manual(item: dict, source_root: Path, chapters_dir: Path, - dry_run: bool) -> None: + dry_run: bool, symbols_dir: Path | None = None, + symbol_registry: dict | None = None) -> None: """Konvertiert eine Hauptanleitung nach chapters//.""" src_pdf = source_root / item["rel_path"] chapter_dir = chapters_dir / item["chapter"] @@ -118,8 +119,9 @@ def convert_manual(item: dict, source_root: Path, chapters_dir: Path, # Text + Bilder positionsbasiert extrahieren (verwoben in Lesereihenfolge, # PDF-Ueberschriftennummern entfernt, Bilder als .. figure:: referenziert). doc_title = CHAPTER_DOC_TITLE.get(item["chapter"], slugify(item["filename"])) - blocks, img_count = pdf_extract.extract_manual(src_pdf, rst_path, img_dir, - title=doc_title) + blocks, img_count = pdf_extract.extract_manual( + src_pdf, rst_path, img_dir, title=doc_title, + symbols_dir=symbols_dir, symbol_registry=symbol_registry) log.info(" %d Textbloecke, %d Bilder (referenziert) -> %s", blocks, img_count, rst_path.name) @@ -268,8 +270,17 @@ def main() -> None: if args.only in (None, "convert"): conv = [i for i in items if i["action"] == "convert"] log.info("--- %d Hauptanleitungen konvertieren ---", len(conv)) + # Symbole dokumentuebergreifend zentral in assets/shared/symbols/ ablegen + # und nur einmal referenzieren (Warndreiecke etc. tauchen in mehreren + # Anleitungen auf). Registry ueber alle Dokumente hinweg teilen. + symbols_dir = chapters_dir.parent / "assets" / "shared" / "symbols" + symbol_registry: dict = {} for it in conv: - convert_manual(it, source_root, chapters_dir, dry_run) + convert_manual(it, source_root, chapters_dir, dry_run, + symbols_dir=symbols_dir, symbol_registry=symbol_registry) + if not dry_run: + log.info("Symbole zentral: %d eindeutige -> %s", + len(symbol_registry), symbols_dir) # 2. Verzeichnislisten if args.only in (None, "list"): diff --git a/lib/pdf_extract.py b/lib/pdf_extract.py index b1e07f8..74f89e9 100644 --- a/lib/pdf_extract.py +++ b/lib/pdf_extract.py @@ -22,12 +22,47 @@ Oeffentliche Funktion: extract_manual(pdf_path, rst_path, img_dir) -> (blocks, i from __future__ import annotations import hashlib +import io import logging import re from pathlib import Path log = logging.getLogger(__name__) +# Schwellwerte fuer die Symbol-/Icon-Erkennung +SYMBOL_MAX_EDGE = 260 # Bilder <= dieser Kantenlaenge gelten als Symbol-Kandidat +SYMBOL_MIN_COUNT = 2 # ... wenn dasselbe Bild mind. so oft vorkommt +AHASH_THRESHOLD = 6 # max. Hamming-Distanz, ab der zwei aHashes als gleich gelten + + +def average_hash(image_bytes: bytes, size: int = 8) -> int: + """Perceptual Average-Hash (aHash) eines Bildes als Integer. + + Graustufen -> size x size -> Bit je Pixel (>= Mittelwert). Zwei visuell + aehnliche Bilder (auch bei leicht anderer Skalierung/Kompression, wie + mehrfach eingebettete Warnsymbole aus Word) haben eine kleine Hamming- + Distanz. Robuster als ein exakter Byte-/Pixel-Hash. + """ + from PIL import Image + + try: + im = Image.open(io.BytesIO(image_bytes)).convert("L").resize((size, size)) + px = list(im.getdata()) + avg = sum(px) / len(px) + bits = 0 + for i, p in enumerate(px): + if p >= avg: + bits |= (1 << i) + return bits + except Exception: + # Fallback: Byte-Hash als Integer, wenn PIL das Bild nicht oeffnen kann + return int(hashlib.md5(image_bytes).hexdigest()[:16], 16) + + +def hamming(a: int, b: int) -> int: + """Hamming-Distanz zweier Integer-Hashes (Anzahl unterschiedlicher Bits).""" + return bin(a ^ b).count("1") + RST_H = {1: "=", 2: "-", 3: "~", 4: "^", 5: '"'} # Symbolfont-Zeichen (Private Use Area) auf Unicode abbilden bzw. entfernen. @@ -80,14 +115,28 @@ def _strip_heading_number(num: str, rest: str) -> str: def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, - title: str | None = None) -> tuple[int, int]: + title: str | None = None, + symbols_dir: Path | None = None, + symbol_registry: dict | None = None) -> tuple[int, int]: """Extrahiert Text + Bilder eines Anleitungs-PDF in eine .rst-Datei. + Symbole/Icons (kleine, mehrfach vorkommende Bilder wie Warndreiecke) werden + NICHT pro Vorkommen dupliziert, sondern einmal zentral nach symbols_dir + (assets/shared/symbols) als symbol_.png abgelegt und im RST als + `.. image::` mit relativem Pfad referenziert. symbol_registry (dict + visual_hash -> relativer Pfad) wird dokumentuebergreifend wiederverwendet. + + Inhaltsbild-Duplikate (gleicher visueller Hash) innerhalb des Dokuments + werden ebenfalls nur einmal abgelegt und mehrfach referenziert. + Returns: (anzahl_textbloecke, anzahl_bilder) """ import fitz + if symbol_registry is None: + symbol_registry = {} + doc = fitz.open(str(pdf_path)) img_dir.mkdir(parents=True, exist_ok=True) @@ -108,11 +157,19 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, and ("datum" in low or "kapitel" in low)): skip_pages.add(pn) - # --- Elemente (Text + Bild) pro Seite in Lesereihenfolge sammeln --- - # element: {"kind": "text"|"image", "y": float, "page": int, ...} + # --- Pass 1: Text- und Bild-Elemente sammeln (Bilder nur puffern) --- + # Bilder werden zunaechst nicht geschrieben, sondern per Perceptual-Hash + # (aHash) einem Cluster zugeordnet. Visuell aehnliche Bilder (gleiches Symbol + # in anderer Skalierung) landen im selben Cluster. Erst nach dem Scan steht + # fest, welches Cluster mehrfach vorkommt (Symbol-Kriterium). elements: list[dict] = [] - seen_hashes: set[str] = set() - img_counter = 0 + clusters: list[dict] = [] # [{ahash, data, ext, w, h, count}] + + def _find_cluster(ah: int) -> int: + for ci, c in enumerate(clusters): + if hamming(ah, c["ahash"]) <= AHASH_THRESHOLD: + return ci + return -1 for pn in range(len(doc)): if pn in skip_pages: @@ -148,8 +205,7 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, "text": text, "size": round(max_size, 1), "bold": bold, }) - # Bilder mit Position. Titelseiten (erste 2 Seiten) enthalten nur - # Logo-/Deko-Bilder -> ueberspringen. + # Bilder mit Position. Titelseiten (erste 2 Seiten) = nur Logo/Deko. if pn < 2: continue for im in page.get_image_info(xrefs=True): @@ -161,7 +217,7 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, # Vollseiten-Deko / Hintergrundbilder ueberspringen if (b[2] - b[0]) > pw * 0.95 and (b[3] - b[1]) > ph * 0.95: continue - if w < 40 or h < 40: # Icons + if w < 40 or h < 40: # winzige Icons continue base = doc.extract_image(xref) if not base: @@ -169,20 +225,69 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, data = base["image"] if len(data) < 3000: continue - md5 = hashlib.md5(data).hexdigest() - if md5 in seen_hashes: - continue # exaktes Duplikat (wiederkehrendes Logo/Symbol) - seen_hashes.add(md5) - img_counter += 1 - ext = base.get("ext", "png") - fname = f"img_{img_counter:03d}.{ext}" - (img_dir / fname).write_bytes(data) + ah = average_hash(data) + ci = _find_cluster(ah) + if ci < 0: + clusters.append({"ahash": ah, "data": data, + "ext": base.get("ext", "png"), + "w": w, "h": h, "count": 1}) + ci = len(clusters) - 1 + else: + clusters[ci]["count"] += 1 elements.append({ - "kind": "image", "y": b[1], "page": pn + 1, "file": fname, + "kind": "image", "y": b[1], "page": pn + 1, "cluster": ci, }) doc.close() + # --- Pass 2: entscheiden, wohin jedes Bild-Cluster geschrieben wird --- + # Symbol: klein UND mehrfach vorkommend -> zentral assets/shared/symbols. + # Sonst (inkl. mehrfach genutzte Inhaltsbilder): einmal in Kapitel-images. + if symbols_dir is None: + symbols_dir = img_dir.parent.parent.parent / "assets" / "shared" / "symbols" + img_dir.mkdir(parents=True, exist_ok=True) + + # relativer Pfad-Praefix vom RST (liegt in img_dir.parent) zu symbols_dir + try: + sym_rel_prefix = Path( + __import__("os").path.relpath(symbols_dir, img_dir.parent) + ).as_posix() + except ValueError: + sym_rel_prefix = symbols_dir.as_posix() + + cluster_target: list[str] = [""] * len(clusters) # ci -> RST-Bildpfad + cluster_is_symbol: list[bool] = [False] * len(clusters) + local_counter = 0 + for ci, c in enumerate(clusters): + is_symbol = (max(c["w"], c["h"]) <= SYMBOL_MAX_EDGE + and c["count"] >= SYMBOL_MIN_COUNT) + cluster_is_symbol[ci] = is_symbol + if is_symbol: + # Dokumentuebergreifende Registry per aHash-Naehe durchsuchen. + match = None + for reg_ah, reg_file in symbol_registry.items(): + if hamming(c["ahash"], reg_ah) <= AHASH_THRESHOLD: + match = reg_file + break + if match is None: + symbols_dir.mkdir(parents=True, exist_ok=True) + match = f"symbol_{c['ahash'] & 0xffffffff:08x}.{c['ext']}" + (symbols_dir / match).write_bytes(c["data"]) + symbol_registry[c["ahash"]] = match + cluster_target[ci] = f"{sym_rel_prefix}/{match}" + else: + local_counter += 1 + fname = f"img_{local_counter:03d}.{c['ext']}" + (img_dir / fname).write_bytes(c["data"]) + cluster_target[ci] = f"images/{fname}" + + # Bild-Elemente mit Zielpfad versehen + for e in elements: + if e["kind"] == "image": + e["path"] = cluster_target[e["cluster"]] + e["is_symbol"] = cluster_is_symbol[e["cluster"]] + img_counter = local_counter + # --- pro Seite nach Y-Position sortieren (Lesereihenfolge) --- elements.sort(key=lambda e: (e["page"], e["y"])) @@ -233,6 +338,19 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, for idx, e in enumerate(merged): if e["kind"] == "image": flush_para() + path = e.get("path", "") + if not path: + continue + if e.get("is_symbol"): + # Symbol/Warnzeichen: kleine, zentrierte Grafik ohne Caption-Box, + # verweist auf die eine zentrale Quelle in assets/shared/symbols. + out.append("") + out.append(f".. image:: {path}") + out.append(" :align: center") + out.append(" :height: 12mm") + out.append("") + blocks += 1 + continue caption = "" if idx + 1 < n and merged[idx + 1]["kind"] == "text": cm = RE_CAPTION.match(merged[idx + 1]["text"]) @@ -240,7 +358,7 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, caption = merged[idx + 1]["text"] merged[idx + 1]["_consumed"] = True out.append("") - out.append(f".. figure:: images/{e['file']}") + out.append(f".. figure:: {path}") out.append(" :align: center") out.append(" :width: 80%") if caption: @@ -255,11 +373,22 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path, text = e["text"] hm = RE_HEADING.match(text) - # Ueberschrift NUR mit fuehrender PDF-Gliederungsnummer akzeptieren. + # Ueberschrift NUR mit fuehrender PDF-Gliederungsnummer akzeptieren und + # offensichtliche Fragmente/Bildlegenden ausschliessen (sonst werden + # Zellen wie "POS", "li.", "re.", "bar, max. 10 bar" zu Ueberschriften). is_heading = False if hm and len(hm.group(2)) < 120: num, rest = hm.group(1), hm.group(2) - if not re.match(r"^(mm|cm|m|kg|x|l|h|min|s)\b", rest, re.I): + rest_stripped = rest.strip() + is_frag = ( + len(rest_stripped) < 4 # zu kurz ("POS", "li.") + or rest_stripped.endswith((",",)) # abgeschnittene Zelle + or re.match(r"^(mm|cm|m|kg|x|l|h|min|s|bar|POS|li|re|Pos)\b", + rest_stripped, re.I) + or re.match(r"^(li|re|oben|unten|vorne|hinten|links|rechts)\.?$", + rest_stripped, re.I) + ) + if not is_frag: is_heading = True if is_heading: