feat: Bild-Routing, Logo/Symbol/Fragment-Filter, Produktprogramm-Bilder
Bild-Verarbeitung der Anleitungen (lib/pdf_extract.py) erweitert:
- Routing nach Kapitel-Typ: generelle Kapitel (Steuerung) -> assets/shared/
images/<subdir>, projektbezogene (Betrieb/Wartung) -> projekte/fortna/images.
Fotos (.jpg) IMMER projektspezifisch, unabhaengig vom Kapitel.
- Benennung aus Bildunterschrift: <Stichwort>_NN.png (z.B. Anlage_Uebersicht_01);
Fotos img_NNN.jpg, SW-Skizzen >10KB sketch_NNN.png, Rest img_NNN.png.
- Logos: nur wiederkehrende (>=3x) breite Randgrafiken gelten als Logo -> alle
Aufloesungen nach assets/shared/logos/logo_<n>_<breite>px, nicht inline.
- Wertlose Fragmente (flach <=55px, breit, SW = gerenderte Kapitelnummern wie
"5.2.6") werden verworfen.
- Ziel-Bildordner werden vor jedem Lauf geleert.
extract_products.py: Produktprogramm-Bilder -> assets/shared/images/
produktprogramm/{omniflo,ils,cpc,geruest,trolley} (generelle Produktbilder).
.gitignore: extrahierte Bildmassen ausgeschlossen, nur kuratierte symbols/ +
logos/ versioniert.
Voller PDF-Build sauber (lualatex exit 0, 589 Seiten, 638 Bild-Refs, 0 fehlend).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Stangl
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
bbfe17ec37
commit
3ebfe06139
+227
-41
@@ -63,6 +63,66 @@ def hamming(a: int, b: int) -> int:
|
||||
"""Hamming-Distanz zweier Integer-Hashes (Anzahl unterschiedlicher Bits)."""
|
||||
return bin(a ^ b).count("1")
|
||||
|
||||
|
||||
# --- Bild-Klassifizierung -------------------------------------------------
|
||||
|
||||
SKETCH_MIN_BYTES = 10_000 # SW-Grafik groesser als ~10 KB gilt als Sketch
|
||||
|
||||
|
||||
def is_junk_fragment(image_bytes: bytes, w: int, h: int) -> bool:
|
||||
"""True fuer wertlose Text-/Nummern-Fragmente (kein echtes Symbol/Bild).
|
||||
|
||||
Typisch: flache (Hoehe <= 55 px), breite, schwarz-weisse Schnipsel, die in
|
||||
Wahrheit gerenderte Kapitelnummern/Beschriftungen sind (z.B. "5.2.6",
|
||||
zwei Quadrate + Zahlen). Diese werden verworfen.
|
||||
"""
|
||||
if h <= 55 and w / max(h, 1) >= 1.6 and is_grayscale(image_bytes):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_grayscale(image_bytes: bytes) -> bool:
|
||||
"""True, wenn das Bild (nahezu) schwarz-weiss/graustufig ist."""
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
rgb = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||
data = list(rgb.getdata())
|
||||
step = max(1, len(data) // 200)
|
||||
for r, g, b in data[::step]:
|
||||
if abs(r - g) > 12 or abs(g - b) > 12:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def classify_image(image_bytes: bytes, ext: str, w: int, h: int) -> str:
|
||||
"""Klassifiziert ein Bild: 'photo' | 'sketch' | 'graphic'.
|
||||
|
||||
- photo: JPEG/JPG (Fotos sind meist projektspezifisch)
|
||||
- sketch: schwarz-weisse Grafik groesser als ~10 KB (Skizze/Zeichnung)
|
||||
- graphic: alles uebrige (farbige PNG-Grafik, Diagramm)
|
||||
"""
|
||||
if ext.lower() in ("jpg", "jpeg"):
|
||||
return "photo"
|
||||
if len(image_bytes) > SKETCH_MIN_BYTES and is_grayscale(image_bytes):
|
||||
return "sketch"
|
||||
return "graphic"
|
||||
|
||||
|
||||
# Logo-Erkennung: grosse, breite Grafik (Schriftzug-Logo) auf oberen/unteren
|
||||
# Seitenbereichen. Wir erkennen sie ueber Seitenposition + Seitenverhaeltnis,
|
||||
# da OCR des Schriftzugs zu aufwendig waere.
|
||||
def looks_like_logo(w: int, h: int, y_top: float, page_height: float) -> bool:
|
||||
"""Heuristik: breites Bild (ratio > 2.2) im Kopf-/Fussbereich der Seite."""
|
||||
if h == 0:
|
||||
return False
|
||||
ratio = w / h
|
||||
in_margin = y_top < page_height * 0.18 or y_top > page_height * 0.82
|
||||
return ratio > 2.2 and in_margin and w >= 200
|
||||
|
||||
|
||||
RST_H = {1: "=", 2: "-", 3: "~", 4: "^", 5: '"'}
|
||||
|
||||
# Symbolfont-Zeichen (Private Use Area) auf Unicode abbilden bzw. entfernen.
|
||||
@@ -114,20 +174,51 @@ def _strip_heading_number(num: str, rest: str) -> str:
|
||||
return rest.strip()
|
||||
|
||||
|
||||
def caption_slug(caption_text: str) -> str:
|
||||
"""Leitet ein kurzes Datei-Stichwort aus einer Bildunterschrift ab.
|
||||
|
||||
"Abbildung 17: Weiche Typ A links" -> "Weiche_Typ_A"
|
||||
Nimmt die ersten Woerter nach der "Abbildung N:"-Kennung.
|
||||
"""
|
||||
m = RE_CAPTION.match(caption_text)
|
||||
body = (m.group(2) if m else caption_text).strip()
|
||||
# Umlaute/Sonderzeichen -> ASCII-nah, Wortgrenzen zu _
|
||||
body = (body.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue")
|
||||
.replace("Ä", "Ae").replace("Ö", "Oe").replace("Ü", "Ue")
|
||||
.replace("ß", "ss"))
|
||||
words = re.findall(r"[A-Za-z0-9]+", body)
|
||||
if not words:
|
||||
return ""
|
||||
slug = "_".join(words[:3]) # max. 3 Woerter fuer kompakten Namen
|
||||
return slug[:40]
|
||||
|
||||
|
||||
def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
title: str | None = None,
|
||||
symbols_dir: Path | None = None,
|
||||
symbol_registry: dict | None = None) -> tuple[int, int]:
|
||||
symbol_registry: dict | None = None,
|
||||
logos_dir: Path | None = None,
|
||||
logo_registry: dict | None = None,
|
||||
image_dest: Path | None = None,
|
||||
photo_dest: Path | None = None) -> tuple[int, int]:
|
||||
"""Extrahiert Text + Bilder eines Anleitungs-PDF in eine .rst-Datei.
|
||||
|
||||
Symbole/Icons (kleine, mehrfach vorkommende Bilder wie Warndreiecke) werden
|
||||
NICHT pro Vorkommen dupliziert, sondern einmal zentral nach symbols_dir
|
||||
(assets/shared/symbols) als symbol_<hash>.png abgelegt und im RST als
|
||||
`.. image::` mit relativem Pfad referenziert. symbol_registry (dict
|
||||
visual_hash -> relativer Pfad) wird dokumentuebergreifend wiederverwendet.
|
||||
Bild-Routing (siehe doc/plan_fortna_extraktion.md):
|
||||
|
||||
Inhaltsbild-Duplikate (gleicher visueller Hash) innerhalb des Dokuments
|
||||
werden ebenfalls nur einmal abgelegt und mehrfach referenziert.
|
||||
- Symbole (klein, mehrfach): einmal zentral -> symbols_dir
|
||||
(assets/shared/symbols), im RST als `.. image::` referenziert.
|
||||
- Logos (breite Schriftzug-Grafik im Seitenrand): pro visueller Variante
|
||||
alle Aufloesungen -> logos_dir (assets/shared/logos), NICHT inline
|
||||
referenziert (reine Deko).
|
||||
- Inhaltsbilder: nach image_dest (assets/... fuer generelle Kapitel bzw.
|
||||
projekte/fortna/images/... fuer projektspezifische). Fotos (.jpg) landen
|
||||
IMMER projektspezifisch; das steuert der Aufrufer ueber image_dest.
|
||||
|
||||
Benennung: PNG mit direkt folgender Bildunterschrift -> <Stichwort>_NN.png,
|
||||
Fotos -> img_NNN.jpg, SW-Skizzen (>10KB) -> sketch_NNN.png, Rest img_NNN.png.
|
||||
|
||||
symbol_registry / logo_registry (aHash -> Datei) werden dokumentuebergreifend
|
||||
wiederverwendet.
|
||||
|
||||
Returns:
|
||||
(anzahl_textbloecke, anzahl_bilder)
|
||||
@@ -136,9 +227,17 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
|
||||
if symbol_registry is None:
|
||||
symbol_registry = {}
|
||||
if logo_registry is None:
|
||||
logo_registry = {}
|
||||
# image_dest: Zielordner fuer Grafiken/Skizzen. photo_dest: fuer Fotos (.jpg,
|
||||
# immer projektspezifisch). Default beider: img_dir.
|
||||
if image_dest is None:
|
||||
image_dest = img_dir
|
||||
if photo_dest is None:
|
||||
photo_dest = image_dest
|
||||
|
||||
doc = fitz.open(str(pdf_path))
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
image_dest.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# --- Seiten identifizieren, die uebersprungen werden ---
|
||||
# a) TOC-Seiten (Punktfueller / "Inhaltsverzeichnis")
|
||||
@@ -163,7 +262,7 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
# in anderer Skalierung) landen im selben Cluster. Erst nach dem Scan steht
|
||||
# fest, welches Cluster mehrfach vorkommt (Symbol-Kriterium).
|
||||
elements: list[dict] = []
|
||||
clusters: list[dict] = [] # [{ahash, data, ext, w, h, count}]
|
||||
clusters: list[dict] = [] # [{ahash, data, ext, w, h, count, cls, margin_hits, variants}]
|
||||
|
||||
def _find_cluster(ah: int) -> int:
|
||||
for ci, c in enumerate(clusters):
|
||||
@@ -225,45 +324,81 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
data = base["image"]
|
||||
if len(data) < 3000:
|
||||
continue
|
||||
# Wertlose Text-/Nummern-Fragmente (flach, SW) verwerfen
|
||||
if is_junk_fragment(data, w, h):
|
||||
continue
|
||||
ext = base.get("ext", "png")
|
||||
ah = average_hash(data)
|
||||
in_margin = looks_like_logo(w, h, b[1], ph)
|
||||
ci = _find_cluster(ah)
|
||||
if ci < 0:
|
||||
clusters.append({"ahash": ah, "data": data,
|
||||
"ext": base.get("ext", "png"),
|
||||
"w": w, "h": h, "count": 1})
|
||||
clusters.append({"ahash": ah, "data": data, "ext": ext,
|
||||
"w": w, "h": h, "count": 1,
|
||||
"cls": classify_image(data, ext, w, h),
|
||||
"margin_hits": 1 if in_margin else 0,
|
||||
# groesste Auflvariante fuer Logo-Ablage merken
|
||||
"variants": [(w * h, w, h, data, ext)]})
|
||||
ci = len(clusters) - 1
|
||||
else:
|
||||
clusters[ci]["count"] += 1
|
||||
if in_margin:
|
||||
clusters[ci]["margin_hits"] += 1
|
||||
clusters[ci]["variants"].append((w * h, w, h, data, ext))
|
||||
elements.append({
|
||||
"kind": "image", "y": b[1], "page": pn + 1, "cluster": ci,
|
||||
})
|
||||
|
||||
doc.close()
|
||||
|
||||
# --- Pass 2: entscheiden, wohin jedes Bild-Cluster geschrieben wird ---
|
||||
# Symbol: klein UND mehrfach vorkommend -> zentral assets/shared/symbols.
|
||||
# Sonst (inkl. mehrfach genutzte Inhaltsbilder): einmal in Kapitel-images.
|
||||
import os as _os
|
||||
|
||||
# --- Zielverzeichnisse + relative Pfad-Praefixe (vom RST aus) ---
|
||||
rst_parent = rst_path.parent
|
||||
if symbols_dir is None:
|
||||
symbols_dir = img_dir.parent.parent.parent / "assets" / "shared" / "symbols"
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
symbols_dir = rst_parent.parent.parent / "assets" / "shared" / "symbols"
|
||||
if logos_dir is None:
|
||||
logos_dir = symbols_dir.parent / "logos"
|
||||
|
||||
# relativer Pfad-Praefix vom RST (liegt in img_dir.parent) zu symbols_dir
|
||||
try:
|
||||
sym_rel_prefix = Path(
|
||||
__import__("os").path.relpath(symbols_dir, img_dir.parent)
|
||||
).as_posix()
|
||||
except ValueError:
|
||||
sym_rel_prefix = symbols_dir.as_posix()
|
||||
def _rel(target: Path) -> str:
|
||||
try:
|
||||
return Path(_os.path.relpath(target, rst_parent)).as_posix()
|
||||
except ValueError:
|
||||
return target.as_posix()
|
||||
|
||||
cluster_target: list[str] = [""] * len(clusters) # ci -> RST-Bildpfad
|
||||
sym_rel_prefix = _rel(symbols_dir)
|
||||
dest_rel_prefix = _rel(image_dest)
|
||||
photo_rel_prefix = _rel(photo_dest)
|
||||
|
||||
# --- Pass 2a: Cluster klassifizieren (Logo / Symbol / Inhaltsbild) ---
|
||||
# Logo: breit (ratio > 2.2), im Seitenrand UND wiederkehrend (>=3x) - das
|
||||
# unterscheidet ein Kopf-/Fusszeilen-Logo von einem einmaligen breiten
|
||||
# Diagramm. Alle Aufloesungen -> assets/shared/logos, NICHT inline.
|
||||
# Symbol: klein (<=260px) UND mehrfach (>=2x) -> assets/shared/symbols.
|
||||
cluster_is_symbol: list[bool] = [False] * len(clusters)
|
||||
local_counter = 0
|
||||
cluster_is_logo: list[bool] = [False] * len(clusters)
|
||||
cluster_sympath: list[str] = [""] * len(clusters)
|
||||
|
||||
for ci, c in enumerate(clusters):
|
||||
ratio = c["w"] / c["h"] if c["h"] else 0
|
||||
is_logo = (ratio > 2.2 and c["margin_hits"] >= 3 and c["count"] >= 3
|
||||
and c["w"] >= 200)
|
||||
if is_logo:
|
||||
cluster_is_logo[ci] = True
|
||||
# bereits bekannte Logo-Variante? -> nicht neu schreiben
|
||||
if any(hamming(c["ahash"], rah) <= AHASH_THRESHOLD + 2
|
||||
for rah in logo_registry):
|
||||
continue
|
||||
logos_dir.mkdir(parents=True, exist_ok=True)
|
||||
idx = len(logo_registry) + 1
|
||||
for (_area, w, h, data, ext) in sorted(set(c["variants"]), reverse=True):
|
||||
(logos_dir / f"logo_{idx:02d}_{w}px.{ext}").write_bytes(data)
|
||||
logo_registry[c["ahash"]] = idx
|
||||
continue
|
||||
|
||||
is_symbol = (max(c["w"], c["h"]) <= SYMBOL_MAX_EDGE
|
||||
and c["count"] >= SYMBOL_MIN_COUNT)
|
||||
cluster_is_symbol[ci] = is_symbol
|
||||
if is_symbol:
|
||||
# Dokumentuebergreifende Registry per aHash-Naehe durchsuchen.
|
||||
match = None
|
||||
for reg_ah, reg_file in symbol_registry.items():
|
||||
if hamming(c["ahash"], reg_ah) <= AHASH_THRESHOLD:
|
||||
@@ -274,19 +409,7 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
match = f"symbol_{c['ahash'] & 0xffffffff:08x}.{c['ext']}"
|
||||
(symbols_dir / match).write_bytes(c["data"])
|
||||
symbol_registry[c["ahash"]] = match
|
||||
cluster_target[ci] = f"{sym_rel_prefix}/{match}"
|
||||
else:
|
||||
local_counter += 1
|
||||
fname = f"img_{local_counter:03d}.{c['ext']}"
|
||||
(img_dir / fname).write_bytes(c["data"])
|
||||
cluster_target[ci] = f"images/{fname}"
|
||||
|
||||
# Bild-Elemente mit Zielpfad versehen
|
||||
for e in elements:
|
||||
if e["kind"] == "image":
|
||||
e["path"] = cluster_target[e["cluster"]]
|
||||
e["is_symbol"] = cluster_is_symbol[e["cluster"]]
|
||||
img_counter = local_counter
|
||||
cluster_sympath[ci] = f"{sym_rel_prefix}/{match}"
|
||||
|
||||
# --- pro Seite nach Y-Position sortieren (Lesereihenfolge) ---
|
||||
elements.sort(key=lambda e: (e["page"], e["y"]))
|
||||
@@ -316,6 +439,69 @@ def extract_manual(pdf_path: Path, rst_path: Path, img_dir: Path,
|
||||
merged.append(e)
|
||||
i += 1
|
||||
|
||||
# --- Pass 2b: Inhaltsbilder schreiben (mit Caption-Benennung + Klasse) ---
|
||||
# Jetzt (nach dem Merge) ist die Folgezeile jedes Bildes bekannt -> Caption.
|
||||
# Pro Cluster genau einmal schreiben; Zielname aus Caption oder Klasse.
|
||||
# Fotos (.jpg) -> photo_dest (projektspezifisch), Rest -> image_dest.
|
||||
image_dest.mkdir(parents=True, exist_ok=True)
|
||||
photo_dest.mkdir(parents=True, exist_ok=True)
|
||||
counters = {"img": 0, "sketch": 0}
|
||||
used_names: set[str] = set()
|
||||
slug_counters: dict[str, int] = {}
|
||||
cluster_path: dict[int, str] = {} # ci -> RST-Pfad (relativ)
|
||||
|
||||
def _next_name(cls: str, ext: str, slug: str) -> str:
|
||||
if slug:
|
||||
slug_counters[slug] = slug_counters.get(slug, 0) + 1
|
||||
name = f"{slug}_{slug_counters[slug]:02d}.{ext}"
|
||||
elif cls == "sketch":
|
||||
counters["sketch"] += 1
|
||||
name = f"sketch_{counters['sketch']:03d}.{ext}"
|
||||
else: # photo / graphic
|
||||
counters["img"] += 1
|
||||
name = f"img_{counters['img']:03d}.{ext}"
|
||||
while name in used_names:
|
||||
counters["img"] += 1
|
||||
name = f"img_{counters['img']:03d}.{ext}"
|
||||
used_names.add(name)
|
||||
return name
|
||||
|
||||
n_m = len(merged)
|
||||
for idx, e in enumerate(merged):
|
||||
if e["kind"] != "image":
|
||||
continue
|
||||
ci = e["cluster"]
|
||||
if cluster_is_logo[ci]:
|
||||
e["path"] = "" # Logos werden nicht inline referenziert
|
||||
e["is_symbol"] = False
|
||||
continue
|
||||
if cluster_is_symbol[ci]:
|
||||
e["path"] = cluster_sympath[ci]
|
||||
e["is_symbol"] = True
|
||||
continue
|
||||
e["is_symbol"] = False
|
||||
if ci in cluster_path: # Duplikat -> bereits geschrieben
|
||||
e["path"] = cluster_path[ci]
|
||||
continue
|
||||
c = clusters[ci]
|
||||
# Caption der Folgezeile? -> Stichwort fuer Benennung
|
||||
slug = ""
|
||||
if idx + 1 < n_m and merged[idx + 1]["kind"] == "text":
|
||||
cm = RE_CAPTION.match(merged[idx + 1]["text"])
|
||||
if cm:
|
||||
slug = caption_slug(merged[idx + 1]["text"])
|
||||
name = _next_name(c["cls"], c["ext"], slug)
|
||||
# Fotos immer projektspezifisch (photo_dest), sonst image_dest
|
||||
if c["cls"] == "photo":
|
||||
(photo_dest / name).write_bytes(c["data"])
|
||||
rel = f"{photo_rel_prefix}/{name}"
|
||||
else:
|
||||
(image_dest / name).write_bytes(c["data"])
|
||||
rel = f"{dest_rel_prefix}/{name}"
|
||||
cluster_path[ci] = rel
|
||||
e["path"] = rel
|
||||
img_counter = len(cluster_path)
|
||||
|
||||
# --- RST rendern ---
|
||||
if title is None:
|
||||
title = pdf_path.stem
|
||||
|
||||
Reference in New Issue
Block a user