webscan mit Playwright statt CutyCapt
- libs/webscan.py: Screenshots von Webseiten (Kundennummer;URL) mit Playwright/Chromium, parallel, Größe aus [webscan] der Konfiguration; der Zuschnitt mit convert entfällt - bin/webscan.sh ruft webscan.py auf (Python aus .venv, falls vorhanden) - bin/webscan und libs/CutyCapt (Qt4-WebKit, 2012) entfernt - requirements.txt, .gitignore, Abschnitt in doc/README.md Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Michael Stangl
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
31c8d97817
commit
7bd43e9e00
@@ -0,0 +1,3 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
-191
@@ -1,191 +0,0 @@
|
||||
#!/bin/bash
|
||||
NUM=0
|
||||
QUEUE=""
|
||||
MAX_NPROC=2 # default
|
||||
REPLACE_CMD=0 # no replacement by default
|
||||
|
||||
USAGE="A simple wrapper for generating jpg pictures out of svg-files in parallel.
|
||||
Usage: `basename $0` [-h] [-i] [-j nb_jobs] command arg_list
|
||||
-h Shows this help
|
||||
-c give the path to the cfg file
|
||||
-i give the file where the web-adresses are
|
||||
-k chek if the output file already exist
|
||||
-s set the sleeptime while there are too many simultanious jobs
|
||||
-j Set number of simultanious jobs. Default is 2
|
||||
-o directory where the files should be moved to after creation.
|
||||
Default ist PICCREATE_OUT
|
||||
Examples:
|
||||
`basename $0` -i file -c config
|
||||
`basename $0` -j 3 -i file -c config"
|
||||
|
||||
function cfg_parser {
|
||||
INI_FILE=$1
|
||||
SECTION=$2
|
||||
eval `sed -n -e "/^\[$SECTION\]/,/^\s*\[/{/^[^;].*\=.*/p;}" < $INI_FILE`
|
||||
}
|
||||
function get_filepath {
|
||||
# probiere alle möglichen Kombinationen wo das file stehen könnte je
|
||||
# nachdem ob es mit oder ohne Endung, mit oder ohne den Komplettpfad
|
||||
# gegeben wurde
|
||||
FILENAME=$1
|
||||
ENDING=$2
|
||||
DEFAULT_DIR=$3
|
||||
if [ -s $FILENAME ] ; then
|
||||
echo $FILENAME
|
||||
elif [ -s $FILENAME.$ENDING ] ; then
|
||||
echo "$FILENAME.$ENDING"
|
||||
elif [ -s $DEFAULT_DIR/$FILENAME ] ; then
|
||||
echo "$DEFAULT_DIR/$FILENAME"
|
||||
elif [ -s $DEFAULT_DIR/$FILENAME.$ENDING ] ; then
|
||||
echo "$DEFAULT_DIR/$FILENAME.$ENDING"
|
||||
fi
|
||||
return
|
||||
}
|
||||
|
||||
# Funktion die auf globalen Parametern beruht.
|
||||
# überprüft ob alle notwendigen Angaben vorhanden sind
|
||||
function check_params {
|
||||
if [ -z "$INPUT_FILENAME" ]; then
|
||||
echo "please give input file with -i."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$CFG_FILENAME" ]; then
|
||||
echo "please give config file with -c."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INPUT_FILE=`get_filepath $INPUT_FILENAME 'csv' $PICCREATE_WORK`
|
||||
if [ ! -s $INPUT_FILE ] ; then
|
||||
echo "-> input filename does not exist"
|
||||
print "given was: -$INPUT_FILENAME-"
|
||||
exit 1
|
||||
fi
|
||||
CFG_PATH=`get_filepath $CFG_FILENAME 'cfg' $PICCREATE_CFG`
|
||||
if [ ! -s $CFG_PATH ]; then
|
||||
echo "-> config file with this name does not exist."
|
||||
print "given was: -$CFG_FILENAME-"
|
||||
exit 1
|
||||
fi
|
||||
echo $INPUT_FILENAME
|
||||
cfg_parser $CFG_PATH "$FILE_FORMAT"
|
||||
echo 'using width '$width
|
||||
echo 'using height '$height
|
||||
if [ -z "$width" ]; then
|
||||
echo "please give width=xxx in section [$FILE_FORMAT] in config $CFG_FILENAME."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$height" ]; then
|
||||
echo "please give height=xxx in section [$FILE_FORMAT] in config $CFG_FILENAME."
|
||||
exit 1
|
||||
fi
|
||||
FORMAT="$format"
|
||||
}
|
||||
|
||||
|
||||
# parse command line
|
||||
if [ $# -eq 0 ]; then # must be at least one arg
|
||||
echo "$USAGE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CHECKFILE=1
|
||||
SLEEPTIME=0.5
|
||||
BACKGROUND=''
|
||||
FILE_FORMAT='webscan'
|
||||
OUTDIR=$PICCREATE_OUT
|
||||
while getopts "j:i:kc:s:o:hc:f" OPT; do # "j:" waits for an argument "h" doesnt
|
||||
case $OPT in
|
||||
h) echo "$USAGE"
|
||||
exit 0 ;;
|
||||
j) MAX_NPROC=$OPTARG ;;
|
||||
k) CHECKFILE=0 ;;
|
||||
c) CFG_FILENAME=$OPTARG ;;
|
||||
o) OUTDIR=$OPTARG ;;
|
||||
s) SLEEPTIME=$OPTARG ;;
|
||||
i) INPUT_FILENAME=$OPTARG ;;
|
||||
f) FORMAT=$OPTARG ;;
|
||||
*) echo "$USAGE" >&2; exit 1; # Default.
|
||||
esac
|
||||
done
|
||||
|
||||
# Main program
|
||||
echo Using $MAX_NPROC parallel threads
|
||||
check_params
|
||||
|
||||
# Starte die Abspeichern der ganzen Webseite als ganzes
|
||||
echo $INPUT_FILE
|
||||
for i in `cat $INPUT_FILENAME`
|
||||
do
|
||||
CUSTOMER_ID=`echo "$i" | sed -e 's/;.*//g'`
|
||||
CUSTOMER_URL=`echo "$i" |sed -e 's/.*;//g'`
|
||||
|
||||
FILE_PATH=$OUTDIR/$CUSTOMER_ID.$FORMAT
|
||||
echo $FILE_PATH
|
||||
wait
|
||||
if [ -z $CUSTOMER_URL ]
|
||||
then
|
||||
echo 'i need a url!!'
|
||||
echo $CUSTOMER_ID 'did not have an url, ist was set to www.syntops.de instead'>> logfile.txt
|
||||
CUSTOMER_URL='www.syntops.de'
|
||||
sleep
|
||||
fi
|
||||
if [ -s $FILE_PATH ] && [ $CHECKFILE -eq 1 ]
|
||||
then
|
||||
echo $FILE_PATH exists
|
||||
else
|
||||
CMDLINE="$CUTYCAPT/./CutyCapt --url=http://$CUSTOMER_URL --max-wait=20000 --min-width=800 --min-height=600 --out=$FILE_PATH"
|
||||
$CMDLINE
|
||||
wait
|
||||
fi
|
||||
done
|
||||
|
||||
wait # wait for all processes to finish before exit
|
||||
echo "...all Webpages have been saved!"
|
||||
echo ""
|
||||
echo "Converting the pictures..."
|
||||
echo " "
|
||||
|
||||
|
||||
# Zuschnitt der seite auf das benötigte Format
|
||||
# Das Bild wir hierfür erst gleichmäßig auf die forgegebene Breite Skaliert
|
||||
# und anschließend die unten herüberstehende Höhe abgeschnitten
|
||||
echo $outdir
|
||||
for i in `ls $OUTDIR| grep .$FORMAT `
|
||||
do
|
||||
FILE_PATH_IN=$OUTDIR/$i
|
||||
FILE_PATH_OUT=$OUTDIR/'beschnitten'/$i
|
||||
|
||||
if [ -s $FILE_PATH_OUT ] && [ $CHECKFILE -eq 1 ]
|
||||
then
|
||||
echo $PIC_NAME exists
|
||||
else
|
||||
RESIZE="${width}"
|
||||
SLICE="${width}x${height}"
|
||||
CMDLINE="convert $FILE_PATH_IN -resize $RESIZE -crop $SLICE $FILE_PATH_OUT"
|
||||
echo $CMDLINE
|
||||
(echo $ID; $CMDLINE) &
|
||||
|
||||
|
||||
while [ `ps clx|grep -w 'convert' |wc -l` -ge $MAX_NPROC ]; do
|
||||
sleep $SLEEPTIME
|
||||
done
|
||||
fi
|
||||
# auskommentiert da bereits runtergeladene seiten behalten werden sollen
|
||||
#rm $FILE_PATH
|
||||
|
||||
done
|
||||
|
||||
wait # wait for all processes to finish before exit
|
||||
|
||||
|
||||
rm $OUTDIR/'beschnitten'/*-[1-9].$FORMAT
|
||||
for i in `ls $OUTDIR/'beschnitten'/`
|
||||
do
|
||||
first=`basename $i -0.jpg`
|
||||
second=`basename $first .jpg`
|
||||
mv $OUTDIR/'beschnitten'/$i $OUTDIR/'beschnitten'/$second.jpg
|
||||
|
||||
done
|
||||
|
||||
|
||||
# vi:set wrapmargin=0 ai showmatch ts=4 syntax=sh:
|
||||
+7
-2
@@ -1,4 +1,9 @@
|
||||
#!/bin/bash
|
||||
. setenv.sh
|
||||
export CUTYCAPT=$PICCREATE_LIBS/CutyCapt
|
||||
$PICCREATE_BIN/webscan "$@"
|
||||
|
||||
# Vorschaubilder von Webseiten erzeugen (Playwright, Python-venv im Projekt)
|
||||
PYTHON=python3
|
||||
if [ -x $PICCREATE/.venv/bin/python ]; then
|
||||
PYTHON=$PICCREATE/.venv/bin/python
|
||||
fi
|
||||
$PYTHON $PICCREATE_LIBS/webscan.py "$@"
|
||||
|
||||
+51
-2
@@ -19,16 +19,17 @@ Die Punkte werden über das ganze Wort fortlaufend nummeriert (1–32).
|
||||
|
||||
## Voraussetzungen
|
||||
|
||||
- Python 3 (keine weiteren Python-Pakete nötig)
|
||||
- Python 3 (für `piccreate.py` und `svgconvert.py` keine weiteren Pakete nötig)
|
||||
- ImageMagick (`convert`) für die Umwandlung SVG → JPG/PNG mit `svgconvert.py`
|
||||
- `bash`, `bc`, `sed`, `grep` für die Font-Werkzeuge in `bin/`
|
||||
- nur für Webseiten-Vorschauen (`webscan`): Playwright, siehe unten
|
||||
|
||||
## Verzeichnisse
|
||||
|
||||
| Verzeichnis | Inhalt |
|
||||
|----------------|-----------------------------------------------------------|
|
||||
| `bin/` | Shell-Skripte (Aufruf, Umgebung, Font-Extraktion) |
|
||||
| `libs/` | Python-Programme `piccreate.py`, `svgconvert.py`, Hilfen |
|
||||
| `libs/` | Python-Programme `piccreate.py`, `svgconvert.py`, `webscan.py` |
|
||||
| `cfg/` | Auftrags-Konfigurationen (`*.cfg`) |
|
||||
| `in/` | Eingabedateien (`*.csv`) und Rohglyphen für neue Fonts |
|
||||
| `out/` | Ausgabe der erzeugten Bilder |
|
||||
@@ -143,6 +144,54 @@ ImageMagick um. `--format` wählt die Sektion aus der Konfiguration.
|
||||
| `-p` / `-v` | Fortschrittsbalken / ausführliche Ausgabe |
|
||||
| `-y` | Probelauf ohne Umwandlung |
|
||||
|
||||
## Vorschaubilder von Webseiten (webscan)
|
||||
|
||||
`libs/webscan.py` lädt für jede Zeile einer Eingabedatei eine Webseite in
|
||||
einem Browser ohne Fenster und speichert den sichtbaren Bereich als Bild
|
||||
`<Kundennummer>.jpg`. Die Größe kommt aus der Sektion `[webscan]` der
|
||||
Konfiguration (in `cfg/webscan.cfg`: 800×500).
|
||||
|
||||
Einmalig einrichten (Python-Umgebung im Projektverzeichnis):
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -r requirements.txt
|
||||
.venv/bin/playwright install chromium
|
||||
```
|
||||
|
||||
Statt des Playwright-Chromiums kann auch ein installiertes Chromium benutzt
|
||||
werden: `-b /usr/bin/chromium` oder `export PICCREATE_BROWSER=/usr/bin/chromium`.
|
||||
|
||||
Eingabedatei, z. B. `in/kunden.csv` (Kopfzeile optional, `http://` wird bei
|
||||
Bedarf ergänzt):
|
||||
|
||||
```
|
||||
Kennummer;URL
|
||||
1001;www.example.org
|
||||
1002;https://www.example.com/start
|
||||
```
|
||||
|
||||
Aufruf:
|
||||
|
||||
```sh
|
||||
./webscan.sh -c webscan -i kunden
|
||||
```
|
||||
|
||||
| Option | Bedeutung |
|
||||
|-----------------|---------------------------------------------------------|
|
||||
| `-c` | Konfiguration (Standard `webscan`) |
|
||||
| `-i` | Eingabedatei aus `in/` oder mit Pfad |
|
||||
| `-o` | Ausgabeverzeichnis (Standard `$PICCREATE_OUT`) |
|
||||
| `--format` | `jpg` oder `png` (überschreibt die Konfiguration) |
|
||||
| `-j` | Anzahl gleichzeitig geladener Seiten (Standard 4) |
|
||||
| `-t` | Wartezeit pro Seite in Sekunden (Standard 20) |
|
||||
| `-f` | vorhandene Bilder überschreiben |
|
||||
| `-b` | Pfad zu einem eigenen Chromium |
|
||||
| `-v` | ausführliche Ausgabe |
|
||||
|
||||
Seiten ohne URL oder mit Ladefehler werden übersprungen und am Ende
|
||||
aufgelistet; der Exit-Code ist dann 1.
|
||||
|
||||
## Fonts
|
||||
|
||||
Ein Font liegt in `fonts/<name>/` und besteht aus
|
||||
|
||||
Binary file not shown.
@@ -1,662 +0,0 @@
|
||||
////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// CutyCapt - A Qt WebKit Web Page Rendering Capture Utility
|
||||
//
|
||||
// Copyright (C) 2003-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or
|
||||
// modify it under the terms of the GNU General Public License
|
||||
// as published by the Free Software Foundation; either version 2
|
||||
// of the License, or (at your option) any later version.
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or
|
||||
// modify it under the terms of the GNU Lesser General Public
|
||||
// License as published by the Free Software Foundation; either
|
||||
// version 2.1 of the License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// $Id$
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include <QApplication>
|
||||
#include <QtWebKit>
|
||||
#include <QtGui>
|
||||
#include <QSvgGenerator>
|
||||
#include <QPrinter>
|
||||
#include <QTimer>
|
||||
#include <QByteArray>
|
||||
#include <QNetworkRequest>
|
||||
#include "CutyCapt.hpp"
|
||||
|
||||
#if QT_VERSION >= 0x040600 && 0
|
||||
#define CUTYCAPT_SCRIPT 1
|
||||
#endif
|
||||
|
||||
#ifdef STATIC_PLUGINS
|
||||
Q_IMPORT_PLUGIN(qjpeg)
|
||||
Q_IMPORT_PLUGIN(qgif)
|
||||
Q_IMPORT_PLUGIN(qtiff)
|
||||
Q_IMPORT_PLUGIN(qsvg)
|
||||
Q_IMPORT_PLUGIN(qmng)
|
||||
Q_IMPORT_PLUGIN(qico)
|
||||
#endif
|
||||
|
||||
static struct _CutyExtMap {
|
||||
CutyCapt::OutputFormat id;
|
||||
const char* extension;
|
||||
const char* identifier;
|
||||
} const CutyExtMap[] = {
|
||||
{ CutyCapt::SvgFormat, ".svg", "svg" },
|
||||
{ CutyCapt::PdfFormat, ".pdf", "pdf" },
|
||||
{ CutyCapt::PsFormat, ".ps", "ps" },
|
||||
{ CutyCapt::InnerTextFormat, ".txt", "itext" },
|
||||
{ CutyCapt::HtmlFormat, ".html", "html" },
|
||||
{ CutyCapt::RenderTreeFormat, ".rtree", "rtree" },
|
||||
{ CutyCapt::JpegFormat, ".jpeg", "jpeg" },
|
||||
{ CutyCapt::PngFormat, ".png", "png" },
|
||||
{ CutyCapt::MngFormat, ".mng", "mng" },
|
||||
{ CutyCapt::TiffFormat, ".tiff", "tiff" },
|
||||
{ CutyCapt::GifFormat, ".gif", "gif" },
|
||||
{ CutyCapt::BmpFormat, ".bmp", "bmp" },
|
||||
{ CutyCapt::PpmFormat, ".ppm", "ppm" },
|
||||
{ CutyCapt::XbmFormat, ".xbm", "xbm" },
|
||||
{ CutyCapt::XpmFormat, ".xpm", "xpm" },
|
||||
{ CutyCapt::OtherFormat, "", "" }
|
||||
};
|
||||
|
||||
QString
|
||||
CutyPage::chooseFile(QWebFrame* /*frame*/, const QString& /*suggestedFile*/) {
|
||||
return QString::null;
|
||||
}
|
||||
|
||||
bool
|
||||
CutyPage::javaScriptConfirm(QWebFrame* /*frame*/, const QString& /*msg*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
CutyPage::javaScriptPrompt(QWebFrame* /*frame*/,
|
||||
const QString& /*msg*/,
|
||||
const QString& /*defaultValue*/,
|
||||
QString* /*result*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::javaScriptConsoleMessage(const QString& /*message*/,
|
||||
int /*lineNumber*/,
|
||||
const QString& /*sourceID*/) {
|
||||
// noop
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::javaScriptAlert(QWebFrame* /*frame*/, const QString& msg) {
|
||||
|
||||
if (mPrintAlerts)
|
||||
qDebug() << "[alert]" << msg;
|
||||
|
||||
if (mAlertString == msg) {
|
||||
QTimer::singleShot(10, mCutyCapt, SLOT(Delayed()));
|
||||
}
|
||||
}
|
||||
|
||||
QString
|
||||
CutyPage::userAgentForUrl(const QUrl& url) const {
|
||||
|
||||
if (!mUserAgent.isNull())
|
||||
return mUserAgent;
|
||||
|
||||
return QWebPage::userAgentForUrl(url);
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::setUserAgent(const QString& userAgent) {
|
||||
mUserAgent = userAgent;
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::setAlertString(const QString& alertString) {
|
||||
mAlertString = alertString;
|
||||
}
|
||||
|
||||
QString
|
||||
CutyPage::getAlertString() {
|
||||
return mAlertString;
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::setCutyCapt(CutyCapt* cutyCapt) {
|
||||
mCutyCapt = cutyCapt;
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::setPrintAlerts(bool printAlerts) {
|
||||
mPrintAlerts = printAlerts;
|
||||
}
|
||||
|
||||
void
|
||||
CutyPage::setAttribute(QWebSettings::WebAttribute option,
|
||||
const QString& value) {
|
||||
|
||||
if (value == "on")
|
||||
settings()->setAttribute(option, true);
|
||||
else if (value == "off")
|
||||
settings()->setAttribute(option, false);
|
||||
else
|
||||
(void)0; // TODO: ...
|
||||
}
|
||||
|
||||
// TODO: Consider merging some of main() and CutyCap
|
||||
|
||||
CutyCapt::CutyCapt(CutyPage* page, const QString& output, int delay, OutputFormat format,
|
||||
const QString& scriptProp, const QString& scriptCode) {
|
||||
mPage = page;
|
||||
mOutput = output;
|
||||
mDelay = delay;
|
||||
mSawInitialLayout = false;
|
||||
mSawDocumentComplete = false;
|
||||
mFormat = format;
|
||||
mScriptProp = scriptProp;
|
||||
mScriptCode = scriptCode;
|
||||
mScriptObj = new QObject();
|
||||
|
||||
// This is not really nice, but some restructuring work is
|
||||
// needed anyway, so this should not be that bad for now.
|
||||
mPage->setCutyCapt(this);
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::InitialLayoutCompleted() {
|
||||
mSawInitialLayout = true;
|
||||
|
||||
if (mSawInitialLayout && mSawDocumentComplete)
|
||||
TryDelayedRender();
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::DocumentComplete(bool /*ok*/) {
|
||||
mSawDocumentComplete = true;
|
||||
|
||||
if (mSawInitialLayout && mSawDocumentComplete)
|
||||
TryDelayedRender();
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::JavaScriptWindowObjectCleared() {
|
||||
|
||||
if (!mScriptProp.isEmpty()) {
|
||||
QVariant var = mPage->mainFrame()->evaluateJavaScript(mScriptProp);
|
||||
QObject* obj = var.value<QObject*>();
|
||||
|
||||
if (obj == mScriptObj)
|
||||
return;
|
||||
|
||||
mPage->mainFrame()->addToJavaScriptWindowObject(mScriptProp, mScriptObj);
|
||||
}
|
||||
|
||||
mPage->mainFrame()->evaluateJavaScript(mScriptCode);
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::TryDelayedRender() {
|
||||
|
||||
if (!mPage->getAlertString().isEmpty())
|
||||
return;
|
||||
|
||||
if (mDelay > 0) {
|
||||
QTimer::singleShot(mDelay, this, SLOT(Delayed()));
|
||||
return;
|
||||
}
|
||||
|
||||
saveSnapshot();
|
||||
QApplication::exit();
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::Timeout() {
|
||||
saveSnapshot();
|
||||
QApplication::exit();
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::Delayed() {
|
||||
saveSnapshot();
|
||||
QApplication::exit();
|
||||
}
|
||||
|
||||
void
|
||||
CutyCapt::saveSnapshot() {
|
||||
QWebFrame *mainFrame = mPage->mainFrame();
|
||||
QPainter painter;
|
||||
const char* format = NULL;
|
||||
|
||||
for (int ix = 0; CutyExtMap[ix].id != OtherFormat; ++ix)
|
||||
if (CutyExtMap[ix].id == mFormat)
|
||||
format = CutyExtMap[ix].identifier; //, break;
|
||||
|
||||
// TODO: sometimes contents/viewport can have size 0x0
|
||||
// in which case saving them will fail. This is likely
|
||||
// the result of the method being called too early. So
|
||||
// far I've been unable to find a workaround, except
|
||||
// using --delay with some substantial wait time. I've
|
||||
// tried to resize multiple time, make a fake render,
|
||||
// check for other events... This is primarily a problem
|
||||
// under my Ubuntu virtual machine.
|
||||
|
||||
mPage->setViewportSize( mainFrame->contentsSize() );
|
||||
|
||||
switch (mFormat) {
|
||||
case SvgFormat: {
|
||||
QSvgGenerator svg;
|
||||
svg.setFileName(mOutput);
|
||||
svg.setSize(mPage->viewportSize());
|
||||
painter.begin(&svg);
|
||||
mainFrame->render(&painter);
|
||||
painter.end();
|
||||
break;
|
||||
}
|
||||
case PdfFormat:
|
||||
case PsFormat: {
|
||||
QPrinter printer;
|
||||
printer.setPageSize(QPrinter::A4);
|
||||
printer.setOutputFileName(mOutput);
|
||||
// TODO: change quality here?
|
||||
mainFrame->print(&printer);
|
||||
break;
|
||||
}
|
||||
case RenderTreeFormat:
|
||||
case InnerTextFormat:
|
||||
case HtmlFormat: {
|
||||
QFile file(mOutput);
|
||||
file.open(QIODevice::WriteOnly | QIODevice::Text);
|
||||
QTextStream s(&file);
|
||||
s.setCodec("utf-8");
|
||||
s << (mFormat == RenderTreeFormat ? mainFrame->renderTreeDump() :
|
||||
mFormat == InnerTextFormat ? mainFrame->toPlainText() :
|
||||
mFormat == HtmlFormat ? mainFrame->toHtml() :
|
||||
"bug");
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
QImage image(mPage->viewportSize(), QImage::Format_ARGB32);
|
||||
painter.begin(&image);
|
||||
mainFrame->render(&painter);
|
||||
painter.end();
|
||||
// TODO: add quality
|
||||
image.save(mOutput, format);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
void
|
||||
CaptHelp(void) {
|
||||
printf("%s",
|
||||
" -----------------------------------------------------------------------------\n"
|
||||
" Usage: CutyCapt --url=http://www.example.org/ --out=localfile.png \n"
|
||||
" -----------------------------------------------------------------------------\n"
|
||||
" --help Print this help page and exit \n"
|
||||
" --url=<url> The URL to capture (http:...|file:...|...) \n"
|
||||
" --out=<path> The target file (.png|pdf|ps|svg|jpeg|...) \n"
|
||||
" --out-format=<f> Like extension in --out, overrides heuristic \n"
|
||||
// " --out-quality=<int> Output format quality from 1 to 100 \n"
|
||||
" --min-width=<int> Minimal width for the image (default: 800) \n"
|
||||
" --min-height=<int> Minimal height for the image (default: 600) \n"
|
||||
" --max-wait=<ms> Don't wait more than (default: 90000, inf: 0)\n"
|
||||
" --delay=<ms> After successful load, wait (default: 0) \n"
|
||||
// " --user-styles=<url> Location of user style sheet (deprecated) \n"
|
||||
" --user-style-path=<path> Location of user style sheet file, if any \n"
|
||||
" --user-style-string=<css> User style rules specified as text \n"
|
||||
" --header=<name>:<value> request header; repeatable; some can't be set\n"
|
||||
" --method=<get|post|put> Specifies the request method (default: get) \n"
|
||||
" --body-string=<string> Unencoded request body (default: none) \n"
|
||||
" --body-base64=<base64> Base64-encoded request body (default: none) \n"
|
||||
" --app-name=<name> appName used in User-Agent; default is none \n"
|
||||
" --app-version=<version> appVers used in User-Agent; default is none \n"
|
||||
" --user-agent=<string> Override the User-Agent header Qt would set \n"
|
||||
" --javascript=<on|off> JavaScript execution (default: on) \n"
|
||||
" --java=<on|off> Java execution (default: unknown) \n"
|
||||
" --plugins=<on|off> Plugin execution (default: unknown) \n"
|
||||
" --private-browsing=<on|off> Private browsing (default: unknown) \n"
|
||||
" --auto-load-images=<on|off> Automatic image loading (default: on) \n"
|
||||
" --js-can-open-windows=<on|off> Script can open windows? (default: unknown) \n"
|
||||
" --js-can-access-clipboard=<on|off> Script clipboard privs (default: unknown)\n"
|
||||
#if QT_VERSION >= 0x040500
|
||||
" --print-backgrounds=<on|off> Backgrounds in PDF/PS output (default: off) \n"
|
||||
" --zoom-factor=<float> Page zoom factor (default: no zooming) \n"
|
||||
" --zoom-text-only=<on|off> Whether to zoom only the text (default: off) \n"
|
||||
" --http-proxy=<url> Address for HTTP proxy server (default: none)\n"
|
||||
#endif
|
||||
#if CUTYCAPT_SCRIPT
|
||||
" --inject-script=<path> JavaScript that will be injected into pages \n"
|
||||
" --script-object=<string> Property to hold state for injected script \n"
|
||||
" --expect-alert=<string> Try waiting for alert(string) before capture \n"
|
||||
" --debug-print-alerts Prints out alert(...) strings for debugging. \n"
|
||||
#endif
|
||||
" -----------------------------------------------------------------------------\n"
|
||||
" <f> is svg,ps,pdf,itext,html,rtree,png,jpeg,mng,tiff,gif,bmp,ppm,xbm,xpm \n"
|
||||
" -----------------------------------------------------------------------------\n"
|
||||
#if CUTYCAPT_SCRIPT
|
||||
" The `inject-script` option can be used to inject script code into loaded web \n"
|
||||
" pages. The code is called whenever the `javaScriptWindowObjectCleared` signal\n"
|
||||
" is received. When `script-object` is set, an object under the specified name \n"
|
||||
" will be available to the script to maintain state across page loads. When the\n"
|
||||
" `expect-alert` option is specified, the shot will be taken when a script in- \n"
|
||||
" vokes alert(string) with the string if that happens before `max-wait`. These \n"
|
||||
" options effectively allow you to remote control the browser and the web page.\n"
|
||||
" This an experimental and easily abused and misused feature. Use with caution.\n"
|
||||
" -----------------------------------------------------------------------------\n"
|
||||
#endif
|
||||
" http://cutycapt.sf.net - (c) 2003-2010 Bjoern Hoehrmann - bjoern@hoehrmann.de\n"
|
||||
"");
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char *argv[]) {
|
||||
|
||||
int argHelp = 0;
|
||||
int argDelay = 0;
|
||||
int argSilent = 0;
|
||||
int argMinWidth = 800;
|
||||
int argMinHeight = 600;
|
||||
int argMaxWait = 90000;
|
||||
int argVerbosity = 0;
|
||||
|
||||
const char* argUrl = NULL;
|
||||
const char* argUserStyle = NULL;
|
||||
const char* argUserStylePath = NULL;
|
||||
const char* argUserStyleString = NULL;
|
||||
const char* argIconDbPath = NULL;
|
||||
const char* argInjectScript = NULL;
|
||||
const char* argScriptObject = NULL;
|
||||
QString argOut;
|
||||
|
||||
CutyCapt::OutputFormat format = CutyCapt::OtherFormat;
|
||||
|
||||
QApplication app(argc, argv, true);
|
||||
CutyPage page;
|
||||
|
||||
QNetworkAccessManager::Operation method =
|
||||
QNetworkAccessManager::GetOperation;
|
||||
QByteArray body;
|
||||
QNetworkRequest req;
|
||||
QNetworkAccessManager manager;
|
||||
|
||||
// Parse command line parameters
|
||||
for (int ax = 1; ax < argc; ++ax) {
|
||||
size_t nlen;
|
||||
|
||||
const char* s = argv[ax];
|
||||
const char* value;
|
||||
|
||||
// boolean options
|
||||
if (strcmp("--silent", s) == 0) {
|
||||
argSilent = 1;
|
||||
continue;
|
||||
|
||||
} else if (strcmp("--help", s) == 0) {
|
||||
argHelp = 1;
|
||||
break;
|
||||
|
||||
} else if (strcmp("--verbose", s) == 0) {
|
||||
argVerbosity++;
|
||||
continue;
|
||||
|
||||
#if CUTYCAPT_SCRIPT
|
||||
} else if (strcmp("--debug-print-alerts", s) == 0) {
|
||||
page.setPrintAlerts(true);
|
||||
continue;
|
||||
#endif
|
||||
}
|
||||
|
||||
value = strchr(s, '=');
|
||||
|
||||
if (value == NULL) {
|
||||
// TODO: error
|
||||
argHelp = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
nlen = value++ - s;
|
||||
|
||||
// --name=value options
|
||||
if (strncmp("--url", s, nlen) == 0) {
|
||||
argUrl = value;
|
||||
|
||||
} else if (strncmp("--min-width", s, nlen) == 0) {
|
||||
// TODO: add error checking here?
|
||||
argMinWidth = (unsigned int)atoi(value);
|
||||
|
||||
} else if (strncmp("--min-height", s, nlen) == 0) {
|
||||
// TODO: add error checking here?
|
||||
argMinHeight = (unsigned int)atoi(value);
|
||||
|
||||
} else if (strncmp("--delay", s, nlen) == 0) {
|
||||
// TODO: see above
|
||||
argDelay = (unsigned int)atoi(value);
|
||||
|
||||
} else if (strncmp("--max-wait", s, nlen) == 0) {
|
||||
// TODO: see above
|
||||
argMaxWait = (unsigned int)atoi(value);
|
||||
|
||||
} else if (strncmp("--out", s, nlen) == 0) {
|
||||
argOut = value;
|
||||
|
||||
if (format == CutyCapt::OtherFormat)
|
||||
for (int ix = 0; CutyExtMap[ix].id != CutyCapt::OtherFormat; ++ix)
|
||||
if (argOut.endsWith(CutyExtMap[ix].extension))
|
||||
format = CutyExtMap[ix].id; //, break;
|
||||
|
||||
} else if (strncmp("--user-styles", s, nlen) == 0) {
|
||||
// This option is provided for backwards-compatibility only
|
||||
argUserStyle = value;
|
||||
|
||||
} else if (strncmp("--user-style-path", s, nlen) == 0) {
|
||||
argUserStylePath = value;
|
||||
|
||||
} else if (strncmp("--user-style-string", s, nlen) == 0) {
|
||||
argUserStyleString = value;
|
||||
|
||||
} else if (strncmp("--icon-database-path", s, nlen) == 0) {
|
||||
argIconDbPath = value;
|
||||
|
||||
} else if (strncmp("--auto-load-images", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::AutoLoadImages, value);
|
||||
|
||||
} else if (strncmp("--javascript", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::JavascriptEnabled, value);
|
||||
|
||||
} else if (strncmp("--java", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::JavaEnabled, value);
|
||||
|
||||
} else if (strncmp("--plugins", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::PluginsEnabled, value);
|
||||
|
||||
} else if (strncmp("--private-browsing", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::PrivateBrowsingEnabled, value);
|
||||
|
||||
} else if (strncmp("--js-can-open-windows", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::JavascriptCanOpenWindows, value);
|
||||
|
||||
} else if (strncmp("--js-can-access-clipboard", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::JavascriptCanAccessClipboard, value);
|
||||
|
||||
} else if (strncmp("--developer-extras", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::DeveloperExtrasEnabled, value);
|
||||
|
||||
} else if (strncmp("--links-included-in-focus-chain", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::LinksIncludedInFocusChain, value);
|
||||
|
||||
#if QT_VERSION >= 0x040500
|
||||
} else if (strncmp("--print-backgrounds", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::PrintElementBackgrounds, value);
|
||||
|
||||
} else if (strncmp("--zoom-factor", s, nlen) == 0) {
|
||||
page.mainFrame()->setZoomFactor(QString(value).toFloat());
|
||||
|
||||
} else if (strncmp("--zoom-text-only", s, nlen) == 0) {
|
||||
page.setAttribute(QWebSettings::ZoomTextOnly, value);
|
||||
|
||||
} else if (strncmp("--http-proxy", s, nlen) == 0) {
|
||||
QUrl p = QUrl::fromEncoded(value);
|
||||
QNetworkProxy proxy = QNetworkProxy(QNetworkProxy::HttpProxy,
|
||||
p.host(), p.port(80), p.userName(), p.password());
|
||||
manager.setProxy(proxy);
|
||||
page.setNetworkAccessManager(&manager);
|
||||
#endif
|
||||
|
||||
#if CUTYCAPT_SCRIPT
|
||||
} else if (strncmp("--inject-script", s, nlen) == 0) {
|
||||
argInjectScript = value;
|
||||
|
||||
} else if (strncmp("--script-object", s, nlen) == 0) {
|
||||
argScriptObject = value;
|
||||
|
||||
} else if (strncmp("--expect-alert", s, nlen) == 0) {
|
||||
page.setAlertString(value);
|
||||
#endif
|
||||
|
||||
} else if (strncmp("--app-name", s, nlen) == 0) {
|
||||
app.setApplicationName(value);
|
||||
|
||||
} else if (strncmp("--app-version", s, nlen) == 0) {
|
||||
app.setApplicationVersion(value);
|
||||
|
||||
} else if (strncmp("--body-base64", s, nlen) == 0) {
|
||||
body = QByteArray::fromBase64(value);
|
||||
|
||||
} else if (strncmp("--body-string", s, nlen) == 0) {
|
||||
body = QByteArray(value);
|
||||
|
||||
} else if (strncmp("--user-agent", s, nlen) == 0) {
|
||||
page.setUserAgent(value);
|
||||
|
||||
} else if (strncmp("--out-format", s, nlen) == 0) {
|
||||
for (int ix = 0; CutyExtMap[ix].id != CutyCapt::OtherFormat; ++ix)
|
||||
if (strcmp(value, CutyExtMap[ix].identifier) == 0)
|
||||
format = CutyExtMap[ix].id; //, break;
|
||||
|
||||
if (format == CutyCapt::OtherFormat) {
|
||||
// TODO: error
|
||||
argHelp = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
} else if (strncmp("--header", s, nlen) == 0) {
|
||||
const char* hv = strchr(value, ':');
|
||||
|
||||
if (hv == NULL) {
|
||||
// TODO: error
|
||||
argHelp = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
req.setRawHeader(QByteArray(value, hv - value), hv + 1);
|
||||
|
||||
} else if (strncmp("--method", s, nlen) == 0) {
|
||||
if (strcmp("value", "get") == 0)
|
||||
method = QNetworkAccessManager::GetOperation;
|
||||
else if (strcmp("value", "put") == 0)
|
||||
method = QNetworkAccessManager::PutOperation;
|
||||
else if (strcmp("value", "post") == 0)
|
||||
method = QNetworkAccessManager::PostOperation;
|
||||
else if (strcmp("value", "head") == 0)
|
||||
method = QNetworkAccessManager::HeadOperation;
|
||||
else
|
||||
(void)0; // TODO: ...
|
||||
|
||||
} else {
|
||||
// TODO: error
|
||||
argHelp = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (argUrl == NULL || argOut == NULL || argHelp) {
|
||||
CaptHelp();
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
// This used to use QUrl(argUrl) but that escapes %hh sequences
|
||||
// even though it should not, as URLs can assumed to be escaped.
|
||||
req.setUrl( QUrl::fromEncoded(argUrl) );
|
||||
|
||||
QString scriptProp(argScriptObject);
|
||||
QString scriptCode;
|
||||
|
||||
if (argInjectScript) {
|
||||
QFile file(argInjectScript);
|
||||
if (file.open(QIODevice::ReadOnly)) {
|
||||
QTextStream stream(&file);
|
||||
stream.setCodec(QTextCodec::codecForName("UTF-8"));
|
||||
stream.setAutoDetectUnicode(true);
|
||||
scriptCode = stream.readAll();
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
|
||||
CutyCapt main(&page, argOut, argDelay, format, scriptProp, scriptCode);
|
||||
|
||||
app.connect(&page,
|
||||
SIGNAL(loadFinished(bool)),
|
||||
&main,
|
||||
SLOT(DocumentComplete(bool)));
|
||||
|
||||
app.connect(page.mainFrame(),
|
||||
SIGNAL(initialLayoutCompleted()),
|
||||
&main,
|
||||
SLOT(InitialLayoutCompleted()));
|
||||
|
||||
if (argMaxWait > 0) {
|
||||
// TODO: Should this also register one for the application?
|
||||
QTimer::singleShot(argMaxWait, &main, SLOT(Timeout()));
|
||||
}
|
||||
|
||||
if (argUserStyle != NULL)
|
||||
// TODO: does this need any syntax checking?
|
||||
page.settings()->setUserStyleSheetUrl( QUrl::fromEncoded(argUserStyle) );
|
||||
|
||||
if (argUserStylePath != NULL) {
|
||||
page.settings()->setUserStyleSheetUrl( QUrl::fromLocalFile(argUserStylePath) );
|
||||
}
|
||||
|
||||
if (argUserStyleString != NULL) {
|
||||
QUrl data("data:text/css;charset=utf-8;base64," +
|
||||
QByteArray(argUserStyleString).toBase64());
|
||||
page.settings()->setUserStyleSheetUrl( data );
|
||||
}
|
||||
|
||||
if (argIconDbPath != NULL)
|
||||
// TODO: does this need any syntax checking?
|
||||
page.settings()->setIconDatabasePath(argUserStyle);
|
||||
|
||||
// The documentation does not say, but it seems the mainFrame
|
||||
// will never change, so we can set this here. Otherwise we'd
|
||||
// have to set this in snapshot and trigger an update, which
|
||||
// is not currently possible (Qt 4.4.0) as far as I can tell.
|
||||
page.mainFrame()->setScrollBarPolicy(Qt::Horizontal, Qt::ScrollBarAlwaysOff);
|
||||
page.mainFrame()->setScrollBarPolicy(Qt::Vertical, Qt::ScrollBarAlwaysOff);
|
||||
page.setViewportSize( QSize(argMinWidth, argMinHeight) );
|
||||
|
||||
#if CUTYCAPT_SCRIPT
|
||||
// javaScriptWindowObjectCleared does not get called on the
|
||||
// initial load unless some JavaScript has been executed.
|
||||
page.mainFrame()->evaluateJavaScript(QString(""));
|
||||
|
||||
app.connect(page.mainFrame(),
|
||||
SIGNAL(javaScriptWindowObjectCleared()),
|
||||
&main,
|
||||
SLOT(JavaScriptWindowObjectCleared()));
|
||||
#endif
|
||||
|
||||
if (!body.isNull())
|
||||
page.mainFrame()->load(req, method, body);
|
||||
else
|
||||
page.mainFrame()->load(req, method);
|
||||
|
||||
return app.exec();
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
#include <QtWebKit>
|
||||
|
||||
class CutyCapt;
|
||||
class CutyPage : public QWebPage {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
void setAttribute(QWebSettings::WebAttribute option, const QString& value);
|
||||
void setUserAgent(const QString& userAgent);
|
||||
void setAlertString(const QString& alertString);
|
||||
void setPrintAlerts(bool printAlerts);
|
||||
void setCutyCapt(CutyCapt* cutyCapt);
|
||||
QString getAlertString();
|
||||
|
||||
protected:
|
||||
QString chooseFile(QWebFrame *frame, const QString& suggestedFile);
|
||||
void javaScriptConsoleMessage(const QString& message, int lineNumber, const QString& sourceID);
|
||||
bool javaScriptPrompt(QWebFrame* frame, const QString& msg, const QString& defaultValue, QString* result);
|
||||
void javaScriptAlert(QWebFrame* frame, const QString& msg);
|
||||
bool javaScriptConfirm(QWebFrame* frame, const QString& msg);
|
||||
QString userAgentForUrl(const QUrl& url) const;
|
||||
QString mUserAgent;
|
||||
QString mAlertString;
|
||||
bool mPrintAlerts;
|
||||
CutyCapt* mCutyCapt;
|
||||
};
|
||||
|
||||
class CutyCapt : public QObject {
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
|
||||
// TODO: This should really be elsewhere and be named differently
|
||||
enum OutputFormat { SvgFormat, PdfFormat, PsFormat, InnerTextFormat, HtmlFormat,
|
||||
RenderTreeFormat, PngFormat, JpegFormat, MngFormat, TiffFormat, GifFormat,
|
||||
BmpFormat, PpmFormat, XbmFormat, XpmFormat, OtherFormat };
|
||||
|
||||
CutyCapt(CutyPage* page,
|
||||
const QString& output,
|
||||
int delay,
|
||||
OutputFormat format,
|
||||
const QString& scriptProp,
|
||||
const QString& scriptCode);
|
||||
|
||||
private slots:
|
||||
void DocumentComplete(bool ok);
|
||||
void InitialLayoutCompleted();
|
||||
void JavaScriptWindowObjectCleared();
|
||||
void Timeout();
|
||||
void Delayed();
|
||||
|
||||
private:
|
||||
void TryDelayedRender();
|
||||
void saveSnapshot();
|
||||
bool mSawInitialLayout;
|
||||
bool mSawDocumentComplete;
|
||||
|
||||
protected:
|
||||
QString mOutput;
|
||||
int mDelay;
|
||||
CutyPage* mPage;
|
||||
OutputFormat mFormat;
|
||||
QObject* mScriptObj;
|
||||
QString mScriptProp;
|
||||
QString mScriptCode;
|
||||
};
|
||||
Binary file not shown.
@@ -1,10 +0,0 @@
|
||||
QT += webkit svg network
|
||||
SOURCES = CutyCapt.cpp
|
||||
HEADERS = CutyCapt.hpp
|
||||
CONFIG += qt console
|
||||
|
||||
contains(CONFIG, static): {
|
||||
QTPLUGIN += qjpeg qgif qsvg qmng qico qtiff
|
||||
DEFINES += STATIC_PLUGINS
|
||||
}
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
#############################################################################
|
||||
# Makefile for building: CutyCapt
|
||||
# Generated by qmake (2.01a) (Qt 4.8.1) on: Thu Aug 30 14:02:45 2012
|
||||
# Project: CutyCapt.pro
|
||||
# Template: app
|
||||
# Command: /usr/bin/qmake -o Makefile CutyCapt.pro
|
||||
#############################################################################
|
||||
|
||||
####### Compiler, tools and options
|
||||
|
||||
CC = gcc
|
||||
CXX = g++
|
||||
DEFINES = -DQT_WEBKIT -DQT_NO_DEBUG -DQT_WEBKIT_LIB -DQT_SVG_LIB -DQT_GUI_LIB -DQT_NETWORK_LIB -DQT_CORE_LIB -DQT_SHARED
|
||||
CFLAGS = -pipe -O2 -D_REENTRANT -Wall -W $(DEFINES)
|
||||
CXXFLAGS = -pipe -O2 -D_REENTRANT -Wall -W $(DEFINES)
|
||||
INCPATH = -I/usr/share/qt4/mkspecs/linux-g++ -I. -I/usr/include/qt4/QtCore -I/usr/include/qt4/QtNetwork -I/usr/include/qt4/QtGui -I/usr/include/qt4/QtSvg -I/usr/include/qt4/QtWebKit -I/usr/include/qt4 -I.
|
||||
LINK = g++
|
||||
LFLAGS = -Wl,-O1
|
||||
LIBS = $(SUBLIBS) -L/usr/lib/i386-linux-gnu -lQtWebKit -lQtSvg -lQtGui -lQtNetwork -lQtCore -lpthread
|
||||
AR = ar cqs
|
||||
RANLIB =
|
||||
QMAKE = /usr/bin/qmake
|
||||
TAR = tar -cf
|
||||
COMPRESS = gzip -9f
|
||||
COPY = cp -f
|
||||
SED = sed
|
||||
COPY_FILE = $(COPY)
|
||||
COPY_DIR = $(COPY) -r
|
||||
STRIP = strip
|
||||
INSTALL_FILE = install -m 644 -p
|
||||
INSTALL_DIR = $(COPY_DIR)
|
||||
INSTALL_PROGRAM = install -m 755 -p
|
||||
DEL_FILE = rm -f
|
||||
SYMLINK = ln -f -s
|
||||
DEL_DIR = rmdir
|
||||
MOVE = mv -f
|
||||
CHK_DIR_EXISTS= test -d
|
||||
MKDIR = mkdir -p
|
||||
|
||||
####### Output directory
|
||||
|
||||
OBJECTS_DIR = ./
|
||||
|
||||
####### Files
|
||||
|
||||
SOURCES = CutyCapt.cpp moc_CutyCapt.cpp
|
||||
OBJECTS = CutyCapt.o \
|
||||
moc_CutyCapt.o
|
||||
DIST = /usr/share/qt4/mkspecs/common/unix.conf \
|
||||
/usr/share/qt4/mkspecs/common/linux.conf \
|
||||
/usr/share/qt4/mkspecs/common/gcc-base.conf \
|
||||
/usr/share/qt4/mkspecs/common/gcc-base-unix.conf \
|
||||
/usr/share/qt4/mkspecs/common/g++-base.conf \
|
||||
/usr/share/qt4/mkspecs/common/g++-unix.conf \
|
||||
/usr/share/qt4/mkspecs/qconfig.pri \
|
||||
/usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
|
||||
/usr/share/qt4/mkspecs/features/qt_functions.prf \
|
||||
/usr/share/qt4/mkspecs/features/qt_config.prf \
|
||||
/usr/share/qt4/mkspecs/features/exclusive_builds.prf \
|
||||
/usr/share/qt4/mkspecs/features/default_pre.prf \
|
||||
/usr/share/qt4/mkspecs/features/release.prf \
|
||||
/usr/share/qt4/mkspecs/features/default_post.prf \
|
||||
/usr/share/qt4/mkspecs/features/qt.prf \
|
||||
/usr/share/qt4/mkspecs/features/unix/thread.prf \
|
||||
/usr/share/qt4/mkspecs/features/moc.prf \
|
||||
/usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \
|
||||
/usr/share/qt4/mkspecs/features/warn_on.prf \
|
||||
/usr/share/qt4/mkspecs/features/resources.prf \
|
||||
/usr/share/qt4/mkspecs/features/uic.prf \
|
||||
/usr/share/qt4/mkspecs/features/yacc.prf \
|
||||
/usr/share/qt4/mkspecs/features/lex.prf \
|
||||
/usr/share/qt4/mkspecs/features/include_source_dir.prf \
|
||||
CutyCapt.pro
|
||||
QMAKE_TARGET = CutyCapt
|
||||
DESTDIR =
|
||||
TARGET = CutyCapt
|
||||
|
||||
first: all
|
||||
####### Implicit rules
|
||||
|
||||
.SUFFIXES: .o .c .cpp .cc .cxx .C
|
||||
|
||||
.cpp.o:
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
|
||||
|
||||
.cc.o:
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
|
||||
|
||||
.cxx.o:
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
|
||||
|
||||
.C.o:
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
|
||||
|
||||
.c.o:
|
||||
$(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<"
|
||||
|
||||
####### Build rules
|
||||
|
||||
all: Makefile $(TARGET)
|
||||
|
||||
$(TARGET): $(OBJECTS)
|
||||
$(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(OBJCOMP) $(LIBS)
|
||||
|
||||
Makefile: CutyCapt.pro /usr/share/qt4/mkspecs/linux-g++/qmake.conf /usr/share/qt4/mkspecs/common/unix.conf \
|
||||
/usr/share/qt4/mkspecs/common/linux.conf \
|
||||
/usr/share/qt4/mkspecs/common/gcc-base.conf \
|
||||
/usr/share/qt4/mkspecs/common/gcc-base-unix.conf \
|
||||
/usr/share/qt4/mkspecs/common/g++-base.conf \
|
||||
/usr/share/qt4/mkspecs/common/g++-unix.conf \
|
||||
/usr/share/qt4/mkspecs/qconfig.pri \
|
||||
/usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \
|
||||
/usr/share/qt4/mkspecs/features/qt_functions.prf \
|
||||
/usr/share/qt4/mkspecs/features/qt_config.prf \
|
||||
/usr/share/qt4/mkspecs/features/exclusive_builds.prf \
|
||||
/usr/share/qt4/mkspecs/features/default_pre.prf \
|
||||
/usr/share/qt4/mkspecs/features/release.prf \
|
||||
/usr/share/qt4/mkspecs/features/default_post.prf \
|
||||
/usr/share/qt4/mkspecs/features/qt.prf \
|
||||
/usr/share/qt4/mkspecs/features/unix/thread.prf \
|
||||
/usr/share/qt4/mkspecs/features/moc.prf \
|
||||
/usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \
|
||||
/usr/share/qt4/mkspecs/features/warn_on.prf \
|
||||
/usr/share/qt4/mkspecs/features/resources.prf \
|
||||
/usr/share/qt4/mkspecs/features/uic.prf \
|
||||
/usr/share/qt4/mkspecs/features/yacc.prf \
|
||||
/usr/share/qt4/mkspecs/features/lex.prf \
|
||||
/usr/share/qt4/mkspecs/features/include_source_dir.prf \
|
||||
/usr/lib/i386-linux-gnu/libQtWebKit.prl \
|
||||
/usr/lib/i386-linux-gnu/libQtSvg.prl \
|
||||
/usr/lib/i386-linux-gnu/libQtGui.prl \
|
||||
/usr/lib/i386-linux-gnu/libQtNetwork.prl \
|
||||
/usr/lib/i386-linux-gnu/libQtCore.prl
|
||||
$(QMAKE) -o Makefile CutyCapt.pro
|
||||
/usr/share/qt4/mkspecs/common/unix.conf:
|
||||
/usr/share/qt4/mkspecs/common/linux.conf:
|
||||
/usr/share/qt4/mkspecs/common/gcc-base.conf:
|
||||
/usr/share/qt4/mkspecs/common/gcc-base-unix.conf:
|
||||
/usr/share/qt4/mkspecs/common/g++-base.conf:
|
||||
/usr/share/qt4/mkspecs/common/g++-unix.conf:
|
||||
/usr/share/qt4/mkspecs/qconfig.pri:
|
||||
/usr/share/qt4/mkspecs/modules/qt_webkit_version.pri:
|
||||
/usr/share/qt4/mkspecs/features/qt_functions.prf:
|
||||
/usr/share/qt4/mkspecs/features/qt_config.prf:
|
||||
/usr/share/qt4/mkspecs/features/exclusive_builds.prf:
|
||||
/usr/share/qt4/mkspecs/features/default_pre.prf:
|
||||
/usr/share/qt4/mkspecs/features/release.prf:
|
||||
/usr/share/qt4/mkspecs/features/default_post.prf:
|
||||
/usr/share/qt4/mkspecs/features/qt.prf:
|
||||
/usr/share/qt4/mkspecs/features/unix/thread.prf:
|
||||
/usr/share/qt4/mkspecs/features/moc.prf:
|
||||
/usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf:
|
||||
/usr/share/qt4/mkspecs/features/warn_on.prf:
|
||||
/usr/share/qt4/mkspecs/features/resources.prf:
|
||||
/usr/share/qt4/mkspecs/features/uic.prf:
|
||||
/usr/share/qt4/mkspecs/features/yacc.prf:
|
||||
/usr/share/qt4/mkspecs/features/lex.prf:
|
||||
/usr/share/qt4/mkspecs/features/include_source_dir.prf:
|
||||
/usr/lib/i386-linux-gnu/libQtWebKit.prl:
|
||||
/usr/lib/i386-linux-gnu/libQtSvg.prl:
|
||||
/usr/lib/i386-linux-gnu/libQtGui.prl:
|
||||
/usr/lib/i386-linux-gnu/libQtNetwork.prl:
|
||||
/usr/lib/i386-linux-gnu/libQtCore.prl:
|
||||
qmake: FORCE
|
||||
@$(QMAKE) -o Makefile CutyCapt.pro
|
||||
|
||||
dist:
|
||||
@$(CHK_DIR_EXISTS) .tmp/CutyCapt1.0.0 || $(MKDIR) .tmp/CutyCapt1.0.0
|
||||
$(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/CutyCapt1.0.0/ && $(COPY_FILE) --parents CutyCapt.hpp .tmp/CutyCapt1.0.0/ && $(COPY_FILE) --parents CutyCapt.cpp .tmp/CutyCapt1.0.0/ && (cd `dirname .tmp/CutyCapt1.0.0` && $(TAR) CutyCapt1.0.0.tar CutyCapt1.0.0 && $(COMPRESS) CutyCapt1.0.0.tar) && $(MOVE) `dirname .tmp/CutyCapt1.0.0`/CutyCapt1.0.0.tar.gz . && $(DEL_FILE) -r .tmp/CutyCapt1.0.0
|
||||
|
||||
|
||||
clean:compiler_clean
|
||||
-$(DEL_FILE) $(OBJECTS)
|
||||
-$(DEL_FILE) *~ core *.core
|
||||
|
||||
|
||||
####### Sub-libraries
|
||||
|
||||
distclean: clean
|
||||
-$(DEL_FILE) $(TARGET)
|
||||
-$(DEL_FILE) Makefile
|
||||
|
||||
|
||||
check: first
|
||||
|
||||
mocclean: compiler_moc_header_clean compiler_moc_source_clean
|
||||
|
||||
mocables: compiler_moc_header_make_all compiler_moc_source_make_all
|
||||
|
||||
compiler_moc_header_make_all: moc_CutyCapt.cpp
|
||||
compiler_moc_header_clean:
|
||||
-$(DEL_FILE) moc_CutyCapt.cpp
|
||||
moc_CutyCapt.cpp: CutyCapt.hpp
|
||||
/usr/bin/moc-qt4 $(DEFINES) $(INCPATH) CutyCapt.hpp -o moc_CutyCapt.cpp
|
||||
|
||||
compiler_rcc_make_all:
|
||||
compiler_rcc_clean:
|
||||
compiler_image_collection_make_all: qmake_image_collection.cpp
|
||||
compiler_image_collection_clean:
|
||||
-$(DEL_FILE) qmake_image_collection.cpp
|
||||
compiler_moc_source_make_all:
|
||||
compiler_moc_source_clean:
|
||||
compiler_uic_make_all:
|
||||
compiler_uic_clean:
|
||||
compiler_yacc_decl_make_all:
|
||||
compiler_yacc_decl_clean:
|
||||
compiler_yacc_impl_make_all:
|
||||
compiler_yacc_impl_clean:
|
||||
compiler_lex_make_all:
|
||||
compiler_lex_clean:
|
||||
compiler_clean: compiler_moc_header_clean
|
||||
|
||||
####### Compile
|
||||
|
||||
CutyCapt.o: CutyCapt.cpp CutyCapt.hpp
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o CutyCapt.o CutyCapt.cpp
|
||||
|
||||
moc_CutyCapt.o: moc_CutyCapt.cpp
|
||||
$(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_CutyCapt.o moc_CutyCapt.cpp
|
||||
|
||||
####### Install
|
||||
|
||||
install: FORCE
|
||||
|
||||
uninstall: FORCE
|
||||
|
||||
FORCE:
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
/****************************************************************************
|
||||
** Meta object code from reading C++ file 'CutyCapt.hpp'
|
||||
**
|
||||
** Created: Thu Aug 30 14:03:03 2012
|
||||
** by: The Qt Meta Object Compiler version 63 (Qt 4.8.1)
|
||||
**
|
||||
** WARNING! All changes made in this file will be lost!
|
||||
*****************************************************************************/
|
||||
|
||||
#include "CutyCapt.hpp"
|
||||
#if !defined(Q_MOC_OUTPUT_REVISION)
|
||||
#error "The header file 'CutyCapt.hpp' doesn't include <QObject>."
|
||||
#elif Q_MOC_OUTPUT_REVISION != 63
|
||||
#error "This file was generated using the moc from 4.8.1. It"
|
||||
#error "cannot be used with the include files from this version of Qt."
|
||||
#error "(The moc has changed too much.)"
|
||||
#endif
|
||||
|
||||
QT_BEGIN_MOC_NAMESPACE
|
||||
static const uint qt_meta_data_CutyPage[] = {
|
||||
|
||||
// content:
|
||||
6, // revision
|
||||
0, // classname
|
||||
0, 0, // classinfo
|
||||
0, 0, // methods
|
||||
0, 0, // properties
|
||||
0, 0, // enums/sets
|
||||
0, 0, // constructors
|
||||
0, // flags
|
||||
0, // signalCount
|
||||
|
||||
0 // eod
|
||||
};
|
||||
|
||||
static const char qt_meta_stringdata_CutyPage[] = {
|
||||
"CutyPage\0"
|
||||
};
|
||||
|
||||
void CutyPage::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a)
|
||||
{
|
||||
Q_UNUSED(_o);
|
||||
Q_UNUSED(_id);
|
||||
Q_UNUSED(_c);
|
||||
Q_UNUSED(_a);
|
||||
}
|
||||
|
||||
const QMetaObjectExtraData CutyPage::staticMetaObjectExtraData = {
|
||||
0, qt_static_metacall
|
||||
};
|
||||
|
||||
const QMetaObject CutyPage::staticMetaObject = {
|
||||
{ &QWebPage::staticMetaObject, qt_meta_stringdata_CutyPage,
|
||||
qt_meta_data_CutyPage, &staticMetaObjectExtraData }
|
||||
};
|
||||
|
||||
#ifdef Q_NO_DATA_RELOCATION
|
||||
const QMetaObject &CutyPage::getStaticMetaObject() { return staticMetaObject; }
|
||||
#endif //Q_NO_DATA_RELOCATION
|
||||
|
||||
const QMetaObject *CutyPage::metaObject() const
|
||||
{
|
||||
return QObject::d_ptr->metaObject ? QObject::d_ptr->metaObject : &staticMetaObject;
|
||||
}
|
||||
|
||||
void *CutyPage::qt_metacast(const char *_clname)
|
||||
{
|
||||
if (!_clname) return 0;
|
||||
if (!strcmp(_clname, qt_meta_stringdata_CutyPage))
|
||||
return static_cast<void*>(const_cast< CutyPage*>(this));
|
||||
return QWebPage::qt_metacast(_clname);
|
||||
}
|
||||
|
||||
int CutyPage::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
|
||||
{
|
||||
_id = QWebPage::qt_metacall(_c, _id, _a);
|
||||
if (_id < 0)
|
||||
return _id;
|
||||
return _id;
|
||||
}
|
||||
static const uint qt_meta_data_CutyCapt[] = {
|
||||
|
||||
// content:
|
||||
6, // revision
|
||||
0, // classname
|
||||
0, 0, // classinfo
|
||||
5, 14, // methods
|
||||
0, 0, // properties
|
||||
0, 0, // enums/sets
|
||||
0, 0, // constructors
|
||||
0, // flags
|
||||
0, // signalCount
|
||||
|
||||
// slots: signature, parameters, type, tag, flags
|
||||
13, 10, 9, 9, 0x08,
|
||||
36, 9, 9, 9, 0x08,
|
||||
61, 9, 9, 9, 0x08,
|
||||
93, 9, 9, 9, 0x08,
|
||||
103, 9, 9, 9, 0x08,
|
||||
|
||||
0 // eod
|
||||
};
|
||||
|
||||
static const char qt_meta_stringdata_CutyCapt[] = {
|
||||
"CutyCapt\0\0ok\0DocumentComplete(bool)\0"
|
||||
"InitialLayoutCompleted()\0"
|
||||
"JavaScriptWindowObjectCleared()\0"
|
||||
"Timeout()\0Delayed()\0"
|
||||
};
|
||||
|
||||
void CutyCapt::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a)
|
||||
{
|
||||
if (_c == QMetaObject::InvokeMetaMethod) {
|
||||
Q_ASSERT(staticMetaObject.cast(_o));
|
||||
CutyCapt *_t = static_cast<CutyCapt *>(_o);
|
||||
switch (_id) {
|
||||
case 0: _t->DocumentComplete((*reinterpret_cast< bool(*)>(_a[1]))); break;
|
||||
case 1: _t->InitialLayoutCompleted(); break;
|
||||
case 2: _t->JavaScriptWindowObjectCleared(); break;
|
||||
case 3: _t->Timeout(); break;
|
||||
case 4: _t->Delayed(); break;
|
||||
default: ;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const QMetaObjectExtraData CutyCapt::staticMetaObjectExtraData = {
|
||||
0, qt_static_metacall
|
||||
};
|
||||
|
||||
const QMetaObject CutyCapt::staticMetaObject = {
|
||||
{ &QObject::staticMetaObject, qt_meta_stringdata_CutyCapt,
|
||||
qt_meta_data_CutyCapt, &staticMetaObjectExtraData }
|
||||
};
|
||||
|
||||
#ifdef Q_NO_DATA_RELOCATION
|
||||
const QMetaObject &CutyCapt::getStaticMetaObject() { return staticMetaObject; }
|
||||
#endif //Q_NO_DATA_RELOCATION
|
||||
|
||||
const QMetaObject *CutyCapt::metaObject() const
|
||||
{
|
||||
return QObject::d_ptr->metaObject ? QObject::d_ptr->metaObject : &staticMetaObject;
|
||||
}
|
||||
|
||||
void *CutyCapt::qt_metacast(const char *_clname)
|
||||
{
|
||||
if (!_clname) return 0;
|
||||
if (!strcmp(_clname, qt_meta_stringdata_CutyCapt))
|
||||
return static_cast<void*>(const_cast< CutyCapt*>(this));
|
||||
return QObject::qt_metacast(_clname);
|
||||
}
|
||||
|
||||
int CutyCapt::qt_metacall(QMetaObject::Call _c, int _id, void **_a)
|
||||
{
|
||||
_id = QObject::qt_metacall(_c, _id, _a);
|
||||
if (_id < 0)
|
||||
return _id;
|
||||
if (_c == QMetaObject::InvokeMetaMethod) {
|
||||
if (_id < 5)
|
||||
qt_static_metacall(this, _c, _id, _a);
|
||||
_id -= 5;
|
||||
}
|
||||
return _id;
|
||||
}
|
||||
QT_END_MOC_NAMESPACE
|
||||
Binary file not shown.
@@ -1,12 +0,0 @@
|
||||
HowTo Install:
|
||||
|
||||
% sudo apt-get install subversion libqt4-webkit libqt4-dev g++
|
||||
% svn co https://cutycapt.svn.sourceforge.net/svnroot/cutycapt
|
||||
% cd cutycapt/CutyCapt
|
||||
% qmake
|
||||
% make
|
||||
|
||||
HowTo Use:
|
||||
% ./CutyCapt --url=http://www.example.org --out=example.png
|
||||
|
||||
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
""" erzeugt Vorschaubilder von Webseiten (Ersatz für CutyCapt + convert)
|
||||
|
||||
Eingabe ist eine Datei mit Zeilen der Form Kundennummer;URL
|
||||
Für jede Zeile wird die Seite in einem Browser ohne Fenster geladen und der
|
||||
sichtbare Bereich (Breite x Höhe aus der Sektion [webscan] der Konfiguration)
|
||||
als <Kundennummer>.<format> gespeichert.
|
||||
"""
|
||||
|
||||
__version__ = '0.1.0'
|
||||
__license__ = 'BSD'
|
||||
|
||||
import asyncio
|
||||
import configparser
|
||||
import csv
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
from playwright.async_api import async_playwright, Error as PlaywrightError
|
||||
|
||||
from utils import CompleteEnding
|
||||
|
||||
|
||||
def read_config(cfg_name, section):
|
||||
''' liest Breite, Höhe und Bildformat aus der Konfiguration '''
|
||||
cfg_path = os.path.expandvars(os.path.join('$PICCREATE_CFG',
|
||||
CompleteEnding(cfg_name, 'cfg')))
|
||||
if not os.path.exists(cfg_path):
|
||||
raise SystemExit('"' + cfg_path + '" does not exist. Please give the name of a config file!')
|
||||
config = configparser.RawConfigParser(strict=False)
|
||||
config.read(cfg_path, encoding='utf-8')
|
||||
if not config.has_section(section):
|
||||
raise SystemExit('section [' + section + '] is missing in ' + cfg_path)
|
||||
return {
|
||||
'width': config.getint(section, 'width'),
|
||||
'height': config.getint(section, 'height'),
|
||||
'format': config.get(section, 'format', fallback='jpg'),
|
||||
}
|
||||
|
||||
|
||||
def read_urls(infile):
|
||||
''' liest die Paare (Kundennummer, URL) aus der Eingabedatei.
|
||||
Eine Kopfzeile wird übersprungen, wenn ihre zweite Spalte keine URL ist.
|
||||
'''
|
||||
in_path = infile
|
||||
if not os.path.exists(in_path):
|
||||
in_path = os.path.expandvars(os.path.join('$PICCREATE_IN',
|
||||
CompleteEnding(infile, 'csv')))
|
||||
if not os.path.exists(in_path):
|
||||
raise SystemExit('"' + in_path + '" does not exist. Please give the name to a input file!')
|
||||
|
||||
jobs = []
|
||||
with open(in_path, encoding='utf-8', newline='') as fh:
|
||||
for n, row in enumerate(csv.reader(fh, delimiter=';')):
|
||||
row = [field.strip() for field in row]
|
||||
if not row or not row[0] or row[0].startswith('#'):
|
||||
continue
|
||||
url = row[-1] if len(row) > 1 else ''
|
||||
if n == 0 and '.' not in url:
|
||||
continue
|
||||
if url and '://' not in url:
|
||||
url = 'http://' + url
|
||||
jobs.append((row[0], url))
|
||||
return jobs
|
||||
|
||||
|
||||
async def capture(browser, customer_id, url, out_path, pic_config, timeout):
|
||||
''' lädt eine Seite und speichert den sichtbaren Bereich als Bild '''
|
||||
page = await browser.new_page(viewport={'width': pic_config['width'],
|
||||
'height': pic_config['height']})
|
||||
try:
|
||||
await page.goto(url, timeout=timeout, wait_until='load')
|
||||
kwargs = {'path': out_path}
|
||||
if pic_config['format'] in ('jpg', 'jpeg'):
|
||||
kwargs['type'] = 'jpeg'
|
||||
kwargs['quality'] = 90
|
||||
else:
|
||||
kwargs['type'] = 'png'
|
||||
await page.screenshot(**kwargs)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
|
||||
async def run(jobs, pic_config, options):
|
||||
out_dir = options.outdir
|
||||
timeout = int(options.timeout) * 1000
|
||||
semaphore = asyncio.Semaphore(int(options.jobs))
|
||||
failed = []
|
||||
|
||||
async with async_playwright() as p:
|
||||
launch_args = {}
|
||||
if options.browser:
|
||||
launch_args['executable_path'] = options.browser
|
||||
browser = await p.chromium.launch(**launch_args)
|
||||
|
||||
async def worker(customer_id, url):
|
||||
out_path = os.path.join(out_dir, customer_id + '.' + pic_config['format'])
|
||||
if not url:
|
||||
print(customer_id, 'has no url - skipped', file=sys.stderr)
|
||||
failed.append(customer_id)
|
||||
return
|
||||
if os.path.exists(out_path) and not options.force:
|
||||
if options.verbose:
|
||||
print(out_path, 'exists')
|
||||
return
|
||||
async with semaphore:
|
||||
if options.verbose:
|
||||
print('loading', customer_id, url)
|
||||
try:
|
||||
await capture(browser, customer_id, url, out_path, pic_config, timeout)
|
||||
print(out_path)
|
||||
except PlaywrightError as e:
|
||||
first_line = str(e).splitlines()[0]
|
||||
print(customer_id, url, 'failed:', first_line, file=sys.stderr)
|
||||
failed.append(customer_id)
|
||||
|
||||
await asyncio.gather(*(worker(cid, url) for cid, url in jobs))
|
||||
await browser.close()
|
||||
return failed
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.isdir(options.outdir):
|
||||
print('"' + options.outdir + '" does not exist. Please give the path to the directory for the pictures!')
|
||||
return 1
|
||||
pic_config = read_config(options.config, options.section)
|
||||
if options.format:
|
||||
pic_config['format'] = options.format
|
||||
jobs = read_urls(options.infile)
|
||||
if not jobs:
|
||||
print('no urls found in', options.infile)
|
||||
return 1
|
||||
failed = asyncio.run(run(jobs, pic_config, options))
|
||||
if failed:
|
||||
print(len(failed), 'of', len(jobs), 'pages failed:', ', '.join(failed), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = optparse.OptionParser(usage="""%prog [options]
|
||||
|
||||
Example usage
|
||||
-------------
|
||||
|
||||
Screenshots of all urls in in/kunden.csv (lines: customerid;url):
|
||||
$ %prog -c webscan -i kunden
|
||||
""", version='%prog ' + __version__)
|
||||
|
||||
parser.add_option("-c", "--cfg", dest="config", default="webscan",
|
||||
help="config file with a [webscan] section (width, height, format)",
|
||||
metavar="CONFIG_FILE")
|
||||
parser.add_option("--section", dest="section", default="webscan",
|
||||
help="section in the config file. default is webscan")
|
||||
parser.add_option("-i", "--infile", dest="infile",
|
||||
help="file with lines customerid;url")
|
||||
parser.add_option("-o", "--outdir", dest="outdir", default=os.path.expandvars('$PICCREATE_OUT'),
|
||||
help="directory where the pictures will be saved")
|
||||
parser.add_option("--format", dest="format",
|
||||
help="jpg or png, overrides the config")
|
||||
parser.add_option("-j", "--jobs", dest="jobs", default=4,
|
||||
help="number of pages loaded in parallel. default is 4")
|
||||
parser.add_option("-t", "--timeout", dest="timeout", default=20,
|
||||
help="seconds to wait for a page. default is 20")
|
||||
parser.add_option("-f", "--force", dest="force", action='store_true',
|
||||
help="create the pictures, no matter if they already exist")
|
||||
parser.add_option("-b", "--browser", dest="browser",
|
||||
default=os.environ.get('PICCREATE_BROWSER'),
|
||||
help="path to a chromium executable instead of the playwright one "
|
||||
"(default: $PICCREATE_BROWSER)")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action='store_true',
|
||||
help="print more information")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
if not options.infile:
|
||||
parser.error('please give an input file with -i')
|
||||
sys.exit(main())
|
||||
|
||||
# vim: set tw=80 ft=python ts=4 sw=4:
|
||||
@@ -0,0 +1 @@
|
||||
playwright
|
||||
Reference in New Issue
Block a user