diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..77ac754 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/bin/webscan b/bin/webscan deleted file mode 100755 index 220233b..0000000 --- a/bin/webscan +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -NUM=0 -QUEUE="" -MAX_NPROC=2 # default -REPLACE_CMD=0 # no replacement by default - -USAGE="A simple wrapper for generating jpg pictures out of svg-files in parallel. -Usage: `basename $0` [-h] [-i] [-j nb_jobs] command arg_list - -h Shows this help - -c give the path to the cfg file - -i give the file where the web-adresses are - -k chek if the output file already exist - -s set the sleeptime while there are too many simultanious jobs - -j Set number of simultanious jobs. Default is 2 - -o directory where the files should be moved to after creation. - Default ist PICCREATE_OUT - Examples: - `basename $0` -i file -c config - `basename $0` -j 3 -i file -c config" - -function cfg_parser { - INI_FILE=$1 - SECTION=$2 - eval `sed -n -e "/^\[$SECTION\]/,/^\s*\[/{/^[^;].*\=.*/p;}" < $INI_FILE` -} -function get_filepath { - # probiere alle möglichen Kombinationen wo das file stehen könnte je - # nachdem ob es mit oder ohne Endung, mit oder ohne den Komplettpfad - # gegeben wurde - FILENAME=$1 - ENDING=$2 - DEFAULT_DIR=$3 - if [ -s $FILENAME ] ; then - echo $FILENAME - elif [ -s $FILENAME.$ENDING ] ; then - echo "$FILENAME.$ENDING" - elif [ -s $DEFAULT_DIR/$FILENAME ] ; then - echo "$DEFAULT_DIR/$FILENAME" - elif [ -s $DEFAULT_DIR/$FILENAME.$ENDING ] ; then - echo "$DEFAULT_DIR/$FILENAME.$ENDING" - fi - return -} - -# Funktion die auf globalen Parametern beruht. -# überprüft ob alle notwendigen Angaben vorhanden sind -function check_params { - if [ -z "$INPUT_FILENAME" ]; then - echo "please give input file with -i." - exit 1 - fi - if [ -z "$CFG_FILENAME" ]; then - echo "please give config file with -c." - exit 1 - fi - - INPUT_FILE=`get_filepath $INPUT_FILENAME 'csv' $PICCREATE_WORK` - if [ ! -s $INPUT_FILE ] ; then - echo "-> input filename does not exist" - print "given was: -$INPUT_FILENAME-" - exit 1 - fi - CFG_PATH=`get_filepath $CFG_FILENAME 'cfg' $PICCREATE_CFG` - if [ ! -s $CFG_PATH ]; then - echo "-> config file with this name does not exist." - print "given was: -$CFG_FILENAME-" - exit 1 - fi - echo $INPUT_FILENAME - cfg_parser $CFG_PATH "$FILE_FORMAT" - echo 'using width '$width - echo 'using height '$height - if [ -z "$width" ]; then - echo "please give width=xxx in section [$FILE_FORMAT] in config $CFG_FILENAME." - exit 1 - fi - if [ -z "$height" ]; then - echo "please give height=xxx in section [$FILE_FORMAT] in config $CFG_FILENAME." - exit 1 - fi - FORMAT="$format" -} - - -# parse command line -if [ $# -eq 0 ]; then # must be at least one arg - echo "$USAGE" >&2 - exit 1 -fi - -CHECKFILE=1 -SLEEPTIME=0.5 -BACKGROUND='' -FILE_FORMAT='webscan' -OUTDIR=$PICCREATE_OUT -while getopts "j:i:kc:s:o:hc:f" OPT; do # "j:" waits for an argument "h" doesnt - case $OPT in - h) echo "$USAGE" - exit 0 ;; - j) MAX_NPROC=$OPTARG ;; - k) CHECKFILE=0 ;; - c) CFG_FILENAME=$OPTARG ;; - o) OUTDIR=$OPTARG ;; - s) SLEEPTIME=$OPTARG ;; - i) INPUT_FILENAME=$OPTARG ;; - f) FORMAT=$OPTARG ;; - *) echo "$USAGE" >&2; exit 1; # Default. - esac -done - -# Main program -echo Using $MAX_NPROC parallel threads -check_params - -# Starte die Abspeichern der ganzen Webseite als ganzes -echo $INPUT_FILE -for i in `cat $INPUT_FILENAME` -do - CUSTOMER_ID=`echo "$i" | sed -e 's/;.*//g'` - CUSTOMER_URL=`echo "$i" |sed -e 's/.*;//g'` - - FILE_PATH=$OUTDIR/$CUSTOMER_ID.$FORMAT - echo $FILE_PATH - wait - if [ -z $CUSTOMER_URL ] - then - echo 'i need a url!!' - echo $CUSTOMER_ID 'did not have an url, ist was set to www.syntops.de instead'>> logfile.txt - CUSTOMER_URL='www.syntops.de' - sleep - fi - if [ -s $FILE_PATH ] && [ $CHECKFILE -eq 1 ] - then - echo $FILE_PATH exists - else - CMDLINE="$CUTYCAPT/./CutyCapt --url=http://$CUSTOMER_URL --max-wait=20000 --min-width=800 --min-height=600 --out=$FILE_PATH" - $CMDLINE - wait - fi -done - -wait # wait for all processes to finish before exit -echo "...all Webpages have been saved!" -echo "" -echo "Converting the pictures..." -echo " " - - -# Zuschnitt der seite auf das benötigte Format -# Das Bild wir hierfür erst gleichmäßig auf die forgegebene Breite Skaliert -# und anschließend die unten herüberstehende Höhe abgeschnitten -echo $outdir -for i in `ls $OUTDIR| grep .$FORMAT ` -do - FILE_PATH_IN=$OUTDIR/$i - FILE_PATH_OUT=$OUTDIR/'beschnitten'/$i - - if [ -s $FILE_PATH_OUT ] && [ $CHECKFILE -eq 1 ] - then - echo $PIC_NAME exists - else - RESIZE="${width}" - SLICE="${width}x${height}" - CMDLINE="convert $FILE_PATH_IN -resize $RESIZE -crop $SLICE $FILE_PATH_OUT" - echo $CMDLINE - (echo $ID; $CMDLINE) & - - - while [ `ps clx|grep -w 'convert' |wc -l` -ge $MAX_NPROC ]; do - sleep $SLEEPTIME - done - fi - # auskommentiert da bereits runtergeladene seiten behalten werden sollen - #rm $FILE_PATH - -done - -wait # wait for all processes to finish before exit - - -rm $OUTDIR/'beschnitten'/*-[1-9].$FORMAT -for i in `ls $OUTDIR/'beschnitten'/` -do - first=`basename $i -0.jpg` - second=`basename $first .jpg` - mv $OUTDIR/'beschnitten'/$i $OUTDIR/'beschnitten'/$second.jpg - -done - - -# vi:set wrapmargin=0 ai showmatch ts=4 syntax=sh: diff --git a/bin/webscan.sh b/bin/webscan.sh index 1caec03..41f5892 100755 --- a/bin/webscan.sh +++ b/bin/webscan.sh @@ -1,4 +1,9 @@ #!/bin/bash . setenv.sh -export CUTYCAPT=$PICCREATE_LIBS/CutyCapt - $PICCREATE_BIN/webscan "$@" + +# Vorschaubilder von Webseiten erzeugen (Playwright, Python-venv im Projekt) +PYTHON=python3 +if [ -x $PICCREATE/.venv/bin/python ]; then + PYTHON=$PICCREATE/.venv/bin/python +fi +$PYTHON $PICCREATE_LIBS/webscan.py "$@" diff --git a/doc/README.md b/doc/README.md index d14e2be..4f87d45 100644 --- a/doc/README.md +++ b/doc/README.md @@ -19,16 +19,17 @@ Die Punkte werden über das ganze Wort fortlaufend nummeriert (1–32). ## Voraussetzungen -- Python 3 (keine weiteren Python-Pakete nötig) +- Python 3 (für `piccreate.py` und `svgconvert.py` keine weiteren Pakete nötig) - ImageMagick (`convert`) für die Umwandlung SVG → JPG/PNG mit `svgconvert.py` - `bash`, `bc`, `sed`, `grep` für die Font-Werkzeuge in `bin/` +- nur für Webseiten-Vorschauen (`webscan`): Playwright, siehe unten ## Verzeichnisse | Verzeichnis | Inhalt | |----------------|-----------------------------------------------------------| | `bin/` | Shell-Skripte (Aufruf, Umgebung, Font-Extraktion) | -| `libs/` | Python-Programme `piccreate.py`, `svgconvert.py`, Hilfen | +| `libs/` | Python-Programme `piccreate.py`, `svgconvert.py`, `webscan.py` | | `cfg/` | Auftrags-Konfigurationen (`*.cfg`) | | `in/` | Eingabedateien (`*.csv`) und Rohglyphen für neue Fonts | | `out/` | Ausgabe der erzeugten Bilder | @@ -143,6 +144,54 @@ ImageMagick um. `--format` wählt die Sektion aus der Konfiguration. | `-p` / `-v` | Fortschrittsbalken / ausführliche Ausgabe | | `-y` | Probelauf ohne Umwandlung | +## Vorschaubilder von Webseiten (webscan) + +`libs/webscan.py` lädt für jede Zeile einer Eingabedatei eine Webseite in +einem Browser ohne Fenster und speichert den sichtbaren Bereich als Bild +`.jpg`. Die Größe kommt aus der Sektion `[webscan]` der +Konfiguration (in `cfg/webscan.cfg`: 800×500). + +Einmalig einrichten (Python-Umgebung im Projektverzeichnis): + +```sh +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +.venv/bin/playwright install chromium +``` + +Statt des Playwright-Chromiums kann auch ein installiertes Chromium benutzt +werden: `-b /usr/bin/chromium` oder `export PICCREATE_BROWSER=/usr/bin/chromium`. + +Eingabedatei, z. B. `in/kunden.csv` (Kopfzeile optional, `http://` wird bei +Bedarf ergänzt): + +``` +Kennummer;URL +1001;www.example.org +1002;https://www.example.com/start +``` + +Aufruf: + +```sh +./webscan.sh -c webscan -i kunden +``` + +| Option | Bedeutung | +|-----------------|---------------------------------------------------------| +| `-c` | Konfiguration (Standard `webscan`) | +| `-i` | Eingabedatei aus `in/` oder mit Pfad | +| `-o` | Ausgabeverzeichnis (Standard `$PICCREATE_OUT`) | +| `--format` | `jpg` oder `png` (überschreibt die Konfiguration) | +| `-j` | Anzahl gleichzeitig geladener Seiten (Standard 4) | +| `-t` | Wartezeit pro Seite in Sekunden (Standard 20) | +| `-f` | vorhandene Bilder überschreiben | +| `-b` | Pfad zu einem eigenen Chromium | +| `-v` | ausführliche Ausgabe | + +Seiten ohne URL oder mit Ladefehler werden übersprungen und am Ende +aufgelistet; der Exit-Code ist dann 1. + ## Fonts Ein Font liegt in `fonts//` und besteht aus diff --git a/libs/CutyCapt/CutyCapt b/libs/CutyCapt/CutyCapt deleted file mode 100755 index d354d3a..0000000 Binary files a/libs/CutyCapt/CutyCapt and /dev/null differ diff --git a/libs/CutyCapt/CutyCapt.cpp b/libs/CutyCapt/CutyCapt.cpp deleted file mode 100644 index 32e6d19..0000000 --- a/libs/CutyCapt/CutyCapt.cpp +++ /dev/null @@ -1,662 +0,0 @@ -//////////////////////////////////////////////////////////////////// -// -// CutyCapt - A Qt WebKit Web Page Rendering Capture Utility -// -// Copyright (C) 2003-2010 Bjoern Hoehrmann -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// as published by the Free Software Foundation; either version 2 -// of the License, or (at your option) any later version. -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// $Id$ -// -//////////////////////////////////////////////////////////////////// - -#include -#include -#include -#include -#include -#include -#include -#include -#include "CutyCapt.hpp" - -#if QT_VERSION >= 0x040600 && 0 -#define CUTYCAPT_SCRIPT 1 -#endif - -#ifdef STATIC_PLUGINS - Q_IMPORT_PLUGIN(qjpeg) - Q_IMPORT_PLUGIN(qgif) - Q_IMPORT_PLUGIN(qtiff) - Q_IMPORT_PLUGIN(qsvg) - Q_IMPORT_PLUGIN(qmng) - Q_IMPORT_PLUGIN(qico) -#endif - -static struct _CutyExtMap { - CutyCapt::OutputFormat id; - const char* extension; - const char* identifier; -} const CutyExtMap[] = { - { CutyCapt::SvgFormat, ".svg", "svg" }, - { CutyCapt::PdfFormat, ".pdf", "pdf" }, - { CutyCapt::PsFormat, ".ps", "ps" }, - { CutyCapt::InnerTextFormat, ".txt", "itext" }, - { CutyCapt::HtmlFormat, ".html", "html" }, - { CutyCapt::RenderTreeFormat, ".rtree", "rtree" }, - { CutyCapt::JpegFormat, ".jpeg", "jpeg" }, - { CutyCapt::PngFormat, ".png", "png" }, - { CutyCapt::MngFormat, ".mng", "mng" }, - { CutyCapt::TiffFormat, ".tiff", "tiff" }, - { CutyCapt::GifFormat, ".gif", "gif" }, - { CutyCapt::BmpFormat, ".bmp", "bmp" }, - { CutyCapt::PpmFormat, ".ppm", "ppm" }, - { CutyCapt::XbmFormat, ".xbm", "xbm" }, - { CutyCapt::XpmFormat, ".xpm", "xpm" }, - { CutyCapt::OtherFormat, "", "" } -}; - -QString -CutyPage::chooseFile(QWebFrame* /*frame*/, const QString& /*suggestedFile*/) { - return QString::null; -} - -bool -CutyPage::javaScriptConfirm(QWebFrame* /*frame*/, const QString& /*msg*/) { - return true; -} - -bool -CutyPage::javaScriptPrompt(QWebFrame* /*frame*/, - const QString& /*msg*/, - const QString& /*defaultValue*/, - QString* /*result*/) { - return true; -} - -void -CutyPage::javaScriptConsoleMessage(const QString& /*message*/, - int /*lineNumber*/, - const QString& /*sourceID*/) { - // noop -} - -void -CutyPage::javaScriptAlert(QWebFrame* /*frame*/, const QString& msg) { - - if (mPrintAlerts) - qDebug() << "[alert]" << msg; - - if (mAlertString == msg) { - QTimer::singleShot(10, mCutyCapt, SLOT(Delayed())); - } -} - -QString -CutyPage::userAgentForUrl(const QUrl& url) const { - - if (!mUserAgent.isNull()) - return mUserAgent; - - return QWebPage::userAgentForUrl(url); -} - -void -CutyPage::setUserAgent(const QString& userAgent) { - mUserAgent = userAgent; -} - -void -CutyPage::setAlertString(const QString& alertString) { - mAlertString = alertString; -} - -QString -CutyPage::getAlertString() { - return mAlertString; -} - -void -CutyPage::setCutyCapt(CutyCapt* cutyCapt) { - mCutyCapt = cutyCapt; -} - -void -CutyPage::setPrintAlerts(bool printAlerts) { - mPrintAlerts = printAlerts; -} - -void -CutyPage::setAttribute(QWebSettings::WebAttribute option, - const QString& value) { - - if (value == "on") - settings()->setAttribute(option, true); - else if (value == "off") - settings()->setAttribute(option, false); - else - (void)0; // TODO: ... -} - -// TODO: Consider merging some of main() and CutyCap - -CutyCapt::CutyCapt(CutyPage* page, const QString& output, int delay, OutputFormat format, - const QString& scriptProp, const QString& scriptCode) { - mPage = page; - mOutput = output; - mDelay = delay; - mSawInitialLayout = false; - mSawDocumentComplete = false; - mFormat = format; - mScriptProp = scriptProp; - mScriptCode = scriptCode; - mScriptObj = new QObject(); - - // This is not really nice, but some restructuring work is - // needed anyway, so this should not be that bad for now. - mPage->setCutyCapt(this); -} - -void -CutyCapt::InitialLayoutCompleted() { - mSawInitialLayout = true; - - if (mSawInitialLayout && mSawDocumentComplete) - TryDelayedRender(); -} - -void -CutyCapt::DocumentComplete(bool /*ok*/) { - mSawDocumentComplete = true; - - if (mSawInitialLayout && mSawDocumentComplete) - TryDelayedRender(); -} - -void -CutyCapt::JavaScriptWindowObjectCleared() { - - if (!mScriptProp.isEmpty()) { - QVariant var = mPage->mainFrame()->evaluateJavaScript(mScriptProp); - QObject* obj = var.value(); - - if (obj == mScriptObj) - return; - - mPage->mainFrame()->addToJavaScriptWindowObject(mScriptProp, mScriptObj); - } - - mPage->mainFrame()->evaluateJavaScript(mScriptCode); - -} - -void -CutyCapt::TryDelayedRender() { - - if (!mPage->getAlertString().isEmpty()) - return; - - if (mDelay > 0) { - QTimer::singleShot(mDelay, this, SLOT(Delayed())); - return; - } - - saveSnapshot(); - QApplication::exit(); -} - -void -CutyCapt::Timeout() { - saveSnapshot(); - QApplication::exit(); -} - -void -CutyCapt::Delayed() { - saveSnapshot(); - QApplication::exit(); -} - -void -CutyCapt::saveSnapshot() { - QWebFrame *mainFrame = mPage->mainFrame(); - QPainter painter; - const char* format = NULL; - - for (int ix = 0; CutyExtMap[ix].id != OtherFormat; ++ix) - if (CutyExtMap[ix].id == mFormat) - format = CutyExtMap[ix].identifier; //, break; - - // TODO: sometimes contents/viewport can have size 0x0 - // in which case saving them will fail. This is likely - // the result of the method being called too early. So - // far I've been unable to find a workaround, except - // using --delay with some substantial wait time. I've - // tried to resize multiple time, make a fake render, - // check for other events... This is primarily a problem - // under my Ubuntu virtual machine. - - mPage->setViewportSize( mainFrame->contentsSize() ); - - switch (mFormat) { - case SvgFormat: { - QSvgGenerator svg; - svg.setFileName(mOutput); - svg.setSize(mPage->viewportSize()); - painter.begin(&svg); - mainFrame->render(&painter); - painter.end(); - break; - } - case PdfFormat: - case PsFormat: { - QPrinter printer; - printer.setPageSize(QPrinter::A4); - printer.setOutputFileName(mOutput); - // TODO: change quality here? - mainFrame->print(&printer); - break; - } - case RenderTreeFormat: - case InnerTextFormat: - case HtmlFormat: { - QFile file(mOutput); - file.open(QIODevice::WriteOnly | QIODevice::Text); - QTextStream s(&file); - s.setCodec("utf-8"); - s << (mFormat == RenderTreeFormat ? mainFrame->renderTreeDump() : - mFormat == InnerTextFormat ? mainFrame->toPlainText() : - mFormat == HtmlFormat ? mainFrame->toHtml() : - "bug"); - break; - } - default: { - QImage image(mPage->viewportSize(), QImage::Format_ARGB32); - painter.begin(&image); - mainFrame->render(&painter); - painter.end(); - // TODO: add quality - image.save(mOutput, format); - } - }; -} - -void -CaptHelp(void) { - printf("%s", - " -----------------------------------------------------------------------------\n" - " Usage: CutyCapt --url=http://www.example.org/ --out=localfile.png \n" - " -----------------------------------------------------------------------------\n" - " --help Print this help page and exit \n" - " --url= The URL to capture (http:...|file:...|...) \n" - " --out= The target file (.png|pdf|ps|svg|jpeg|...) \n" - " --out-format= Like extension in --out, overrides heuristic \n" -// " --out-quality= Output format quality from 1 to 100 \n" - " --min-width= Minimal width for the image (default: 800) \n" - " --min-height= Minimal height for the image (default: 600) \n" - " --max-wait= Don't wait more than (default: 90000, inf: 0)\n" - " --delay= After successful load, wait (default: 0) \n" -// " --user-styles= Location of user style sheet (deprecated) \n" - " --user-style-path= Location of user style sheet file, if any \n" - " --user-style-string= User style rules specified as text \n" - " --header=: request header; repeatable; some can't be set\n" - " --method= Specifies the request method (default: get) \n" - " --body-string= Unencoded request body (default: none) \n" - " --body-base64= Base64-encoded request body (default: none) \n" - " --app-name= appName used in User-Agent; default is none \n" - " --app-version= appVers used in User-Agent; default is none \n" - " --user-agent= Override the User-Agent header Qt would set \n" - " --javascript= JavaScript execution (default: on) \n" - " --java= Java execution (default: unknown) \n" - " --plugins= Plugin execution (default: unknown) \n" - " --private-browsing= Private browsing (default: unknown) \n" - " --auto-load-images= Automatic image loading (default: on) \n" - " --js-can-open-windows= Script can open windows? (default: unknown) \n" - " --js-can-access-clipboard= Script clipboard privs (default: unknown)\n" -#if QT_VERSION >= 0x040500 - " --print-backgrounds= Backgrounds in PDF/PS output (default: off) \n" - " --zoom-factor= Page zoom factor (default: no zooming) \n" - " --zoom-text-only= Whether to zoom only the text (default: off) \n" - " --http-proxy= Address for HTTP proxy server (default: none)\n" -#endif -#if CUTYCAPT_SCRIPT - " --inject-script= JavaScript that will be injected into pages \n" - " --script-object= Property to hold state for injected script \n" - " --expect-alert= Try waiting for alert(string) before capture \n" - " --debug-print-alerts Prints out alert(...) strings for debugging. \n" -#endif - " -----------------------------------------------------------------------------\n" - " is svg,ps,pdf,itext,html,rtree,png,jpeg,mng,tiff,gif,bmp,ppm,xbm,xpm \n" - " -----------------------------------------------------------------------------\n" -#if CUTYCAPT_SCRIPT - " The `inject-script` option can be used to inject script code into loaded web \n" - " pages. The code is called whenever the `javaScriptWindowObjectCleared` signal\n" - " is received. When `script-object` is set, an object under the specified name \n" - " will be available to the script to maintain state across page loads. When the\n" - " `expect-alert` option is specified, the shot will be taken when a script in- \n" - " vokes alert(string) with the string if that happens before `max-wait`. These \n" - " options effectively allow you to remote control the browser and the web page.\n" - " This an experimental and easily abused and misused feature. Use with caution.\n" - " -----------------------------------------------------------------------------\n" -#endif - " http://cutycapt.sf.net - (c) 2003-2010 Bjoern Hoehrmann - bjoern@hoehrmann.de\n" - ""); -} - -int -main(int argc, char *argv[]) { - - int argHelp = 0; - int argDelay = 0; - int argSilent = 0; - int argMinWidth = 800; - int argMinHeight = 600; - int argMaxWait = 90000; - int argVerbosity = 0; - - const char* argUrl = NULL; - const char* argUserStyle = NULL; - const char* argUserStylePath = NULL; - const char* argUserStyleString = NULL; - const char* argIconDbPath = NULL; - const char* argInjectScript = NULL; - const char* argScriptObject = NULL; - QString argOut; - - CutyCapt::OutputFormat format = CutyCapt::OtherFormat; - - QApplication app(argc, argv, true); - CutyPage page; - - QNetworkAccessManager::Operation method = - QNetworkAccessManager::GetOperation; - QByteArray body; - QNetworkRequest req; - QNetworkAccessManager manager; - - // Parse command line parameters - for (int ax = 1; ax < argc; ++ax) { - size_t nlen; - - const char* s = argv[ax]; - const char* value; - - // boolean options - if (strcmp("--silent", s) == 0) { - argSilent = 1; - continue; - - } else if (strcmp("--help", s) == 0) { - argHelp = 1; - break; - - } else if (strcmp("--verbose", s) == 0) { - argVerbosity++; - continue; - -#if CUTYCAPT_SCRIPT - } else if (strcmp("--debug-print-alerts", s) == 0) { - page.setPrintAlerts(true); - continue; -#endif - } - - value = strchr(s, '='); - - if (value == NULL) { - // TODO: error - argHelp = 1; - break; - } - - nlen = value++ - s; - - // --name=value options - if (strncmp("--url", s, nlen) == 0) { - argUrl = value; - - } else if (strncmp("--min-width", s, nlen) == 0) { - // TODO: add error checking here? - argMinWidth = (unsigned int)atoi(value); - - } else if (strncmp("--min-height", s, nlen) == 0) { - // TODO: add error checking here? - argMinHeight = (unsigned int)atoi(value); - - } else if (strncmp("--delay", s, nlen) == 0) { - // TODO: see above - argDelay = (unsigned int)atoi(value); - - } else if (strncmp("--max-wait", s, nlen) == 0) { - // TODO: see above - argMaxWait = (unsigned int)atoi(value); - - } else if (strncmp("--out", s, nlen) == 0) { - argOut = value; - - if (format == CutyCapt::OtherFormat) - for (int ix = 0; CutyExtMap[ix].id != CutyCapt::OtherFormat; ++ix) - if (argOut.endsWith(CutyExtMap[ix].extension)) - format = CutyExtMap[ix].id; //, break; - - } else if (strncmp("--user-styles", s, nlen) == 0) { - // This option is provided for backwards-compatibility only - argUserStyle = value; - - } else if (strncmp("--user-style-path", s, nlen) == 0) { - argUserStylePath = value; - - } else if (strncmp("--user-style-string", s, nlen) == 0) { - argUserStyleString = value; - - } else if (strncmp("--icon-database-path", s, nlen) == 0) { - argIconDbPath = value; - - } else if (strncmp("--auto-load-images", s, nlen) == 0) { - page.setAttribute(QWebSettings::AutoLoadImages, value); - - } else if (strncmp("--javascript", s, nlen) == 0) { - page.setAttribute(QWebSettings::JavascriptEnabled, value); - - } else if (strncmp("--java", s, nlen) == 0) { - page.setAttribute(QWebSettings::JavaEnabled, value); - - } else if (strncmp("--plugins", s, nlen) == 0) { - page.setAttribute(QWebSettings::PluginsEnabled, value); - - } else if (strncmp("--private-browsing", s, nlen) == 0) { - page.setAttribute(QWebSettings::PrivateBrowsingEnabled, value); - - } else if (strncmp("--js-can-open-windows", s, nlen) == 0) { - page.setAttribute(QWebSettings::JavascriptCanOpenWindows, value); - - } else if (strncmp("--js-can-access-clipboard", s, nlen) == 0) { - page.setAttribute(QWebSettings::JavascriptCanAccessClipboard, value); - - } else if (strncmp("--developer-extras", s, nlen) == 0) { - page.setAttribute(QWebSettings::DeveloperExtrasEnabled, value); - - } else if (strncmp("--links-included-in-focus-chain", s, nlen) == 0) { - page.setAttribute(QWebSettings::LinksIncludedInFocusChain, value); - -#if QT_VERSION >= 0x040500 - } else if (strncmp("--print-backgrounds", s, nlen) == 0) { - page.setAttribute(QWebSettings::PrintElementBackgrounds, value); - - } else if (strncmp("--zoom-factor", s, nlen) == 0) { - page.mainFrame()->setZoomFactor(QString(value).toFloat()); - - } else if (strncmp("--zoom-text-only", s, nlen) == 0) { - page.setAttribute(QWebSettings::ZoomTextOnly, value); - - } else if (strncmp("--http-proxy", s, nlen) == 0) { - QUrl p = QUrl::fromEncoded(value); - QNetworkProxy proxy = QNetworkProxy(QNetworkProxy::HttpProxy, - p.host(), p.port(80), p.userName(), p.password()); - manager.setProxy(proxy); - page.setNetworkAccessManager(&manager); -#endif - -#if CUTYCAPT_SCRIPT - } else if (strncmp("--inject-script", s, nlen) == 0) { - argInjectScript = value; - - } else if (strncmp("--script-object", s, nlen) == 0) { - argScriptObject = value; - - } else if (strncmp("--expect-alert", s, nlen) == 0) { - page.setAlertString(value); -#endif - - } else if (strncmp("--app-name", s, nlen) == 0) { - app.setApplicationName(value); - - } else if (strncmp("--app-version", s, nlen) == 0) { - app.setApplicationVersion(value); - - } else if (strncmp("--body-base64", s, nlen) == 0) { - body = QByteArray::fromBase64(value); - - } else if (strncmp("--body-string", s, nlen) == 0) { - body = QByteArray(value); - - } else if (strncmp("--user-agent", s, nlen) == 0) { - page.setUserAgent(value); - - } else if (strncmp("--out-format", s, nlen) == 0) { - for (int ix = 0; CutyExtMap[ix].id != CutyCapt::OtherFormat; ++ix) - if (strcmp(value, CutyExtMap[ix].identifier) == 0) - format = CutyExtMap[ix].id; //, break; - - if (format == CutyCapt::OtherFormat) { - // TODO: error - argHelp = 1; - break; - } - - } else if (strncmp("--header", s, nlen) == 0) { - const char* hv = strchr(value, ':'); - - if (hv == NULL) { - // TODO: error - argHelp = 1; - break; - } - - req.setRawHeader(QByteArray(value, hv - value), hv + 1); - - } else if (strncmp("--method", s, nlen) == 0) { - if (strcmp("value", "get") == 0) - method = QNetworkAccessManager::GetOperation; - else if (strcmp("value", "put") == 0) - method = QNetworkAccessManager::PutOperation; - else if (strcmp("value", "post") == 0) - method = QNetworkAccessManager::PostOperation; - else if (strcmp("value", "head") == 0) - method = QNetworkAccessManager::HeadOperation; - else - (void)0; // TODO: ... - - } else { - // TODO: error - argHelp = 1; - } - } - - if (argUrl == NULL || argOut == NULL || argHelp) { - CaptHelp(); - return EXIT_FAILURE; - } - - // This used to use QUrl(argUrl) but that escapes %hh sequences - // even though it should not, as URLs can assumed to be escaped. - req.setUrl( QUrl::fromEncoded(argUrl) ); - - QString scriptProp(argScriptObject); - QString scriptCode; - - if (argInjectScript) { - QFile file(argInjectScript); - if (file.open(QIODevice::ReadOnly)) { - QTextStream stream(&file); - stream.setCodec(QTextCodec::codecForName("UTF-8")); - stream.setAutoDetectUnicode(true); - scriptCode = stream.readAll(); - file.close(); - } - } - - CutyCapt main(&page, argOut, argDelay, format, scriptProp, scriptCode); - - app.connect(&page, - SIGNAL(loadFinished(bool)), - &main, - SLOT(DocumentComplete(bool))); - - app.connect(page.mainFrame(), - SIGNAL(initialLayoutCompleted()), - &main, - SLOT(InitialLayoutCompleted())); - - if (argMaxWait > 0) { - // TODO: Should this also register one for the application? - QTimer::singleShot(argMaxWait, &main, SLOT(Timeout())); - } - - if (argUserStyle != NULL) - // TODO: does this need any syntax checking? - page.settings()->setUserStyleSheetUrl( QUrl::fromEncoded(argUserStyle) ); - - if (argUserStylePath != NULL) { - page.settings()->setUserStyleSheetUrl( QUrl::fromLocalFile(argUserStylePath) ); - } - - if (argUserStyleString != NULL) { - QUrl data("data:text/css;charset=utf-8;base64," + - QByteArray(argUserStyleString).toBase64()); - page.settings()->setUserStyleSheetUrl( data ); - } - - if (argIconDbPath != NULL) - // TODO: does this need any syntax checking? - page.settings()->setIconDatabasePath(argUserStyle); - - // The documentation does not say, but it seems the mainFrame - // will never change, so we can set this here. Otherwise we'd - // have to set this in snapshot and trigger an update, which - // is not currently possible (Qt 4.4.0) as far as I can tell. - page.mainFrame()->setScrollBarPolicy(Qt::Horizontal, Qt::ScrollBarAlwaysOff); - page.mainFrame()->setScrollBarPolicy(Qt::Vertical, Qt::ScrollBarAlwaysOff); - page.setViewportSize( QSize(argMinWidth, argMinHeight) ); - -#if CUTYCAPT_SCRIPT - // javaScriptWindowObjectCleared does not get called on the - // initial load unless some JavaScript has been executed. - page.mainFrame()->evaluateJavaScript(QString("")); - - app.connect(page.mainFrame(), - SIGNAL(javaScriptWindowObjectCleared()), - &main, - SLOT(JavaScriptWindowObjectCleared())); -#endif - - if (!body.isNull()) - page.mainFrame()->load(req, method, body); - else - page.mainFrame()->load(req, method); - - return app.exec(); -} diff --git a/libs/CutyCapt/CutyCapt.hpp b/libs/CutyCapt/CutyCapt.hpp deleted file mode 100644 index b7c26eb..0000000 --- a/libs/CutyCapt/CutyCapt.hpp +++ /dev/null @@ -1,66 +0,0 @@ -#include - -class CutyCapt; -class CutyPage : public QWebPage { - Q_OBJECT - -public: - void setAttribute(QWebSettings::WebAttribute option, const QString& value); - void setUserAgent(const QString& userAgent); - void setAlertString(const QString& alertString); - void setPrintAlerts(bool printAlerts); - void setCutyCapt(CutyCapt* cutyCapt); - QString getAlertString(); - -protected: - QString chooseFile(QWebFrame *frame, const QString& suggestedFile); - void javaScriptConsoleMessage(const QString& message, int lineNumber, const QString& sourceID); - bool javaScriptPrompt(QWebFrame* frame, const QString& msg, const QString& defaultValue, QString* result); - void javaScriptAlert(QWebFrame* frame, const QString& msg); - bool javaScriptConfirm(QWebFrame* frame, const QString& msg); - QString userAgentForUrl(const QUrl& url) const; - QString mUserAgent; - QString mAlertString; - bool mPrintAlerts; - CutyCapt* mCutyCapt; -}; - -class CutyCapt : public QObject { - Q_OBJECT - -public: - - // TODO: This should really be elsewhere and be named differently - enum OutputFormat { SvgFormat, PdfFormat, PsFormat, InnerTextFormat, HtmlFormat, - RenderTreeFormat, PngFormat, JpegFormat, MngFormat, TiffFormat, GifFormat, - BmpFormat, PpmFormat, XbmFormat, XpmFormat, OtherFormat }; - - CutyCapt(CutyPage* page, - const QString& output, - int delay, - OutputFormat format, - const QString& scriptProp, - const QString& scriptCode); - -private slots: - void DocumentComplete(bool ok); - void InitialLayoutCompleted(); - void JavaScriptWindowObjectCleared(); - void Timeout(); - void Delayed(); - -private: - void TryDelayedRender(); - void saveSnapshot(); - bool mSawInitialLayout; - bool mSawDocumentComplete; - -protected: - QString mOutput; - int mDelay; - CutyPage* mPage; - OutputFormat mFormat; - QObject* mScriptObj; - QString mScriptProp; - QString mScriptCode; -}; diff --git a/libs/CutyCapt/CutyCapt.o b/libs/CutyCapt/CutyCapt.o deleted file mode 100644 index 41fc3ca..0000000 Binary files a/libs/CutyCapt/CutyCapt.o and /dev/null differ diff --git a/libs/CutyCapt/CutyCapt.pro b/libs/CutyCapt/CutyCapt.pro deleted file mode 100644 index 5064a20..0000000 --- a/libs/CutyCapt/CutyCapt.pro +++ /dev/null @@ -1,10 +0,0 @@ -QT += webkit svg network -SOURCES = CutyCapt.cpp -HEADERS = CutyCapt.hpp -CONFIG += qt console - -contains(CONFIG, static): { - QTPLUGIN += qjpeg qgif qsvg qmng qico qtiff - DEFINES += STATIC_PLUGINS -} - diff --git a/libs/CutyCapt/Makefile b/libs/CutyCapt/Makefile deleted file mode 100644 index cb7460d..0000000 --- a/libs/CutyCapt/Makefile +++ /dev/null @@ -1,228 +0,0 @@ -############################################################################# -# Makefile for building: CutyCapt -# Generated by qmake (2.01a) (Qt 4.8.1) on: Thu Aug 30 14:02:45 2012 -# Project: CutyCapt.pro -# Template: app -# Command: /usr/bin/qmake -o Makefile CutyCapt.pro -############################################################################# - -####### Compiler, tools and options - -CC = gcc -CXX = g++ -DEFINES = -DQT_WEBKIT -DQT_NO_DEBUG -DQT_WEBKIT_LIB -DQT_SVG_LIB -DQT_GUI_LIB -DQT_NETWORK_LIB -DQT_CORE_LIB -DQT_SHARED -CFLAGS = -pipe -O2 -D_REENTRANT -Wall -W $(DEFINES) -CXXFLAGS = -pipe -O2 -D_REENTRANT -Wall -W $(DEFINES) -INCPATH = -I/usr/share/qt4/mkspecs/linux-g++ -I. -I/usr/include/qt4/QtCore -I/usr/include/qt4/QtNetwork -I/usr/include/qt4/QtGui -I/usr/include/qt4/QtSvg -I/usr/include/qt4/QtWebKit -I/usr/include/qt4 -I. -LINK = g++ -LFLAGS = -Wl,-O1 -LIBS = $(SUBLIBS) -L/usr/lib/i386-linux-gnu -lQtWebKit -lQtSvg -lQtGui -lQtNetwork -lQtCore -lpthread -AR = ar cqs -RANLIB = -QMAKE = /usr/bin/qmake -TAR = tar -cf -COMPRESS = gzip -9f -COPY = cp -f -SED = sed -COPY_FILE = $(COPY) -COPY_DIR = $(COPY) -r -STRIP = strip -INSTALL_FILE = install -m 644 -p -INSTALL_DIR = $(COPY_DIR) -INSTALL_PROGRAM = install -m 755 -p -DEL_FILE = rm -f -SYMLINK = ln -f -s -DEL_DIR = rmdir -MOVE = mv -f -CHK_DIR_EXISTS= test -d -MKDIR = mkdir -p - -####### Output directory - -OBJECTS_DIR = ./ - -####### Files - -SOURCES = CutyCapt.cpp moc_CutyCapt.cpp -OBJECTS = CutyCapt.o \ - moc_CutyCapt.o -DIST = /usr/share/qt4/mkspecs/common/unix.conf \ - /usr/share/qt4/mkspecs/common/linux.conf \ - /usr/share/qt4/mkspecs/common/gcc-base.conf \ - /usr/share/qt4/mkspecs/common/gcc-base-unix.conf \ - /usr/share/qt4/mkspecs/common/g++-base.conf \ - /usr/share/qt4/mkspecs/common/g++-unix.conf \ - /usr/share/qt4/mkspecs/qconfig.pri \ - /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \ - /usr/share/qt4/mkspecs/features/qt_functions.prf \ - /usr/share/qt4/mkspecs/features/qt_config.prf \ - /usr/share/qt4/mkspecs/features/exclusive_builds.prf \ - /usr/share/qt4/mkspecs/features/default_pre.prf \ - /usr/share/qt4/mkspecs/features/release.prf \ - /usr/share/qt4/mkspecs/features/default_post.prf \ - /usr/share/qt4/mkspecs/features/qt.prf \ - /usr/share/qt4/mkspecs/features/unix/thread.prf \ - /usr/share/qt4/mkspecs/features/moc.prf \ - /usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ - /usr/share/qt4/mkspecs/features/warn_on.prf \ - /usr/share/qt4/mkspecs/features/resources.prf \ - /usr/share/qt4/mkspecs/features/uic.prf \ - /usr/share/qt4/mkspecs/features/yacc.prf \ - /usr/share/qt4/mkspecs/features/lex.prf \ - /usr/share/qt4/mkspecs/features/include_source_dir.prf \ - CutyCapt.pro -QMAKE_TARGET = CutyCapt -DESTDIR = -TARGET = CutyCapt - -first: all -####### Implicit rules - -.SUFFIXES: .o .c .cpp .cc .cxx .C - -.cpp.o: - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" - -.cc.o: - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" - -.cxx.o: - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" - -.C.o: - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" - -.c.o: - $(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<" - -####### Build rules - -all: Makefile $(TARGET) - -$(TARGET): $(OBJECTS) - $(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(OBJCOMP) $(LIBS) - -Makefile: CutyCapt.pro /usr/share/qt4/mkspecs/linux-g++/qmake.conf /usr/share/qt4/mkspecs/common/unix.conf \ - /usr/share/qt4/mkspecs/common/linux.conf \ - /usr/share/qt4/mkspecs/common/gcc-base.conf \ - /usr/share/qt4/mkspecs/common/gcc-base-unix.conf \ - /usr/share/qt4/mkspecs/common/g++-base.conf \ - /usr/share/qt4/mkspecs/common/g++-unix.conf \ - /usr/share/qt4/mkspecs/qconfig.pri \ - /usr/share/qt4/mkspecs/modules/qt_webkit_version.pri \ - /usr/share/qt4/mkspecs/features/qt_functions.prf \ - /usr/share/qt4/mkspecs/features/qt_config.prf \ - /usr/share/qt4/mkspecs/features/exclusive_builds.prf \ - /usr/share/qt4/mkspecs/features/default_pre.prf \ - /usr/share/qt4/mkspecs/features/release.prf \ - /usr/share/qt4/mkspecs/features/default_post.prf \ - /usr/share/qt4/mkspecs/features/qt.prf \ - /usr/share/qt4/mkspecs/features/unix/thread.prf \ - /usr/share/qt4/mkspecs/features/moc.prf \ - /usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ - /usr/share/qt4/mkspecs/features/warn_on.prf \ - /usr/share/qt4/mkspecs/features/resources.prf \ - /usr/share/qt4/mkspecs/features/uic.prf \ - /usr/share/qt4/mkspecs/features/yacc.prf \ - /usr/share/qt4/mkspecs/features/lex.prf \ - /usr/share/qt4/mkspecs/features/include_source_dir.prf \ - /usr/lib/i386-linux-gnu/libQtWebKit.prl \ - /usr/lib/i386-linux-gnu/libQtSvg.prl \ - /usr/lib/i386-linux-gnu/libQtGui.prl \ - /usr/lib/i386-linux-gnu/libQtNetwork.prl \ - /usr/lib/i386-linux-gnu/libQtCore.prl - $(QMAKE) -o Makefile CutyCapt.pro -/usr/share/qt4/mkspecs/common/unix.conf: -/usr/share/qt4/mkspecs/common/linux.conf: -/usr/share/qt4/mkspecs/common/gcc-base.conf: -/usr/share/qt4/mkspecs/common/gcc-base-unix.conf: -/usr/share/qt4/mkspecs/common/g++-base.conf: -/usr/share/qt4/mkspecs/common/g++-unix.conf: -/usr/share/qt4/mkspecs/qconfig.pri: -/usr/share/qt4/mkspecs/modules/qt_webkit_version.pri: -/usr/share/qt4/mkspecs/features/qt_functions.prf: -/usr/share/qt4/mkspecs/features/qt_config.prf: -/usr/share/qt4/mkspecs/features/exclusive_builds.prf: -/usr/share/qt4/mkspecs/features/default_pre.prf: -/usr/share/qt4/mkspecs/features/release.prf: -/usr/share/qt4/mkspecs/features/default_post.prf: -/usr/share/qt4/mkspecs/features/qt.prf: -/usr/share/qt4/mkspecs/features/unix/thread.prf: -/usr/share/qt4/mkspecs/features/moc.prf: -/usr/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf: -/usr/share/qt4/mkspecs/features/warn_on.prf: -/usr/share/qt4/mkspecs/features/resources.prf: -/usr/share/qt4/mkspecs/features/uic.prf: -/usr/share/qt4/mkspecs/features/yacc.prf: -/usr/share/qt4/mkspecs/features/lex.prf: -/usr/share/qt4/mkspecs/features/include_source_dir.prf: -/usr/lib/i386-linux-gnu/libQtWebKit.prl: -/usr/lib/i386-linux-gnu/libQtSvg.prl: -/usr/lib/i386-linux-gnu/libQtGui.prl: -/usr/lib/i386-linux-gnu/libQtNetwork.prl: -/usr/lib/i386-linux-gnu/libQtCore.prl: -qmake: FORCE - @$(QMAKE) -o Makefile CutyCapt.pro - -dist: - @$(CHK_DIR_EXISTS) .tmp/CutyCapt1.0.0 || $(MKDIR) .tmp/CutyCapt1.0.0 - $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/CutyCapt1.0.0/ && $(COPY_FILE) --parents CutyCapt.hpp .tmp/CutyCapt1.0.0/ && $(COPY_FILE) --parents CutyCapt.cpp .tmp/CutyCapt1.0.0/ && (cd `dirname .tmp/CutyCapt1.0.0` && $(TAR) CutyCapt1.0.0.tar CutyCapt1.0.0 && $(COMPRESS) CutyCapt1.0.0.tar) && $(MOVE) `dirname .tmp/CutyCapt1.0.0`/CutyCapt1.0.0.tar.gz . && $(DEL_FILE) -r .tmp/CutyCapt1.0.0 - - -clean:compiler_clean - -$(DEL_FILE) $(OBJECTS) - -$(DEL_FILE) *~ core *.core - - -####### Sub-libraries - -distclean: clean - -$(DEL_FILE) $(TARGET) - -$(DEL_FILE) Makefile - - -check: first - -mocclean: compiler_moc_header_clean compiler_moc_source_clean - -mocables: compiler_moc_header_make_all compiler_moc_source_make_all - -compiler_moc_header_make_all: moc_CutyCapt.cpp -compiler_moc_header_clean: - -$(DEL_FILE) moc_CutyCapt.cpp -moc_CutyCapt.cpp: CutyCapt.hpp - /usr/bin/moc-qt4 $(DEFINES) $(INCPATH) CutyCapt.hpp -o moc_CutyCapt.cpp - -compiler_rcc_make_all: -compiler_rcc_clean: -compiler_image_collection_make_all: qmake_image_collection.cpp -compiler_image_collection_clean: - -$(DEL_FILE) qmake_image_collection.cpp -compiler_moc_source_make_all: -compiler_moc_source_clean: -compiler_uic_make_all: -compiler_uic_clean: -compiler_yacc_decl_make_all: -compiler_yacc_decl_clean: -compiler_yacc_impl_make_all: -compiler_yacc_impl_clean: -compiler_lex_make_all: -compiler_lex_clean: -compiler_clean: compiler_moc_header_clean - -####### Compile - -CutyCapt.o: CutyCapt.cpp CutyCapt.hpp - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o CutyCapt.o CutyCapt.cpp - -moc_CutyCapt.o: moc_CutyCapt.cpp - $(CXX) -c $(CXXFLAGS) $(INCPATH) -o moc_CutyCapt.o moc_CutyCapt.cpp - -####### Install - -install: FORCE - -uninstall: FORCE - -FORCE: - diff --git a/libs/CutyCapt/google.jpg b/libs/CutyCapt/google.jpg deleted file mode 100644 index e69de29..0000000 diff --git a/libs/CutyCapt/moc_CutyCapt.cpp b/libs/CutyCapt/moc_CutyCapt.cpp deleted file mode 100644 index e2bac5c..0000000 --- a/libs/CutyCapt/moc_CutyCapt.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/**************************************************************************** -** Meta object code from reading C++ file 'CutyCapt.hpp' -** -** Created: Thu Aug 30 14:03:03 2012 -** by: The Qt Meta Object Compiler version 63 (Qt 4.8.1) -** -** WARNING! All changes made in this file will be lost! -*****************************************************************************/ - -#include "CutyCapt.hpp" -#if !defined(Q_MOC_OUTPUT_REVISION) -#error "The header file 'CutyCapt.hpp' doesn't include ." -#elif Q_MOC_OUTPUT_REVISION != 63 -#error "This file was generated using the moc from 4.8.1. It" -#error "cannot be used with the include files from this version of Qt." -#error "(The moc has changed too much.)" -#endif - -QT_BEGIN_MOC_NAMESPACE -static const uint qt_meta_data_CutyPage[] = { - - // content: - 6, // revision - 0, // classname - 0, 0, // classinfo - 0, 0, // methods - 0, 0, // properties - 0, 0, // enums/sets - 0, 0, // constructors - 0, // flags - 0, // signalCount - - 0 // eod -}; - -static const char qt_meta_stringdata_CutyPage[] = { - "CutyPage\0" -}; - -void CutyPage::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a) -{ - Q_UNUSED(_o); - Q_UNUSED(_id); - Q_UNUSED(_c); - Q_UNUSED(_a); -} - -const QMetaObjectExtraData CutyPage::staticMetaObjectExtraData = { - 0, qt_static_metacall -}; - -const QMetaObject CutyPage::staticMetaObject = { - { &QWebPage::staticMetaObject, qt_meta_stringdata_CutyPage, - qt_meta_data_CutyPage, &staticMetaObjectExtraData } -}; - -#ifdef Q_NO_DATA_RELOCATION -const QMetaObject &CutyPage::getStaticMetaObject() { return staticMetaObject; } -#endif //Q_NO_DATA_RELOCATION - -const QMetaObject *CutyPage::metaObject() const -{ - return QObject::d_ptr->metaObject ? QObject::d_ptr->metaObject : &staticMetaObject; -} - -void *CutyPage::qt_metacast(const char *_clname) -{ - if (!_clname) return 0; - if (!strcmp(_clname, qt_meta_stringdata_CutyPage)) - return static_cast(const_cast< CutyPage*>(this)); - return QWebPage::qt_metacast(_clname); -} - -int CutyPage::qt_metacall(QMetaObject::Call _c, int _id, void **_a) -{ - _id = QWebPage::qt_metacall(_c, _id, _a); - if (_id < 0) - return _id; - return _id; -} -static const uint qt_meta_data_CutyCapt[] = { - - // content: - 6, // revision - 0, // classname - 0, 0, // classinfo - 5, 14, // methods - 0, 0, // properties - 0, 0, // enums/sets - 0, 0, // constructors - 0, // flags - 0, // signalCount - - // slots: signature, parameters, type, tag, flags - 13, 10, 9, 9, 0x08, - 36, 9, 9, 9, 0x08, - 61, 9, 9, 9, 0x08, - 93, 9, 9, 9, 0x08, - 103, 9, 9, 9, 0x08, - - 0 // eod -}; - -static const char qt_meta_stringdata_CutyCapt[] = { - "CutyCapt\0\0ok\0DocumentComplete(bool)\0" - "InitialLayoutCompleted()\0" - "JavaScriptWindowObjectCleared()\0" - "Timeout()\0Delayed()\0" -}; - -void CutyCapt::qt_static_metacall(QObject *_o, QMetaObject::Call _c, int _id, void **_a) -{ - if (_c == QMetaObject::InvokeMetaMethod) { - Q_ASSERT(staticMetaObject.cast(_o)); - CutyCapt *_t = static_cast(_o); - switch (_id) { - case 0: _t->DocumentComplete((*reinterpret_cast< bool(*)>(_a[1]))); break; - case 1: _t->InitialLayoutCompleted(); break; - case 2: _t->JavaScriptWindowObjectCleared(); break; - case 3: _t->Timeout(); break; - case 4: _t->Delayed(); break; - default: ; - } - } -} - -const QMetaObjectExtraData CutyCapt::staticMetaObjectExtraData = { - 0, qt_static_metacall -}; - -const QMetaObject CutyCapt::staticMetaObject = { - { &QObject::staticMetaObject, qt_meta_stringdata_CutyCapt, - qt_meta_data_CutyCapt, &staticMetaObjectExtraData } -}; - -#ifdef Q_NO_DATA_RELOCATION -const QMetaObject &CutyCapt::getStaticMetaObject() { return staticMetaObject; } -#endif //Q_NO_DATA_RELOCATION - -const QMetaObject *CutyCapt::metaObject() const -{ - return QObject::d_ptr->metaObject ? QObject::d_ptr->metaObject : &staticMetaObject; -} - -void *CutyCapt::qt_metacast(const char *_clname) -{ - if (!_clname) return 0; - if (!strcmp(_clname, qt_meta_stringdata_CutyCapt)) - return static_cast(const_cast< CutyCapt*>(this)); - return QObject::qt_metacast(_clname); -} - -int CutyCapt::qt_metacall(QMetaObject::Call _c, int _id, void **_a) -{ - _id = QObject::qt_metacall(_c, _id, _a); - if (_id < 0) - return _id; - if (_c == QMetaObject::InvokeMetaMethod) { - if (_id < 5) - qt_static_metacall(this, _c, _id, _a); - _id -= 5; - } - return _id; -} -QT_END_MOC_NAMESPACE diff --git a/libs/CutyCapt/moc_CutyCapt.o b/libs/CutyCapt/moc_CutyCapt.o deleted file mode 100644 index db4d7af..0000000 Binary files a/libs/CutyCapt/moc_CutyCapt.o and /dev/null differ diff --git a/libs/CutyCapt/readme.txt b/libs/CutyCapt/readme.txt deleted file mode 100644 index e073f1e..0000000 --- a/libs/CutyCapt/readme.txt +++ /dev/null @@ -1,12 +0,0 @@ -HowTo Install: - -% sudo apt-get install subversion libqt4-webkit libqt4-dev g++ -% svn co https://cutycapt.svn.sourceforge.net/svnroot/cutycapt -% cd cutycapt/CutyCapt -% qmake -% make - -HowTo Use: -% ./CutyCapt --url=http://www.example.org --out=example.png - - diff --git a/libs/webscan.py b/libs/webscan.py new file mode 100644 index 0000000..ba9e848 --- /dev/null +++ b/libs/webscan.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" erzeugt Vorschaubilder von Webseiten (Ersatz für CutyCapt + convert) + +Eingabe ist eine Datei mit Zeilen der Form Kundennummer;URL +Für jede Zeile wird die Seite in einem Browser ohne Fenster geladen und der +sichtbare Bereich (Breite x Höhe aus der Sektion [webscan] der Konfiguration) +als . gespeichert. +""" + +__version__ = '0.1.0' +__license__ = 'BSD' + +import asyncio +import configparser +import csv +import optparse +import os +import sys + +from playwright.async_api import async_playwright, Error as PlaywrightError + +from utils import CompleteEnding + + +def read_config(cfg_name, section): + ''' liest Breite, Höhe und Bildformat aus der Konfiguration ''' + cfg_path = os.path.expandvars(os.path.join('$PICCREATE_CFG', + CompleteEnding(cfg_name, 'cfg'))) + if not os.path.exists(cfg_path): + raise SystemExit('"' + cfg_path + '" does not exist. Please give the name of a config file!') + config = configparser.RawConfigParser(strict=False) + config.read(cfg_path, encoding='utf-8') + if not config.has_section(section): + raise SystemExit('section [' + section + '] is missing in ' + cfg_path) + return { + 'width': config.getint(section, 'width'), + 'height': config.getint(section, 'height'), + 'format': config.get(section, 'format', fallback='jpg'), + } + + +def read_urls(infile): + ''' liest die Paare (Kundennummer, URL) aus der Eingabedatei. + Eine Kopfzeile wird übersprungen, wenn ihre zweite Spalte keine URL ist. + ''' + in_path = infile + if not os.path.exists(in_path): + in_path = os.path.expandvars(os.path.join('$PICCREATE_IN', + CompleteEnding(infile, 'csv'))) + if not os.path.exists(in_path): + raise SystemExit('"' + in_path + '" does not exist. Please give the name to a input file!') + + jobs = [] + with open(in_path, encoding='utf-8', newline='') as fh: + for n, row in enumerate(csv.reader(fh, delimiter=';')): + row = [field.strip() for field in row] + if not row or not row[0] or row[0].startswith('#'): + continue + url = row[-1] if len(row) > 1 else '' + if n == 0 and '.' not in url: + continue + if url and '://' not in url: + url = 'http://' + url + jobs.append((row[0], url)) + return jobs + + +async def capture(browser, customer_id, url, out_path, pic_config, timeout): + ''' lädt eine Seite und speichert den sichtbaren Bereich als Bild ''' + page = await browser.new_page(viewport={'width': pic_config['width'], + 'height': pic_config['height']}) + try: + await page.goto(url, timeout=timeout, wait_until='load') + kwargs = {'path': out_path} + if pic_config['format'] in ('jpg', 'jpeg'): + kwargs['type'] = 'jpeg' + kwargs['quality'] = 90 + else: + kwargs['type'] = 'png' + await page.screenshot(**kwargs) + finally: + await page.close() + + +async def run(jobs, pic_config, options): + out_dir = options.outdir + timeout = int(options.timeout) * 1000 + semaphore = asyncio.Semaphore(int(options.jobs)) + failed = [] + + async with async_playwright() as p: + launch_args = {} + if options.browser: + launch_args['executable_path'] = options.browser + browser = await p.chromium.launch(**launch_args) + + async def worker(customer_id, url): + out_path = os.path.join(out_dir, customer_id + '.' + pic_config['format']) + if not url: + print(customer_id, 'has no url - skipped', file=sys.stderr) + failed.append(customer_id) + return + if os.path.exists(out_path) and not options.force: + if options.verbose: + print(out_path, 'exists') + return + async with semaphore: + if options.verbose: + print('loading', customer_id, url) + try: + await capture(browser, customer_id, url, out_path, pic_config, timeout) + print(out_path) + except PlaywrightError as e: + first_line = str(e).splitlines()[0] + print(customer_id, url, 'failed:', first_line, file=sys.stderr) + failed.append(customer_id) + + await asyncio.gather(*(worker(cid, url) for cid, url in jobs)) + await browser.close() + return failed + + +def main(): + if not os.path.isdir(options.outdir): + print('"' + options.outdir + '" does not exist. Please give the path to the directory for the pictures!') + return 1 + pic_config = read_config(options.config, options.section) + if options.format: + pic_config['format'] = options.format + jobs = read_urls(options.infile) + if not jobs: + print('no urls found in', options.infile) + return 1 + failed = asyncio.run(run(jobs, pic_config, options)) + if failed: + print(len(failed), 'of', len(jobs), 'pages failed:', ', '.join(failed), file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + parser = optparse.OptionParser(usage="""%prog [options] + +Example usage +------------- + +Screenshots of all urls in in/kunden.csv (lines: customerid;url): + $ %prog -c webscan -i kunden +""", version='%prog ' + __version__) + + parser.add_option("-c", "--cfg", dest="config", default="webscan", + help="config file with a [webscan] section (width, height, format)", + metavar="CONFIG_FILE") + parser.add_option("--section", dest="section", default="webscan", + help="section in the config file. default is webscan") + parser.add_option("-i", "--infile", dest="infile", + help="file with lines customerid;url") + parser.add_option("-o", "--outdir", dest="outdir", default=os.path.expandvars('$PICCREATE_OUT'), + help="directory where the pictures will be saved") + parser.add_option("--format", dest="format", + help="jpg or png, overrides the config") + parser.add_option("-j", "--jobs", dest="jobs", default=4, + help="number of pages loaded in parallel. default is 4") + parser.add_option("-t", "--timeout", dest="timeout", default=20, + help="seconds to wait for a page. default is 20") + parser.add_option("-f", "--force", dest="force", action='store_true', + help="create the pictures, no matter if they already exist") + parser.add_option("-b", "--browser", dest="browser", + default=os.environ.get('PICCREATE_BROWSER'), + help="path to a chromium executable instead of the playwright one " + "(default: $PICCREATE_BROWSER)") + parser.add_option("-v", "--verbose", dest="verbose", action='store_true', + help="print more information") + + (options, args) = parser.parse_args() + if not options.infile: + parser.error('please give an input file with -i') + sys.exit(main()) + +# vim: set tw=80 ft=python ts=4 sw=4: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..508a5f4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +playwright