Commits

David Mugnai committed 5f3198c

First bunch of code

No UI, just a cache for apod

  • Participants
  • Parent commits 953eb30

Comments (0)

Files changed (10)

File apodviewer.pro

+######################################################################
+# Automatically generated by qmake (2.01a) Sat Sep 22 17:53:10 2012
+######################################################################
+
+TEMPLATE = app
+TARGET = apod
+
+QT += widgets network concurrent
+
+# Input
+HEADERS += fsclient.h index.h metadata.h netclient.h
+SOURCES += fsclient.cpp \
+           index.cpp \
+           main.cpp \
+           metadata.cpp \
+           netclient.cpp
+
+QMAKE_CXXFLAGS += -std=c++0x -g
+LIBS += -lhtmlcxx
+
+MOC_DIR = ./mocs

File fsclient.cpp

+#include "fsclient.h"
+#include <QtDebug>
+
+APODFSCrawler::APODFSCrawler(QDir path, QObject *parent) : QObject(parent), path(path) {
+}
+
+bool APODFSCrawler::exists(const ImageMetaData& metadata) {
+    QString prefix = filePrefix(metadata);
+    QFile f(path.absoluteFilePath(prefix + ".dump"));
+    return f.exists();
+}
+
+QString APODFSCrawler::filePrefix(const ImageMetaData& metadata) {
+    return metadata.date.toString("ddMMyy");
+}
+
+void APODFSCrawler::run() {
+    qDebug() << "FSCrawler started";
+    QStringList filters("*.dump");
+    ImageMetaData metadata;
+    auto entries = path.entryInfoList(filters);
+    for(auto& finfo : entries) {
+        QFile image(path.absoluteFilePath(finfo.baseName() + ".jpg"));
+        if(!image.exists()) {
+            continue;
+        }
+        QFile dump(finfo.absoluteFilePath());
+        dump.open(QIODevice::ReadOnly);
+        QDataStream reader(&dump);
+        reader >> metadata;
+        dump.close();
+        emit imageFound(metadata);
+    }
+    emit finished();
+}
+
+void APODFSCrawler::save(const ImageMetaData& metadata, const QImage image) {
+    QString prefix = filePrefix(metadata);
+
+    QFile meta(path.absoluteFilePath(prefix + ".dump.temp"));
+    meta.open(QIODevice::WriteOnly);
+    QDataStream writer(&meta);
+    writer << metadata;
+    meta.close();
+
+    image.save(path.absoluteFilePath(prefix + ".jpg"));
+
+    meta.rename(path.absoluteFilePath(prefix + ".dump"));
+}
+#include <QObject>
+#include <QDir>
+#include <QImage>
+#include "metadata.h"
+
+class APODFSCrawler : public QObject {
+    Q_OBJECT
+public:
+    APODFSCrawler(QDir path, QObject *parent=0);
+    bool exists(const ImageMetaData&);
+    void save(const ImageMetaData&, const QImage);
+
+private:
+    QDir path;
+    QString filePrefix(const ImageMetaData&);
+
+signals:
+    void imageFound(ImageMetaData);
+    void finished();
+
+public slots:
+    void run();
+};
+#include "index.h"
+#include "netclient.h"
+#include <QtWidgets/QApplication>
+
+class AsyncFSCrawler : public QThread {
+public:
+    AsyncFSCrawler(APODFSCrawler *fs, QObject *parent=0);
+    void run();
+private:
+    APODFSCrawler *fs;
+};
+
+AsyncFSCrawler::AsyncFSCrawler(APODFSCrawler *fs, QObject *parent) : QThread(parent), fs(fs) {
+    fs->moveToThread(this);
+}
+void AsyncFSCrawler::run() {
+    fs->run();
+    fs->moveToThread(QApplication::instance()->thread());
+}
+
+APODIndexer::APODIndexer(QDir cache, QObject *parent) : QObject(parent), cache(cache), state(NOT_READY) {
+    manager = new QNetworkAccessManager(this);
+}
+
+void APODIndexer::update() {
+    auto t = new AsyncFSCrawler(&cache);
+    QObject::connect(
+        &cache, SIGNAL(imageFound(ImageMetaData)),
+        this, SLOT(imageFound(ImageMetaData)));
+    QObject::connect(
+        &cache, SIGNAL(finished()),
+        this, SLOT(fsCrawlerFinished()));
+    QObject::connect(
+        t, SIGNAL(finished()),
+        t, SLOT(deleteLater()) );
+    t->start();
+
+    auto index = new APODMainIndexDownloader(manager);
+    QObject::connect(
+        index, SIGNAL(indexDownloaded(QList<ImageMetaData*>)),
+        this, SLOT(indexDownloaded(QList<ImageMetaData*>)));
+}
+
+void APODIndexer::fsCrawlerFinished() {
+    if(state == NOT_READY) {
+        state = PARTIAL;
+    }
+    else {
+        mergeIndexes();
+    }
+}
+
+void APODIndexer::mergeIndexes() {
+    qDebug() << "time to merge!" << mainIndex.size() << remoteIndex.size();
+    state = COMPLETE;
+    for(auto v : remoteIndex) {
+        if(mainIndex.contains(v->pageUrl))
+            continue;
+        if(v->date.year() == 2012 && v->date.month() == 9) {
+            auto d = new APODImageMetaDataDownloader(manager, v);
+            QObject::connect(
+                d, SIGNAL(finished(ImageMetaData*)),
+                this, SLOT(imageMetaDataReady(ImageMetaData*)));
+        }
+    }
+    remoteIndex.clear();
+}
+
+void APODIndexer::imageFound(ImageMetaData metadata) {
+    qDebug() << "image found" << metadata.title;
+    mainIndex[metadata.pageUrl] = metadata;
+}
+
+void APODIndexer::indexDownloaded(QList<ImageMetaData*> index) {
+    qDebug() << "remote index downloaded" << index.size();
+    remoteIndex = index;
+
+    if(state == NOT_READY) {
+        state = PARTIAL;
+        // in order to not waste bandwith we wait for the fs crawler to
+        // complete; the only exception is for the first image, there is a good
+        // chance that we need it just now.
+        if(!cache.exists(*index[0])) {
+            auto d = new APODImageMetaDataDownloader(manager, index[0]);
+            QObject::connect(
+                d, SIGNAL(finished(ImageMetaData*)),
+                this, SLOT(imageMetaDataReady(ImageMetaData*)));
+        }
+    }
+    else {
+        mergeIndexes();
+    }
+}
+
+void APODIndexer::imageMetaDataReady(ImageMetaData *img) {
+    qDebug() << "image meta data ready" << img->date << img->title;
+    auto d = new APODImageDownloader(manager, img);
+    QObject::connect(
+        d, SIGNAL(finished(ImageMetaData*, QImage)),
+        this, SLOT(imageDownloaded(ImageMetaData*, QImage)));
+}
+
+void APODIndexer::imageDownloaded(ImageMetaData *i, QImage img) {
+    qDebug() << "image downloaded... writing in cache";
+    cache.save(*i, img);
+    imageFound(*i);
+    delete i;
+}
+
+#ifndef INDEX_H
+#define INDEX_H
+
+#include <QObject>
+#include <QImage>
+#include <QtNetwork>
+#include "metadata.h"
+#include "fsclient.h"
+
+class APODIndexer : public QObject {
+    Q_OBJECT
+public:
+    APODIndexer(QDir cache, QObject *parent=0);
+    void update();
+    enum State { NOT_READY, PARTIAL, COMPLETE };
+
+private:
+    APODFSCrawler cache;
+    QNetworkAccessManager *manager;
+    QHash<QUrl, ImageMetaData> mainIndex;
+    QList<ImageMetaData*> remoteIndex;
+    State state;
+    void mergeIndexes();
+
+public slots:
+    void fsCrawlerFinished();
+    void indexDownloaded(QList<ImageMetaData*>);
+    void imageMetaDataReady(ImageMetaData*);
+    void imageDownloaded(ImageMetaData*, QImage);
+    void imageFound(ImageMetaData);
+};
+#endif
+#include <QtWidgets/QApplication>
+#include <QDir>
+#include <QtDebug>
+#include <QMetaType>
+#include "index.h"
+
+int main(int argc, char *argv[]) {
+    QApplication app(argc, argv);
+    qRegisterMetaType<ImageMetaData>();
+    qDebug() << "ready to go";
+
+    QDir cache("./cache");
+    if(!cache.exists()) {
+        QDir(".").mkdir("cache");
+    }
+    APODIndexer x(cache);
+    x.update();
+    return app.exec();
+}

File metadata.cpp

+#include "metadata.h"
+
+QDataStream& operator<<(QDataStream &stream, const ImageMetaData &image) {
+    stream << image.pageUrl << image.title << image.date << image.imageUrl << image.explanation;
+    return stream;
+}
+
+QDataStream& operator>>(QDataStream &stream, ImageMetaData &image) {
+    stream >> image.pageUrl >> image.title >> image.date >> image.imageUrl >> image.explanation;
+    return stream;
+}
+
+#ifndef METADATA_H
+#define METADATA_H
+
+#include <QObject>
+#include <QDate>
+#include <QUrl>
+#include <QDataStream>
+#include <QMetaType>
+
+struct ImageMetaData {
+    QUrl    pageUrl;
+    QString title;
+    QDate   date;
+    QUrl    imageUrl;
+    QString explanation;
+};
+Q_DECLARE_METATYPE(ImageMetaData);
+
+QDataStream& operator<<(QDataStream &, const ImageMetaData&);
+QDataStream& operator>>(QDataStream &, ImageMetaData&);
+
+#endif

File netclient.cpp

+#include "netclient.h"
+#include <QtConcurrent/QtConcurrent>
+#include <QtDebug>
+#include <QPainter>
+#include <QRegExp>
+#include <htmlcxx/html/ParserDom.h>
+
+APODMainIndexDownloader::APODMainIndexDownloader(QNetworkAccessManager *manager, QObject *parent) : QObject(parent) {
+    qDebug() << "start remote index downloading";
+    QUrl index(QUrl("http://apod.nasa.gov/apod/archivepix.html"));
+    reply = manager->get(QNetworkRequest(index));
+    QObject::connect(
+        reply, SIGNAL(finished()),
+        this, SLOT(requestFinished()));
+}
+
+void APODMainIndexDownloader::requestFinished() {
+    qDebug() << "index downloaded";
+
+    QList<ImageMetaData*> index;
+    QString doc(reply->readAll());
+    QRegExp rx("<a href=\"(ap\\d{6}.html)\">([^<]+)</a>");
+    int pos = 0;
+    while ((pos = rx.indexIn(doc, pos)) != -1) {
+        QStringList groups = rx.capturedTexts();
+        auto metadata = new ImageMetaData();
+        parseIndexRow(groups[1], groups[2], metadata);
+        index.append(metadata);
+        pos += rx.matchedLength();
+    }
+    emit indexDownloaded(index);
+
+    reply->deleteLater();
+    deleteLater();
+}
+
+void APODMainIndexDownloader::parseIndexRow(QString pagename, QString title, ImageMetaData *metadata) {
+    int year = pagename.mid(2, 2).toInt();
+    int month = pagename.mid(4, 2).toInt();
+    int day = pagename.mid(6, 2).toInt();
+    year += year >= 95 ? 1900 : 2000;
+
+    metadata->pageUrl = QUrl("http://apod.nasa.gov/apod/"  + pagename);
+    metadata->title = title;
+    metadata->date = QDate(year, month, day);
+}
+
+APODImageMetaDataDownloader::APODImageMetaDataDownloader(QNetworkAccessManager *manager, ImageMetaData *img) : image(img) {
+    parseWatcher = new QFutureWatcher<QPair<QString, QString>>(this);
+    QObject::connect(
+        parseWatcher, SIGNAL(finished()),
+        this, SLOT(parseFinished()));
+
+    qDebug() << img->pageUrl;
+    reply = manager->get(QNetworkRequest(img->pageUrl));
+    QObject::connect(
+        reply, SIGNAL(finished()),
+        this, SLOT(requestFinished()));
+}
+
+void APODImageMetaDataDownloader::requestFinished() {
+    qDebug() << "page downloaded";
+    auto future = QtConcurrent::run(APODImageMetaDataDownloader::parseHtml, reply->readAll());
+    parseWatcher->setFuture(future);
+    reply->deleteLater();
+}
+
+void APODImageMetaDataDownloader::parseFinished() {
+    qDebug() << "parse finished";
+    auto result = parseWatcher->future().result();
+    image->imageUrl = image->pageUrl.resolved(QUrl(result.first));
+    image->explanation = result.second;
+
+    emit finished(image);
+    deleteLater();
+}
+
+QPair<QString, QString> APODImageMetaDataDownloader::parseHtml(QString doc) {
+    using namespace htmlcxx;
+    HTML::ParserDom parser;
+    tree<HTML::Node> dom = parser.parseTree(doc.toStdString());
+
+    QString imageSrc;
+    QString explanation;
+
+    auto node = dom.begin();
+    auto end = dom.end();
+    for(; node != end; ++node) {
+        auto tag = QString::fromStdString(node->tagName()).toLower();
+        if(tag == "img") {
+            auto anchor = dom.parent(node);
+            anchor->parseAttributes();
+            auto src = anchor->attribute("href");
+            if(src.first) {
+                imageSrc = QString::fromStdString(src.second);
+            }
+        }
+        else if(!node->isTag() && !node->isComment()) {
+            auto text = QString::fromStdString(node->text()).trimmed();
+            if(text.mid(0, 11).toLower() == "explanation") {
+                QStringList expl;
+                auto xit = dom.next_sibling(dom.parent(node));
+                while(dom.is_valid(xit)) {
+                    auto text = QString::fromStdString(xit->text());
+                    if(text.toLower().trimmed().mid(0, 18) == "tomorrow's picture") {
+                        break;
+                    }
+                    expl<< text;
+                    xit = dom.next_sibling(xit);
+                }
+                auto purge = expl.end();
+                auto purge_end = expl.begin();
+                while(purge != purge_end) {
+                    --purge;
+                    if(purge->trimmed().length() == 0 || ((*purge)[0] == '<' && (*purge)[1] != '/')) {
+                        purge = expl.erase(purge);
+                    }
+                    else {
+                        break;
+                    }
+                }
+                explanation = expl.join("");
+            }
+        }
+    }
+
+    return qMakePair(imageSrc, explanation);
+}
+
+QSize APODImageDownloader::thumbSize = QSize(800, 600);
+
+APODImageDownloader::APODImageDownloader(QNetworkAccessManager *manager, ImageMetaData *img) : image(img) {
+    resizeWatcher = new QFutureWatcher<QImage>(this);
+    QObject::connect(
+        resizeWatcher, SIGNAL(finished()),
+        this, SLOT(resizeFinished()));
+
+    reply = manager->get(QNetworkRequest(image->imageUrl));
+    QObject::connect(
+        reply, SIGNAL(finished()),
+        this, SLOT(requestFinished()));
+}
+
+void APODImageDownloader::requestFinished() {
+    auto future = QtConcurrent::run(APODImageDownloader::resizeImage, reply->readAll());
+    resizeWatcher->setFuture(future);
+    reply->deleteLater();
+}
+
+QImage APODImageDownloader::resizeImage(QByteArray buffer) {
+    auto img = QImage::fromData(buffer);
+    if(img.isNull()) {
+        return img;
+    }
+
+    auto size = img.size();
+    if(size.width() > thumbSize.width() || size.height() > thumbSize.height()) {
+        img = img.scaled(thumbSize, Qt::KeepAspectRatio);
+        size = img.size();
+    }
+    if(size != thumbSize) {
+        auto enlarged = QImage(thumbSize, QImage::Format_RGB32);
+        enlarged.fill(Qt::black);
+        QPainter painter(&enlarged);
+        painter.drawImage(
+            (thumbSize.width() - size.width()) / 2,
+            (thumbSize.height() - size.height()) / 2,
+            img);
+        painter.end();
+        return enlarged;
+    }
+    else {
+        return img;
+    }
+}
+
+void APODImageDownloader::resizeFinished() {
+    auto result = resizeWatcher->future().result();
+    emit finished(image, result);
+    deleteLater();
+}
+#include <QObject>
+#include <QtConcurrent/QFutureWatcher>
+#include <QImage>
+#include <QtNetwork>
+#include "metadata.h"
+
+class APODMainIndexDownloader : public QObject {
+    Q_OBJECT
+public:
+    APODMainIndexDownloader(QNetworkAccessManager *manager, QObject *parent=0);
+
+private:
+    QNetworkReply *reply;
+    void parseIndexRow(QString, QString, ImageMetaData*);
+
+signals:
+    void indexDownloaded(QList<ImageMetaData*>);
+
+private slots:
+    void requestFinished();
+};
+
+class APODImageMetaDataDownloader : public QObject {
+    Q_OBJECT
+public:
+    APODImageMetaDataDownloader(QNetworkAccessManager*, ImageMetaData*);
+
+private:
+    ImageMetaData *image;
+    QNetworkReply *reply;
+    QFutureWatcher<QPair<QString, QString>> *parseWatcher;
+    static QPair<QString, QString> parseHtml(QString);
+
+signals:
+    void finished(ImageMetaData*);
+
+private slots:
+    void requestFinished();
+    void parseFinished();
+};
+
+class APODImageDownloader : public QObject {
+    Q_OBJECT
+public:
+    APODImageDownloader(QNetworkAccessManager*, ImageMetaData*);
+    static QSize thumbSize;
+
+private:
+    ImageMetaData *image;
+    QNetworkReply *reply;
+    QFutureWatcher<QImage> *resizeWatcher;
+    static QImage resizeImage(QByteArray);
+
+signals:
+    void finished(ImageMetaData*, QImage);
+
+private slots:
+    void requestFinished();
+    void resizeFinished();
+};
+