downloader.cpp

Go to the documentation of this file.

/*****************************************************************************
 * Copyright 2016-2017 Karolin Stange, Christoph Wick, and Haye Hinrichsen
 *
 * This file is part of JustIntonation.
 *
 * JustIntonation is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * JustIntonation is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along 
 * with JustIntonation. If not, see http://www.gnu.org/licenses/.
 *****************************************************************************/

//=============================================================================
//                               Downloader
//=============================================================================

#include "downloader.h"

#include <QNetworkInterface>
#include <QNetworkProxy>
#include <QFileInfo>
#include <QDir>
#include <QStandardPaths>
#include <QStorageInfo>
#include <QTimer>

#include "config.h"
#include "platformtools/platformtools.h"

//-----------------------------------------------------------------------------
//                         Downloader Constructor
//-----------------------------------------------------------------------------


Downloader::Downloader(QString url, QObject *parent)
    : QObject(parent)
    , mLocalPath(QStandardPaths::writableLocation(QStandardPaths::AppDataLocation))
    , mRemotePath(url)
    , mIndexFileName("index.txt")
    , mNetworkManager()
    , mIndexOfFiles()
    , pNetworkReply(nullptr)
    , mFile()
    , mIsWaitingForConnection(false)
    , mForcedDownload(false)
    , mIsDownloading(false)
{
    setModuleName("Downloader");
}


//-----------------------------------------------------------------------------
//                         Check internet connection
//-----------------------------------------------------------------------------

bool Downloader::isConnectedToInternet()
{
    QList<QNetworkInterface> networkInterfaces = QNetworkInterface::allInterfaces();
    bool connectionFound = false;
    LOGMESSAGE << "Checking network interfaces and internet connections:";
    for (auto &networkInterface : networkInterfaces)
    {
        if ( networkInterface.flags().testFlag(QNetworkInterface::IsUp)
             and not networkInterface.flags().testFlag(QNetworkInterface::IsLoopBack))
        {
           LOGMESSAGE << "Testing" << networkInterface.name()
                    << "MAC:" << networkInterface.hardwareAddress() << ":";
            if (networkInterface.addressEntries().empty())
                { LOGMESSAGE << "This adapter is not connected."; }
            else for (QNetworkAddressEntry &entry : networkInterface.addressEntries())
            {
                // ignore MAC utun-devices
                if (networkInterface.name().startsWith("utun"))
                {
                    LOGMESSAGE << "Ignoring connected MAC device" <<  networkInterface.name();
                }
                else
                {
                    LOGMESSAGE << "This adapter IS CONNECTED.";
                    LOGMESSAGE << "IP:" << entry.ip().toString();
                    LOGMESSAGE << "/ Netmask:" << entry.netmask().toString();

                    connectionFound = true;
                }
            }
        }
    }
    return connectionFound;
}


//-----------------------------------------------------------------------------
//                            Start downloading
//-----------------------------------------------------------------------------

void Downloader::start (bool forced)
{
    LOGMESSAGE << "Start downloading, forced =" << forced;
    if (mIsDownloading)
    {
        if (forced) stop();
        else
        {
            LOGERROR << "Downloader is already downloading. Do not call start() twice.";
            return;
        }
    }

    mForcedDownload = forced;
    if (mIsWaitingForConnection)
    {
        LOGMESSAGE << "Downloader has already been started and is waiting for internet connection. Ignore call to start().";
        return;
    }

    // Check for writeable app data storage location
    QDir directory(mLocalPath);
    if (not directory.exists()) directory.mkpath(".");
    if (not directory.exists())
    {
        LOGERROR << "Cannot create local app data storage in" << mLocalPath;
        return;
    }

    mIsWaitingForConnection = not isConnectedToInternet();
    if (mIsWaitingForConnection)
    {
        LOGWARNING << "Not connected to the internet, retry in 5 secs.";
        if (forced or getPathsOfDownloadedFiles().size()==0)
        {
            LOGMESSAGE << "Emit a warning to Qml that forced download failed because of missing connection";
            emit signalNoInternetConnection(forced);
        }
        QTimer::singleShot(5000,this,&Downloader::downloadIndexFile);
        return;
    }

    // go to the next step
    downloadIndexFile();
}


//-----------------------------------------------------------------------------
//                          Download index file
//-----------------------------------------------------------------------------

void Downloader::downloadIndexFile()
{
    LOGMESSAGE << "entering downloadIndexFile()";

    // Check internet connection
    if (mIsWaitingForConnection)
    {
        mIsWaitingForConnection = not isConnectedToInternet();
        if (mIsWaitingForConnection)
        {
            LOGWARNING << "Still not connected to the internet, retry in 60 secs.";
            QTimer::singleShot(60000,this,&Downloader::downloadIndexFile);
            return;
        }
    }

    // Detect proxy if necessary
#ifndef QT_NO_NETWORKPROXY
    QNetworkProxyQuery query(QUrl(INT_SAMPLES_REPOSITORY));
    QList<QNetworkProxy> listOfProxies = QNetworkProxyFactory::systemProxyForQuery(query);
    LOGMESSAGE << "Number of available proxies" << listOfProxies.size();
    LOGMESSAGE << "Proxy list:" << listOfProxies;
    if (listOfProxies.size())
    {
        LOGMESSAGE << "Set the first proxy in the list";
        QNetworkProxy::setApplicationProxy(listOfProxies[0]);
    }
#endif

    // Start to download index file
    connect(&mNetworkManager, &QNetworkAccessManager::finished,
            this, &Downloader::indexFileDownloadFinished,Qt::QueuedConnection);
    QUrl index = QUrl(fullPath(mRemotePath,mIndexFileName));
    pNetworkReply = mNetworkManager.get(QNetworkRequest(index));
    if (pNetworkReply->error())
    {
        LOGWARNING << "Error requesting file" << index.fileName() << "for download";
        LOGWARNING << "Message:" << pNetworkReply->errorString();
        stop();
    }
    else
    {
        LOGMESSAGE << "Downloading file, waiting for network reply on the URL:";
        LOGMESSAGE << index;
        mIsDownloading = true;
    }
    // Next step: indexFileDownloadFinished
}


//-----------------------------------------------------------------------------
//                            Stop downloading
//-----------------------------------------------------------------------------

void Downloader::stop()
{
    mIsWaitingForConnection = false;
    if (mIsDownloading)
    {
        LOGMESSAGE << "Request to stop downloading";
        if (pNetworkReply) if (pNetworkReply->isOpen()) pNetworkReply->abort();
        if (mFile.isOpen()) mFile.close();
        mIsDownloading = false;
    }
    return;
}


//-----------------------------------------------------------------------------
//                   Private slot: Index file downloaded
//-----------------------------------------------------------------------------

void Downloader::indexFileDownloadFinished(QNetworkReply *pReply)
{
    // Disconnect this slot from the network manager
    disconnect(&mNetworkManager, &QNetworkAccessManager::finished,
               this, &Downloader::indexFileDownloadFinished);

    // Detect possible errors in the reply
    if (pReply->error())
    {
        LOGWARNING << "A network error occured during download of the index file";
        LOGWARNING << pReply->errorString();
        stop();
        LOGWARNING << "After network error: Retry downloading in 60 secs.";
        QTimer::singleShot(60000,this,&Downloader::downloadIndexFile);
        return;
    }

    // Read the index file into the QStringList mIndexOfFiles and delete reply
    QTextStream inputStream(pReply);
    mIndexOfFiles.clear();
    while (not inputStream.atEnd()) mIndexOfFiles << inputStream.readLine();
    pReply->deleteLater();
    if (mIndexOfFiles.isEmpty())
    {
        LOGWARNING << mIndexFileName << "is empty or could not be downloaded.";
        LOGWARNING << mIndexOfFiles;
        mIsDownloading=false;
        return;
    }

    // Write the index file to the local app storage
    QFile indexFile(fullPath(mLocalPath,mIndexFileName));
    if (not indexFile.open(QIODevice::WriteOnly))
    {
        LOGWARNING << "Could not open index file for writing: " << mIndexFileName;
        mIsDownloading=false;
        return;
    }
    QTextStream outputStream(&indexFile);
    for (auto &line : mIndexOfFiles) outputStream << line << "\n";
    if (indexFile.isOpen()) indexFile.close();
    LOGMESSAGE << "Successfully downloaded" << pReply->url().toString();

    // Start downloading the listed files
    initiateDownloadOfNextFile();
}



//-----------------------------------------------------------------------------
//            Get a list of paths of all downloaded files
//-----------------------------------------------------------------------------

QStringList Downloader::getPathsOfDownloadedFiles()
{
    QStringList nameFilter("*.int");
    QDir directory(mLocalPath);
    QStringList list = directory.entryList(nameFilter);
    QFile indexFile(fullPath(mLocalPath,mIndexFileName));
    QStringList filteredList;
    if (not indexFile.open(QIODevice::ReadOnly)) filteredList = list;
    else
    {
        QStringList indexFileContent;
        QTextStream inputStream(&indexFile);
        while (not inputStream.atEnd()) indexFileContent << inputStream.readLine();
        if (indexFile.isOpen()) indexFile.close();
        for (QString &e : indexFileContent) if (list.contains(e))
            filteredList.append(fullPath(directory.path(),e));
    }

    for (QString &e : filteredList) if (mIndexOfFiles.contains(e))
        filteredList.append(fullPath(directory.path(),e));
    return filteredList;
}


//-----------------------------------------------------------------------------
//                 Private slot: Initiate download of next file
//-----------------------------------------------------------------------------

void Downloader::initiateDownloadOfNextFile()
{
    if (mIndexOfFiles.isEmpty()) { mIsDownloading=false; return; }
    QString versiontag = "." + QString::number(INT_FILEFORMAT_ROLLING_VERSION);
    QString remotefile = fullPath(mRemotePath,mIndexOfFiles.first()+versiontag);
    QNetworkRequest request;
    request.setUrl(QUrl(remotefile));
    connect(&mNetworkManager, &QNetworkAccessManager::finished,this,
            &Downloader::checkWhetherFileExistsAndDownload,Qt::QueuedConnection);
    pNetworkReply = mNetworkManager.head(request);
}


//-----------------------------------------------------------------------------
//              Private slot: Start downloading if necessary
//-----------------------------------------------------------------------------

void Downloader::checkWhetherFileExistsAndDownload()
{
    // Disconnect this slot from the network manager
    disconnect(&mNetworkManager, &QNetworkAccessManager::finished,this,
               &Downloader::checkWhetherFileExistsAndDownload);

    // Determine remote file size and last-modified date
    qint64 remotefilesize = pNetworkReply->header(QNetworkRequest::ContentLengthHeader).toUInt();
    QDateTime remotefilelastmodified =
            pNetworkReply->header(QNetworkRequest::LastModifiedHeader).value<QDateTime>();
    pNetworkReply->deleteLater();
    if (mIndexOfFiles.empty()) { mIsDownloading=false; return; }
    QString filename = mIndexOfFiles.first();
    QString versiontag = "." + QString::number(INT_FILEFORMAT_ROLLING_VERSION);
    QString remotefile = fullPath(mRemotePath,filename+versiontag);
    LOGMESSAGE << "Found file " << remotefile << ", size =" << remotefilesize;

    // Check whether there is already a local file of the same name
    QString localfile = fullPath(mLocalPath,filename);
    QFileInfo fileinfo(localfile);
    if (fileinfo.exists() and fileinfo.isFile())
    {
        // If so, compare file size
        qint64 localfilesize = fileinfo.size();
        QDateTime localfilecreated = fileinfo.created();
        LOGSTATUS << "Local file data:" << localfilecreated;
        LOGSTATUS << "Remote file date:" << remotefilelastmodified;
        if (localfilesize != remotefilesize or localfilecreated < remotefilelastmodified)
        {
            // If file are different delete local file
            LOGWARNING << "Length of date is different, hence delete file" << localfile;
            QFile::remove(localfile);
        }
        else if (mForcedDownload)
        {
            LOGMESSAGE << "Forced download: File" << localfile << "will be deleted";
            QFile::remove(localfile);
        }
        else
        {
            // If files are identical
            LOGMESSAGE << "File" << localfile << "already exists, skipping download.";
            emit finalize();
            return;
        }
    }

    // The download is started only if there is at least twice as much disk space
    qint64 diskspace = PlatformTools::getSingleton().getFreeDiskSpace(mLocalPath);
    diskspace = 10000000000;
    LOGSTATUS << "Determining free disk space at location" << mLocalPath;
    LOGSTATUS << "Free disk space =" << diskspace;
    if (2*remotefilesize > diskspace)
    {
        LOGWARNING << "Device does not provide enough disk space for downloading.";
        mIsDownloading=false;
        return;
    }

    // Open temporary local file for writing
    QString tmpfile = localfile + ".part";
    mFile.setFileName(tmpfile);
    if (not mFile.open(QIODevice::WriteOnly))
    {
        LOGWARNING << "Could not open file for writing: " << localfile;
        mIsDownloading=false;
        return;
    }

    // Start downloading
    LOGMESSAGE << "downloading" << remotefile << "to" << localfile;
    connect(&mNetworkManager, &QNetworkAccessManager::finished,
            this, &Downloader::downloadComplete,Qt::QueuedConnection);
    pNetworkReply = mNetworkManager.get(QNetworkRequest(QUrl(remotefile)));
    connect(pNetworkReply,&QNetworkReply::downloadProgress,
            this,&Downloader::downloadProgress,Qt::QueuedConnection);
    emit signalDownloading(true);
}




//-----------------------------------------------------------------------------
//            Private slot: Download a data chunk and report progress
//-----------------------------------------------------------------------------

void Downloader::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
    if (pNetworkReply->isOpen())
    {
        LOGSTATUS << bytesReceived << "of" << bytesTotal << "bytes downloaded.";
        emit signalProgress (mIndexOfFiles.size(),100.0*bytesReceived/bytesTotal);
        QByteArray chunk = pNetworkReply->readAll();
        mFile.write(chunk);
    }
}


//-----------------------------------------------------------------------------
//                    Private slot: Download completed
//-----------------------------------------------------------------------------

void Downloader::downloadComplete(QNetworkReply* pReply)
{
    // Read the last chunk of data and cloe file
    emit signalDownloading(false);
    disconnect(&mNetworkManager, &QNetworkAccessManager::finished,
              this, &Downloader::downloadComplete);
    disconnect(pNetworkReply,&QNetworkReply::downloadProgress,
              this,&Downloader::downloadProgress);
    if (pReply->isOpen())
    {
        QByteArray lastchunk = pReply->readAll();
        mFile.write(lastchunk);
        if (mFile.isOpen()) mFile.close();

        // Rename partial temporary file to the final filename
        if (mIndexOfFiles.isEmpty()) { mIsDownloading=false; return; }
        QString filename = mIndexOfFiles.first();
        QString localfile = fullPath(mLocalPath,filename);
        QString tmpfile = localfile + ".part";
        if (QFile::rename(tmpfile,localfile))
        { LOGMESSAGE << "Completed download of file" << localfile; }
        else LOGWARNING << "Could not rename temporary file" << tmpfile;
        emit finalize();
        emit signalNewFileDownloaded(filename);
    }
    else
    {
        LOGMESSAGE << "Download cancelled. Abort.";
        QString filename = mIndexOfFiles.first();
        QString localfile = fullPath(mLocalPath,filename);
        QString tmpfile = localfile + ".part";
        QFile::remove(tmpfile);
    }
    pReply->deleteLater();
}



//-----------------------------------------------------------------------------
//                    Private slot: Finialize download
//-----------------------------------------------------------------------------

void Downloader::finalize()
{
    // Finalize
    mIndexOfFiles.removeFirst();
    if (not mIndexOfFiles.isEmpty()) emit initiateDownloadOfNextFile();
    else
    {
        LOGMESSAGE << "Downloader shutting down.";
        emit signalAllFilesDownloaded();
        mIsDownloading = false;
    }
}


//-----------------------------------------------------------------------------
//    Helper function: Construct a full path out of a path and a file name
//-----------------------------------------------------------------------------

QString Downloader::fullPath(const QString &path, const QString &file) const
{
    if (not path.endsWith('/')) return path+"/"+file;
    else return path+file;
}