I'm currently working on a small crawler app for a hobby project. I'm using Qt 5.10 here. This is the relevant portion of code:
std::string result = "";
url = QUrl(inURL.c_str());
jar->setCookiesFromUrl(list,url);
while(true){
QNetworkRequest request = QNetworkRequest(url);
QEventLoop loop;
connect(manager, SIGNAL(finished(QNetworkReply*)),&loop, SLOT(quit()));
netReply = manager->get(request);
loop.exec();
if(netReply->error() == QNetworkReply::NoError){
result = QByteArray(netReply->readAll()).constData();
break;
}
else{
DebugMsg << "Error loading: " << inURL << ". Retrying";
}
}
_usedWProxy++;
if(checkForIdentity() && torified)renewIdentity();
return result;
Here jar, url, manager and netReply are member of my object. This code works in most of cases, but eventually crashes with the "gdb where" output below. It does NOT crash on a certain URL or server, so it is not really reproducible for me at this moment. I'm iterating over a lot of URLs and just crashes from time to time. The last time it crashed at this website:
https://sofifa.com/players?v=18&e=159061&set=true&offset=9120
GDB out:
Thread 10 "QNetworkAccessM" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffe2492700 (LWP 22676)]
0x00007fffee007540 in lh_insert () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
(gdb) where
0 0x00007fffee007540 in lh_insert () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
1 0x00007fffedf51314 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
2 0x00007fffedf51917 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
3 0x00007fffedffd962 in BIO_free () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
4 0x00007fffedffe2e4 in BIO_free_all () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
5 0x00007fffee370f3f in SSL_free () from /lib/x86_64-linux-gnu/libssl.so.1.0.0
6 0x00007ffff5cda6ea in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
7 0x00007ffff5cbc578 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
8 0x00007ffff5cc2451 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
9 0x00007ffff72a3bc9 in QMetaObject::activate(QObject*, int, int, void**) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
10 0x00007ffff5c92078 in QAbstractSocket::disconnectFromHost() () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
11 0x00007ffff5cc27e1 in QSslSocket::close() () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
12 0x00007ffff5bf69e9 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
13 0x00007ffff5bf96c5 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
14 0x00007ffff5bf9525 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
15 0x00007ffff5bf793a in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
16 0x00007ffff72a3bc9 in QMetaObject::activate(QObject*, int, int, void**) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
17 0x00007ffff5cddb48 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
18 0x00007ffff5cc24b1 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
19 0x00007ffff72a3bc9 in QMetaObject::activate(QObject*, int, int, void**) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
20 0x00007ffff5c926f3 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
21 0x00007ffff5c927ac in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
22 0x00007ffff5c9b123 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
23 0x00007ffff5ce9209 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Network.so.5
24 0x00007ffff72a4b21 in QObject::event(QEvent*) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
25 0x00007ffff72789e3 in QCoreApplication::notifyInternal2(QObject*, QEvent*) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
26 0x00007ffff727b48b in QCoreApplicationPrivate::sendPostedEvents(QObject*, int, QThreadData*) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
27 0x00007ffff72cd563 in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
28 0x00007ffff2766197 in g_main_context_dispatch () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
29 0x00007ffff27663f0 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
30 0x00007ffff276649c in g_main_context_iteration () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
31 0x00007ffff72ccbaf in QEventDispatcherGlib::processEvents(QFlags) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
32 0x00007ffff727726a in QEventLoop::exec(QFlags) () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
33 0x00007ffff70a16dc in QThread::exec() () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
34 0x00007ffff70a660c in ?? () from /data/local/Qt/5.10.0/gcc_64/lib/libQt5Core.so.5
35 0x00007ffff59946ba in start_thread (arg=0x7fffe2492700) at pthread_create.c:333
36 0x00007ffff627541d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
Since there is not a single line of my own code mentioned in the GDB output, i suspect, that there is a problem with OpenSSL and/or Qt here. This is my OpenSSL version:
user@machine:~$ openssl version
OpenSSL 1.0.2g 1 Mar 2016
I am running Ubuntu 16.04.03 by the way.
As far as I understand, it crashes while disconnecting from the socket.
I have absolutely no idea, what steps to take, to approach this problem. I will be thankful for all your suggestions. Thanks in advance Markus
Minimal example to compile with BJAM:
main.cpp:
#include <iostream>
#include <QtCore/QCoreApplication>
#include "loader.hpp"
int main(int argc, char *argv[])
{
QCoreApplication a(argc, argv);
TestLoader *load = new TestLoader();
for(unsigned int i = 0; i < 5000; i++){
std::cout << load->getContent("https://sofifa.com/players?v=18&e=159061&set=true&offset=9120").length() << std::endl;
}
load->deleteLater();
a.exec();
}
loader.hpp
#define TESTLOADER_H
#include <QObject>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QUrl>
#include <QEventLoop>
#include <iostream>
#include <unistd.h>
#include <stdio.h> //printf
#include <string.h> //strlen
#include <string> //string
#include <sys/socket.h> //socket
#include <arpa/inet.h> //inet_addr
#include <netdb.h> //hostent
class TestLoader : public QObject
{
Q_OBJECT
private:
QNetworkAccessManager *manager;
QNetworkReply *netReply;
QUrl url;
public:
explicit TestLoader(QObject *parent = 0){
manager = new QNetworkAccessManager(this);
};
std::string getContent(std::string inURL){
std::string result = "";
url = QUrl(inURL.c_str());
while(true){
QNetworkRequest request = QNetworkRequest(url);
QEventLoop loop;
connect(manager, SIGNAL(finished(QNetworkReply*)),&loop, SLOT(quit()));
netReply = manager->get(request);
loop.exec();
if(netReply->error() == QNetworkReply::NoError){
result = QByteArray(netReply->readAll()).constData();
break;
}
else{
std::cout << "Error loading: " << inURL << ". Retrying";
}
}
return result;
};
signals:
public slots:
private slots:
};
#endif // TESTLOADER_H
Jamfile:
alias install : install-bin ;
explicit install ;
install install-bin : LoaderTestApp
: <location>bin
<install-dependencies>on
<install-type>EXE
;
project :
requirements
<include>./
<cxxflags>-std=c++11
<cxxflags>-I/usr/include
<cxxflags>-I/usr/local/include
<cxxflags>-Wall
<cxxflags>-Wno-deprecated
<cxxflags>-g
<linkflags>-L/usr/lib
<linkflags>-L/usr/local/lib
<linkflags>-L/usr/lib/x86_64-linux-gnu
;
exe LoaderTestApp : loadertestapp.cpp
/qt5//QtCore
/qt5//QtNetwork
loader.hpp
:
;
These 3 files should be placed in a directory called utils (this is from my structure) Then you need the Jamroot: (placed in top directory, please change path to Qt)
using gcc ;
using qt5 : /data/local/Qt/5.10.0/gcc_64/ ;
project :
requirements
;
build-project utils
I have figured out, what the problem with my project was. I was using the QNAM for crawling websites and on another thread I was using CURL to put the crawling results to a couchDB. In OpenSSL 1.1.0, the multi-threading capabilities were improved (see here), so updating OpenSSL did the trick for me. Sorry for the question, that didnt even contain the problem and thank you to @eyllanesc, who tested my code on his machine.