deadlocked backend

[brian@…]

I seem to have a deadlocked backend using master @ 8a5977d. I will attach the full threaded backtrace. I suspect the locked thread is this one:

Thread 1 (Thread 0xb63c4720 (LWP 3074)):
#0  0x0011d416 in __kernel_vsyscall ()
No symbol table info available.
#1  0x088a448c in pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_wait.S:169
No locals.
#2  0x03459daa in ?? () from /usr/lib/libQtCore.so.4
No symbol table info available.
#3  0x034564b2 in QMutex::lock() () from /usr/lib/libQtCore.so.4
No symbol table info available.
#4  0x080f2885 in QMutexLocker (this=0xa30bf18, cardid=11, chanid=1024, startts=..., recstatus=rsRecorded, recendts=...) at /usr/include/qt4/QtCore/qmutex.h:102
No locals.
#5  Scheduler::UpdateRecStatus (this=0xa30bf18, cardid=11, chanid=1024, startts=..., recstatus=rsRecorded, recendts=...) at scheduler.cpp:614
        lockit = {val = 3214543840}
        __FUNCTION__ = "UpdateRecStatus"
#6  0x080bc630 in MainServer::customEvent (this=0xa29a018, e=0xb72a808) at mainserver.cpp:1052
        tokens = {<QList<QString>> = {{p = {static shared_null = {ref = {_q_value = 58927}, alloc = 0, begin = 0, end = 0, sharable = 1, array = {0x0}}, d = 0xa619ce50}, d = 0xa619ce50}}, <No data fields>}
        startts = {d = {d = 0xa61a3c08}}
        recendts = {d = {d = 0xa61920d8}}
        cardid = 11
        chanid = 1024
        recstatus = rsRecorded
        me = 0xb72a808
        message = {static null = {<No data fields>}, static shared_null = {ref = {_q_value = 39469}, alloc = 0, size = 0, data = 0x368b49e, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, static shared_empty = {ref = {_q_value = 36551}, alloc = 0, size = 0, data = 0x368b4b2, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, d = 0xb3deae8, static codecForCStrings = 0x0}
        error = {static null = {<No data fields>}, static shared_null = {ref = {_q_value = 39469}, alloc = 0, size = 0, data = 0x368b49e, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, static shared_empty = {ref = {_q_value = 36551}, alloc = 0, size = 0, data = 0x368b4b2, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, d = 0x368b48c, static codecForCStrings = 0x0}
        mod_me = warning: RTTI symbol not found for class 'QEvent'
{<QEvent> = {<No data fields>}, static MythEventMessage = 65534, static MythUserMessage = 65533, static kUpdateTvProgressEventType = 65532, static kExitToMainMenuEventType = 65531, static kMythPostShowEventType = 65530, static kEnableDrawingEventType = 65529, static kDisableDrawingEventType = 65528, static kPushDisableDrawingEventType = 65527, static kPopDisableDrawingEventType = 65526, static kLockInputDevicesEventType = 65525, static kUnlockInputDevicesEventType = 65524, static kUpdateBrowseInfoEventType = 65523, message = {static null = {<No data fields>}, static shared_null = {ref = {_q_value = 39469}, alloc = 0, size = 0, data = 0x368b49e, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, static shared_empty = {ref = {_q_value = 36551}, alloc = 0, size = 0, data = 0x368b4b2, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, d = 0x368b4a0, static codecForCStrings = 0x0}, extradata = {<QList<QString>> = {{p = {static shared_null = {ref = {_q_value = 58927}, alloc = 0, begin = 0, end = 0, sharable = 1, array = {0x0}}, d = 0xa619d070}, d = 0xa619d070}}, <No data fields>}}
        broadcast = {<QList<QString>> = {{p = {static shared_null = {ref = {_q_value = 58927}, alloc = 0, begin = 0, end = 0, sharable = 1, array = {0x0}}, d = 0x368b414}, d = 0x368b414}}, <No data fields>}
        receivers = {q_hash = {{d = 0x368b3e0, e = 0x368b3e0}}}
        __FUNCTION__ = "customEvent"
#7  0x0356442c in QObject::event(QEvent*) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#8  0x0354f30a in QCoreApplicationPrivate::notify_helper(QObject*, QEvent*) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#9  0x0354f393 in QCoreApplication::notify(QObject*, QEvent*) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#10 0x0354f0bb in QCoreApplication::notifyInternal(QObject*, QEvent*) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#11 0x03552c79 in QCoreApplicationPrivate::sendPostedEvents(QObject*, int, QThreadData*) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#12 0x03552e0d in QCoreApplication::sendPostedEvents(QObject*, int) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#13 0x0357c3c4 in ?? () from /usr/lib/libQtCore.so.4
No symbol table info available.
#14 0x095a0aa8 in g_main_dispatch (context=0x9e46c20) at /build/buildd/glib2.0-2.28.6/./glib/gmain.c:2440
        dispatch = 0x357c390
        was_in_call = 0
        user_data = 0x0
        callback = 0
        cb_funcs = 0x0
        cb_data = 0x0
        current_source_link = {data = 0x9e47698, next = 0x0}
        need_destroy = <value optimized out>
        source = 0x9e47698
        current = 0x9e461c8
        i = <value optimized out>
#15 g_main_context_dispatch (context=0x9e46c20) at /build/buildd/glib2.0-2.28.6/./glib/gmain.c:3013
No locals.
#16 0x095a1270 in g_main_context_iterate (context=0x9e46c20, block=156964896, dispatch=1, self=<value optimized out>) at /build/buildd/glib2.0-2.28.6/./glib/gmain.c:3091
        max_priority = 2147483647
        timeout = 13327
        some_ready = 1
        nfds = <value optimized out>
        allocated_nfds = <value optimized out>
        fds = 0xa487d90
#17 0x095a1524 in g_main_context_iteration (context=0x9e46c20, may_block=1) at /build/buildd/glib2.0-2.28.6/./glib/gmain.c:3154
        retval = <value optimized out>
#18 0x0357c53c in QEventDispatcherGlib::processEvents(QFlags<QEventLoop::ProcessEventsFlag>) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#19 0x0354e289 in QEventLoop::processEvents(QFlags<QEventLoop::ProcessEventsFlag>) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#20 0x0354e522 in QEventLoop::exec(QFlags<QEventLoop::ProcessEventsFlag>) () from /usr/lib/libQtCore.so.4
No symbol table info available.
#21 0x03552ecc in QCoreApplication::exec() () from /usr/lib/libQtCore.so.4
No symbol table info available.
#22 0x0815ebd0 in run_backend (cmdline=...) at main_helpers.cpp:654
        ismaster = 24
        __FUNCTION__ = "run_backend"
        fatal_error = false
        runsched = false
        httpStatus = 0x9ffc940
        pHS = 0xa29a018
        port = 6543
        sched = 0xa30bf18
#23 0x08091a00 in main (argc=-670438560, argv=0x9009e777) at main.cpp:127
        a = <incomplete type>
        retval = <value optimized out>
        mask = {static null = {<No data fields>}, static shared_null = {ref = {_q_value = 39469}, alloc = 0, size = 0, data = 0x368b49e, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, static shared_empty = {ref = {_q_value = 36551}, alloc = 0, size = 0, data = 0x368b4b2, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, d = 0x9e6ef88, static codecForCStrings = 0x0}
        __FUNCTION__ = "main"
        cmdline = {<MythCommandLineParser> = {_vptr.MythCommandLineParser = 0x8205da0, m_appname = {static null = {<No data fields>}, static shared_null = {ref = {_q_value = 39469}, alloc = 0, size = 0, data = 0x368b49e, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, static shared_empty = {ref = {_q_value = 36551}, alloc = 0, size = 0, data = 0x368b4b2, clean = 0, simpletext = 0, righttoleft = 0, asciiCache = 0, capacity = 0, reserved = 0, array = {0}}, d = 0x9e3aba0, static codecForCStrings = 0x0}, m_optionedArgs = {{d = 0x9e3b300, e = 0x9e3b300}}, m_namedArgs = {{d = 0x9e3b248, e = 0x9e3b248}}, m_passthroughActive = false, m_overridesImported = true, m_verbose = false}, <No data fields>}
        daemonize = true

[brian@…]

threaded stack trace

[brian@…]

Damn. It was so hard a hang/deadlock (harder than any I've ever seen before) that it failed to record anything for the last 4 days while I wasn't paying attention (long weekend, etc.).

WAF is going to hit rock bottom now I think. :-(

[Raymond Wagner]

This sounds like #10302 or #10188, but with some variation in the recorder code that gets it caught up in the same deadlocked mutex. Were your frontends complaining of readStringList timeouts?

[danielk]

Duplicate of #10453. This was fixed in [79d7af902c7].