INFO Sourcing add config line: /local/execute/dir_779764/glide_Vq9soE/add_config_line.source INFO Discarding path '/hadoop'. File does not exist INFO Discarding path '/hdfs'. File does not exist INFO Discarding path '/lizard'. File does not exist INFO Discarding path '/mnt/hadoop'. File does not exist INFO Discarding path '/mnt/hdfs'. File does not exist WARNING: Container does not have an exec helper script, calling '/srv/.gwms-user-job-wrapper.sh' directly WARNING glidein_config not definied () in singularity_lib.sh. Some functions like advertise and error_gen will be limited. JANA ERROR>> JANA ERROR>>?JException: code = 0 text = WARNING: unknown bank type (0x4f) JANA ERROR>>ype (0x4f) JANA ERROR>>ries/DAQ/swap_bank.cc, Line: 91 =========================================================== There was a crash. This is the entire stack trace of all threads: =========================================================== Thread 8 (Thread 0x2b7069319700 (LWP 252)): #0 0x00002b7051e2b965 in pthread_cond_wait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010862e0 in jana::JApplication::EventBufferThread (this=this entry=0x7ffff89fa8e0) at src/JANA/JApplication.cc:742 #2 0x00000000010865a9 in LaunchEventBufferThread (arg=0x7ffff89fa8e0) at src/JANA/JApplication.cc:666 #3 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x2b7077471700 (LWP 253)): #0 0x00002b7052b24e2d in nanosleep () from /lib64/libc.so.6 #1 0x00002b7052b24cc4 in sleep () from /lib64/libc.so.6 #2 0x00002b70553bbf8c in GarbageCollectorThread (arg=0x2b706c006730, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientConnMgr.cc:93 #3 0x00002b705561608f in XrdSysThread_Xeq (myargs=0x2b706c0089e0) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #4 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x2b7077a73700 (LWP 254)): #0 0x00002b7052b5320d in poll () from /lib64/libc.so.6 #1 0x00002b70553a5591 in XrdClientSock::RecvRaw (this=0x2b706c10f220, buffer=0x2b70780009c0, length=8, substreamid=, usedsubstreamid=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientSock.cc:156 #2 0x00002b70553c50af in XrdClientPhyConnection::ReadRaw (this=0x2b706c10dd40, buf=0x2b70780009c0, len=8, substreamid=-1, usedsubstreamid=0x2b7077a6c068) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:381 #3 0x00002b70553c7b73 in XrdClientMessage::ReadRaw (this=0x2b7078000980, phy=0x2b706c10dd40) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientMessage.cc:172 #4 0x00002b70553c3a58 in XrdClientPhyConnection::BuildMessage (this=0x2b706c10dd40, IgnoreTimeouts=true, Enqueue=true) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:462 #5 0x00002b70553c679a in SocketReaderThread (arg=0x2b706c10dd40, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:79 #6 0x00002b705561608f in XrdSysThread_Xeq (myargs=0x2b706c10f170) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #7 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x2b7077c74700 (LWP 255)): #0 0x00002b7052b24a89 in waitpid () from /lib64/libc.so.6 #1 0x00002b7052aa2de2 in do_system () from /lib64/libc.so.6 #2 0x00002b7052aa3191 in system () from /lib64/libc.so.6 #3 0x00002b704c3e3439 in TUnixSystem::StackTrace (this=0x264e230) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/root/root-6.08.06/core/unix/src/TUnixSystem.cxx:2405 #4 0x00002b704c3e5d0c in TUnixSystem::DispatchSignals (this=0x264e230, sig=kSigSegmentationViolation) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/root/root-6.08.06/core/unix/src/TUnixSystem.cxx:3625 #5 #6 clear (this=) at /usr/include/c++/4.8.2/bits/stl_vector.h:1126 #7 CopyTo (data=std::vector of length 0, capacity 0, this=0x1a0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JFactory.h:435 #8 DParsedEvent::CopyToFactories (this=this entry=0x2b709c0670d0, loop=loop entry=0x2b707c0008c0) at libraries/DAQ/DParsedEvent.h:260 #9 0x0000000000efeb13 in JEventSource_EVIOpp::GetObjects (this=0x2b706c109fa0, event=..., factory=) at libraries/DAQ/JEventSource_EVIOpp.cc:599 #10 0x0000000000aa57c7 in jana::JEvent::GetObjects (this=this entry=0x2b707c0008c8, t=std::vector of length 0, capacity 0, factory=factory entry=0x2b707c1a7090) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEvent.h:97 #11 0x0000000000aa5ae0 in GetFromSource (factory=0x2b707c1a7090, t=std::vector of length 0, capacity 0, this=0x2b707c0008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:477 #12 jana::JEventLoop::GetFromFactory (this=this entry=0x2b707c0008c0, t=std::vector of length 0, capacity 0, tag=tag entry=0x2b7068f0cd8f "", data_source= 0x2b7077c6ce20: 2770269488, allow_deftag=allow_deftag entry=true) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:427 #13 0x0000000000aa5dc3 in jana::JEventLoop::Get (this=0x2b707c0008c0, t=std::vector of length 0, capacity 0, tag=0x2b7068f0cd8f "", allow_deftag=) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:309 #14 0x00002b7068eec8ac in GetSingle (exception_if_not_one=true, tag=0x2b7068f0cd8f "", t=, this=0x2b707c0008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:239 #15 JEventProcessor_TPOL_tree::evnt (this=0x3b65890, loop=0x2b707c0008c0, eventnumber=160389182) at plugins/Analysis/TPOL_tree/JEventProcessor_TPOL_tree.cc:169 #16 0x00000000010a0b5a in jana::JEventLoop::OneEvent (this=this entry=0x2b707c0008c0) at src/JANA/JEventLoop.cc:693 #17 0x00000000010a1af4 in jana::JEventLoop::Loop (this=this entry=0x2b707c0008c0) at src/JANA/JEventLoop.cc:496 #18 0x0000000001078daa in LaunchThread (arg=0x7ffff89fa8e0) at src/JANA/JApplication.cc:1382 #19 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #20 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x2b7080200700 (LWP 256)): #0 0x00002b7052b5320d in poll () from /lib64/libc.so.6 #1 0x00002b70553a5591 in XrdClientSock::RecvRaw (this=0x2b706c10fcb0, buffer=0x2b7084000950, length=8, substreamid=, usedsubstreamid=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientSock.cc:156 #2 0x00002b70553c50af in XrdClientPhyConnection::ReadRaw (this=0x2b706c10afc0, buf=0x2b7084000950, len=8, substreamid=-1, usedsubstreamid=0x2b70801f9068) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:381 #3 0x00002b70553c7b73 in XrdClientMessage::ReadRaw (this=0x2b7084000910, phy=0x2b706c10afc0) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientMessage.cc:172 #4 0x00002b70553c3a58 in XrdClientPhyConnection::BuildMessage (this=0x2b706c10afc0, IgnoreTimeouts=true, Enqueue=true) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:462 #5 0x00002b70553c679a in SocketReaderThread (arg=0x2b706c10afc0, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:79 #6 0x00002b705561608f in XrdSysThread_Xeq (myargs=0x2b706c10ac90) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #7 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x2b7080401700 (LWP 259)): #0 0x00002b7052b5320d in poll () from /lib64/libc.so.6 #1 0x00002b70553a5591 in XrdClientSock::RecvRaw (this=0x2b706c11d830, buffer=0x2b70880009f0, length=8, substreamid=, usedsubstreamid=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientSock.cc:156 #2 0x00002b70553c50af in XrdClientPhyConnection::ReadRaw (this=0x2b706c11bef0, buf=0x2b70880009f0, len=8, substreamid=-1, usedsubstreamid=0x2b70803fa068) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:381 #3 0x00002b70553c7b73 in XrdClientMessage::ReadRaw (this=0x2b70880009b0, phy=0x2b706c11bef0) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientMessage.cc:172 #4 0x00002b70553c3a58 in XrdClientPhyConnection::BuildMessage (this=0x2b706c11bef0, IgnoreTimeouts=true, Enqueue=true) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:462 #5 0x00002b70553c679a in SocketReaderThread (arg=0x2b706c11bef0, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:79 #6 0x00002b705561608f in XrdSysThread_Xeq (myargs=0x2b706c11d000) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #7 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x2b7080803700 (LWP 271)): #0 0x00002b7051e2b965 in pthread_cond_wait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b70522f182c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x0000000000f092ea in async_filebuf::readloop (this=0x2b706c004e30) at libraries/DAQ/async_filebuf.cc:104 #3 0x00002b70522f5070 in ?? () from /lib64/libstdc++.so.6 #4 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b7052b5dead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x2b70564f0c00 (LWP 243)): #0 0x00002b7051e2ee3d in nanosleep () from /lib64/libpthread.so.0 #1 0x000000000108214d in jana::JApplication::Run (this=this entry=0x7ffff89fa8e0, proc=proc entry=0x0, Nthreads=, Nthreads entry=1) at src/JANA/JApplication.cc:1613 #2 0x000000000064162f in main (narg=17, argv=) at programs/Analysis/hd_ana/hd_ana.cc:24 =========================================================== The lines below might hint at the cause of the crash. You may get help by asking at the ROOT forum http://root.cern.ch/forum. Only if you are really convinced it is a bug in ROOT then please submit a report at http://root.cern.ch/bugs. Please post the ENTIRE stack trace from above as an attachment in addition to anything else that might help us fixing this issue. =========================================================== #6 clear (this=) at /usr/include/c++/4.8.2/bits/stl_vector.h:1126 #7 CopyTo (data=std::vector of length 0, capacity 0, this=0x1a0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JFactory.h:435 #8 DParsedEvent::CopyToFactories (this=this entry=0x2b709c0670d0, loop=loop entry=0x2b707c0008c0) at libraries/DAQ/DParsedEvent.h:260 #9 0x0000000000efeb13 in JEventSource_EVIOpp::GetObjects (this=0x2b706c109fa0, event=..., factory=) at libraries/DAQ/JEventSource_EVIOpp.cc:599 #10 0x0000000000aa57c7 in jana::JEvent::GetObjects (this=this entry=0x2b707c0008c8, t=std::vector of length 0, capacity 0, factory=factory entry=0x2b707c1a7090) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEvent.h:97 #11 0x0000000000aa5ae0 in GetFromSource (factory=0x2b707c1a7090, t=std::vector of length 0, capacity 0, this=0x2b707c0008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:477 #12 jana::JEventLoop::GetFromFactory (this=this entry=0x2b707c0008c0, t=std::vector of length 0, capacity 0, tag=tag entry=0x2b7068f0cd8f "", data_source= 0x2b7077c6ce20: 2770269488, allow_deftag=allow_deftag entry=true) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:427 #13 0x0000000000aa5dc3 in jana::JEventLoop::Get (this=0x2b707c0008c0, t=std::vector of length 0, capacity 0, tag=0x2b7068f0cd8f "", allow_deftag=) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:309 #14 0x00002b7068eec8ac in GetSingle (exception_if_not_one=true, tag=0x2b7068f0cd8f "", t=, this=0x2b707c0008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:239 #15 JEventProcessor_TPOL_tree::evnt (this=0x3b65890, loop=0x2b707c0008c0, eventnumber=160389182) at plugins/Analysis/TPOL_tree/JEventProcessor_TPOL_tree.cc:169 #16 0x00000000010a0b5a in jana::JEventLoop::OneEvent (this=this entry=0x2b707c0008c0) at src/JANA/JEventLoop.cc:693 #17 0x00000000010a1af4 in jana::JEventLoop::Loop (this=this entry=0x2b707c0008c0) at src/JANA/JEventLoop.cc:496 #18 0x0000000001078daa in LaunchThread (arg=0x7ffff89fa8e0) at src/JANA/JApplication.cc:1382 #19 0x00002b7051e27dd5 in start_thread () from /lib64/libpthread.so.0 #20 0x00002b7052b5dead in clone () from /lib64/libc.so.6 ===========================================================