Reweighting crashes in multicore mode

Hi,

when running the reweighting in multicore mode some of the reweighting jobs crash for reasons I do not understand.
This is the case in v2.5.5 as well as v2.6.0.
Is this a known problem and is there a solution for this problem?

Below the contents of two debug.log files.

Cheers,
Hannes

===run_02_tag_1_debug.log ======================================
#************************************************************
#* MadGraph5_aMC@NLO/MadEvent *
#* *
#* * * *
#* * * * * *
#* * * * * 5 * * * * *
#* * * * * *
#* * * *
#* *
#* *
#* VERSION 2.5.5 2017-05-26 *
#* *
#* The MadGraph5_aMC@NLO Development Team - Find us at *
#* https://server06.fynu.ucl.ac.be/projects/madgraph *
#* *
#************************************************************
#* *
#* Command File for MadEvent *
#* *
#* run as ./bin/madevent.py filename *
#* *
#************************************************************
generate_events run_02
Traceback (most recent call last):
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1430, in onecmd
    return self.onecmd_orig(line, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1384, in onecmd_orig
    return func(arg, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/madevent_interface.py", line 2118, in do_generate_events
    self.exec_cmd('reweight -from_cards', postcmd=False)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1457, in exec_cmd
    stop = Cmd.onecmd_orig(current_interface, line, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1384, in onecmd_orig
    return func(arg, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/common_run_interface.py", line 1927, in do_reweight
    raise Exception, "Some of the run failed: Please read %s_%s_debug.log" % (f, self.run_tag)
Exception: Some of the run failed: Please read /scratch/mildner/standalone/testsl/Events/run_02/unweighted_events.lhe.gz_2.lhe_tag_1_debug.log
...
==========================================================

===/scratch/mildner/standalone/testsl/Events/run_02/unweighted_events.lhe.gz_2.lhe_tag_1_debug.log ======================

#************************************************************
#* MadGraph5_aMC@NLO/MadEvent *
#* *
#* * * *
#* * * * * *
#* * * * * 5 * * * * *
#* * * * * *
#* * * *
#* *
#* *
#* VERSION 2.5.5 20xx-xx-xx *
#* *
#* The MadGraph5_aMC@NLO Development Team - Find us at *
#* https://server06.fynu.ucl.ac.be/projects/madgraph *
#* *
#************************************************************
#* *
#* Command File for MadEvent *
#* *
#* run as ./bin/madevent.py filename *
#* *
#************************************************************
reweight /scratch/mildner/standalone/testsl/Events/run_02/unweighted_events.lhe.gz_2.lhe -from_cards --multicore=wait
Traceback (most recent call last):
  File "/scratch/mildner/standalone/testsl/bin/internal/extended_cmd.py", line 1430, in onecmd
    return self.onecmd_orig(line, **opt)
  File "/scratch/mildner/standalone/testsl/bin/internal/extended_cmd.py", line 1384, in onecmd_orig
    return func(arg, **opt)
  File "/scratch/mildner/standalone/testsl/bin/internal/common_run_interface.py", line 1963, in do_reweight
    reweight_cmd.import_command_file(path)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1572, in import_command_file
    self.exec_cmd(line, precmd=True)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1457, in exec_cmd
    stop = Cmd.onecmd_orig(current_interface, line, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/extended_cmd.py", line 1384, in onecmd_orig
    return func(arg, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/various/misc.py", line 99, in f_with_no_logger
    out = f(self, *args, **opt)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/madgraph/interface/reweight_interface.py", line 562, in do_launch
    card_diff = old_param.create_diff(new_param)
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/models/check_param_card.py", line 527, in create_diff
    new_value = new_card[blockname].get(lhacode).value
  File "/scratch/mildner/standalone/MG5_aMC_v2_5_5/models/check_param_card.py", line 373, in __getitem__
    return dict.__getitem__(self,name.lower())
KeyError: 'yukawa'
                              Run Options
                              -----------
               stdout_level : None

                         MadEvent Options
                         ----------------
     automatic_html_opening : True
        notification_center : True
          cluster_temp_path : None
             cluster_memory : None
               cluster_size : 100
              cluster_queue : None
                    nb_core : 16 (user set)
               cluster_time : None
                   run_mode : 2

                      Configuration Options
                      ---------------------
                text_editor : None
         cluster_local_path : None
      cluster_status_update : (600, 30)
               pythia8_path : None (user set)
                  hwpp_path : None (user set)
            pythia-pgs_path : None (user set)
                    td_path : None (user set)
               delphes_path : None (user set)
                thepeg_path : None (user set)
               cluster_type : condor
          madanalysis5_path : None (user set)
           cluster_nb_retry : 1
                 eps_viewer : None
                web_browser : None
               syscalc_path : None (user set)
           madanalysis_path : None (user set)
                     lhapdf : lhapdf-config
              f2py_compiler : None
                 hepmc_path : None (user set)
         cluster_retry_wait : 300
           fortran_compiler : None
                auto_update : 7 (user set)
        exrootanalysis_path : None (user set)
                    timeout : 60
               cpp_compiler : None
#************************************************************
#* MadGraph5_aMC@NLO *
#* *
#* * * *
#* * * * * *
#* * * * * 5 * * * * *
#* * * * * *
#* * * *
#* *
#* *
#* VERSION 2.5.5 2017-05-26 *
#* *
#* The MadGraph5_aMC@NLO Development Team - Find us at *
#* https://server06.fynu.ucl.ac.be/projects/madgraph *
#* *
#************************************************************
#* *
#* Command File for MadGraph5_aMC@NLO *
#* *
#* run as ./bin/mg5_aMC filename *
#* *
#************************************************************
set group_subprocesses Auto
set ignore_six_quark_processes False
set loop_optimized_output True
set loop_color_flows False
set gauge unitary
set complex_mass_scheme False
set max_npoint_for_channel 0
import model sm
define p = g u c d s u~ c~ d~ s~
define j = g u c d s u~ c~ d~ s~
define l+ = e+ mu+
define l- = e- mu-
define vl = ve vm vt
define vl~ = ve~ vm~ vt~
import model EWdim6
generate p p > w+ w- NP=2 HIG=0
output testsl
######################################################################
## PARAM_CARD AUTOMATICALY GENERATED BY MG5 FOLLOWING UFO MODEL ####
######################################################################
## ##
## Width set on Auto will be computed following the information ##
## present in the decay.py files of the model. ##
## See arXiv:1402.1178 for more details. ##
## ##
######################################################################

###################################
## INFORMATION FOR DIM6
###################################
Block dim6
    1 1.000000e+00 # CWWWL2
    2 1.000000e+00 # CWL2
    3 1.000000e+01 # CBL2
    4 1.000000e+02 # CPWWWL2
    5 1.000000e+03 # CPWL2

###################################
## INFORMATION FOR MASS
###################################
Block mass
    5 4.700000e+00 # MB
    6 1.720000e+02 # MT
   15 1.777000e+00 # MTA
   23 9.118760e+01 # MZ
   25 1.250000e+02 # MH
  9000006 1.250120e+02 # MP
## Dependent parameters, given by model restrictions.
## Those values should be edited following the
## analytical expression. MG5 ignores those values
## but they are important for interfacing the output of MG5
## to external program such as Pythia.
  1 0.000000 # d : 0.0
  2 0.000000 # u : 0.0
  3 0.000000 # s : 0.0
  4 0.000000 # c : 0.0
  11 0.000000 # e- : 0.0
  12 0.000000 # ve : 0.0
  13 0.000000 # mu- : 0.0
  14 0.000000 # vm : 0.0
  16 0.000000 # vt : 0.0
  21 0.000000 # g : 0.0
  22 0.000000 # a : 0.0
  24 79.824360 # w+ : cmath.sqrt(MZ__exp__2/2. + cmath.sqrt(MZ__exp__4/4. - (aEW*cmath.pi*MZ__exp__2)/(Gf*sqrt__2)))

###################################
## INFORMATION FOR SMINPUTS
###################################
Block sminputs
    1 1.279000e+02 # aEWM1
    2 1.166370e-05 # Gf
    3 1.184000e-01 # aS

###################################
## INFORMATION FOR YUKAWA
###################################
Block yukawa
    5 4.700000e+00 # ymb
    6 1.720000e+02 # ymt
   15 1.777000e+00 # ymtau

###################################
## INFORMATION FOR DECAY
###################################
DECAY 6 1.508336e+00 # WT
DECAY 15 2.270000e-12 # WTau
DECAY 23 2.495200e+00 # WZ
DECAY 24 2.085000e+00 # WW
DECAY 25 6.382339e-03 # WH
DECAY 9000006 6.382339e-03 # WH1
## Dependent parameters, given by model restrictions.
## Those values should be edited following the
## analytical expression. MG5 ignores those values
## but they are important for interfacing the output of MG5
## to external program such as Pythia.
DECAY 1 0.000000 # d : 0.0
DECAY 2 0.000000 # u : 0.0
DECAY 3 0.000000 # s : 0.0
DECAY 4 0.000000 # c : 0.0
DECAY 5 0.000000 # b : 0.0
DECAY 11 0.000000 # e- : 0.0
DECAY 12 0.000000 # ve : 0.0
DECAY 13 0.000000 # mu- : 0.0
DECAY 14 0.000000 # vm : 0.0
DECAY 16 0.000000 # vt : 0.0
DECAY 21 0.000000 # g : 0.0
DECAY 22 0.000000 # a : 0.0
#===========================================================
# QUANTUM NUMBERS OF NEW STATE(S) (NON SM PDG CODE)
#===========================================================

Block QNUMBERS 9000006 # h1
        1 0 # 3 times electric charge
        2 1 # number of spin states (2S+1)
        3 1 # colour rep (1: singlet, 3: triplet, 8: octet)
        4 0 # Particle/Antiparticle distinction (0=own anti)
#*********************************************************************
# MadGraph5_aMC@NLO *
# *
# run_card.dat MadEvent *
# *
# This file is used to set the parameters of the run. *
# *
# Some notation/conventions: *
# *
# Lines starting with a '# ' are info or comments *
# *
# mind the format: value = variable ! comment *
#*********************************************************************
#
#*******************
# Running parameters
#*******************
#
#*********************************************************************
# Tag name for the run (one word) *
#*********************************************************************
  tag_1 = run_tag ! name of the run
#*********************************************************************
# Number of events and rnd seed *
# Warning: Do not generate more than 1M events in a single run *
# If you want to run Pythia, avoid more than 50k events in a run. *
#*********************************************************************
  500000 = nevents ! Number of unweighted events requested
  0 = iseed ! rnd seed (0=assigned automatically=default))
#*********************************************************************
# Collider type and energy *
# lpp: 0=No PDF, 1=proton, -1=antiproton, 2=photon from proton, *
# 3=photon from electron *
#*********************************************************************
     1 = lpp1 ! beam 1 type
     1 = lpp2 ! beam 2 type
     6500.0 = ebeam1 ! beam 1 total energy in GeV
     6500.0 = ebeam2 ! beam 2 total energy in GeV
#*********************************************************************
# Beam polarization from -100 (left-handed) to 100 (right-handed) *
#*********************************************************************
     0.0 = polbeam1 ! beam polarization for beam 1
     0.0 = polbeam2 ! beam polarization for beam 2
#*********************************************************************
# PDF CHOICE: this automatically fixes also alpha_s and its evol. *
#*********************************************************************
     nn23lo1 = pdlabel ! PDF set
     230000 = lhaid ! if pdlabel=lhapdf, this is the lhapdf number
#*********************************************************************
# Renormalization and factorization scales *
#*********************************************************************
 False = fixed_ren_scale ! if .true. use fixed ren scale
 False = fixed_fac_scale ! if .true. use fixed fac scale
 91.188 = scale ! fixed ren scale
 91.188 = dsqrt_q2fact1 ! fixed fact scale for pdf1
 91.188 = dsqrt_q2fact2 ! fixed fact scale for pdf2
 -1 = dynamical_scale_choice ! Choose one of the preselected dynamical choices
 1.0 = scalefact ! scale factor for event-by-event scales
#*********************************************************************
# Type and output format
#*********************************************************************
  False = gridpack !True = setting up the grid pack
  -1.0 = time_of_flight ! threshold (in mm) below which the invariant livetime is not written (-1 means not written)
  3.0 = lhe_version ! Change the way clustering information pass to shower.
  True = clusinfo ! include clustering tag in output
  average = event_norm ! average/sum. Normalization of the weight in the LHEF

#*********************************************************************
# Matching parameter (MLM only)
#*********************************************************************
 0 = ickkw ! 0 no matching, 1 MLM
 1.0 = alpsfact ! scale factor for QCD emission vx
 False = chcluster ! cluster only according to channel diag
 4 = asrwgtflavor ! highest quark flavor for a_s reweight
 False = auto_ptj_mjj ! Automatic setting of ptj and mjj if xqcut >0
                                   ! (turn off for VBF and single top processes)
 0.0 = xqcut ! minimum kt jet measure between partons
#*********************************************************************
#
#*********************************************************************
# handling of the helicities:
# 0: sum over all helicities
# 1: importance sampling over helicities
#*********************************************************************
   0 = nhel ! using helicities importance sampling or not.
#*********************************************************************
# Generation bias, check the wiki page below for more information: *
# 'cp3.irmp.ucl.ac.be/projects/madgraph/wiki/LOEventGenerationBias' *
#*********************************************************************
 None = bias_module ! Bias type of bias, [None, ptj_bias, -custom_folder-]
 {} = bias_parameters ! Specifies the parameters of the module.
#
#*******************************
# Parton level cuts definition *
#*******************************
#
#
#*********************************************************************
# BW cutoff (M+/-bwcutoff*Gamma) ! Define on/off-shell for "$" and decay
#*********************************************************************
  15.0 = bwcutoff ! (M+/-bwcutoff*Gamma)
#*********************************************************************
# Apply pt/E/eta/dr/mij/kt_durham cuts on decay products or not
# (note that etmiss/ptll/ptheavy/ht/sorted cuts always apply)
#*********************************************************************
   False = cut_decays ! Cut decay products
#*********************************************************************
# Standard Cuts *
#*********************************************************************
# Minimum and maximum pt's (for max, -1 means no cut) *
#*********************************************************************
 20.0 = ptj ! minimum pt for the jets
 0.0 = ptb ! minimum pt for the b
 10.0 = pta ! minimum pt for the photons
 10.0 = ptl ! minimum pt for the charged leptons
 0.0 = misset ! minimum missing Et (sum of neutrino's momenta)
 0.0 = ptheavy ! minimum pt for one heavy final state
 -1.0 = ptjmax ! maximum pt for the jets
 -1.0 = ptbmax ! maximum pt for the b
 -1.0 = ptamax ! maximum pt for the photons
 -1.0 = ptlmax ! maximum pt for the charged leptons
 -1.0 = missetmax ! maximum missing Et (sum of neutrino's momenta)
#*********************************************************************
# Minimum and maximum E's (in the center of mass frame) *
#*********************************************************************
  0.0 = ej ! minimum E for the jets
  0.0 = eb ! minimum E for the b
  0.0 = ea ! minimum E for the photons
  0.0 = el ! minimum E for the charged leptons
  -1.0 = ejmax ! maximum E for the jets
 -1.0 = ebmax ! maximum E for the b
 -1.0 = eamax ! maximum E for the photons
 -1.0 = elmax ! maximum E for the charged leptons
#*********************************************************************
# Maximum and minimum absolute rapidity (for max, -1 means no cut) *
#*********************************************************************
  5.0 = etaj ! max rap for the jets
  -1.0 = etab ! max rap for the b
 2.5 = etaa ! max rap for the photons
 2.5 = etal ! max rap for the charged leptons
 0.0 = etajmin ! min rap for the jets
 0.0 = etabmin ! min rap for the b
 0.0 = etaamin ! min rap for the photons
 0.0 = etalmin ! main rap for the charged leptons
#*********************************************************************
# Minimum and maximum DeltaR distance *
#*********************************************************************
 0.4 = drjj ! min distance between jets
 0.0 = drbb ! min distance between b's
 0.4 = drll ! min distance between leptons
 0.4 = draa ! min distance between gammas
 0.0 = drbj ! min distance between b and jet
 0.4 = draj ! min distance between gamma and jet
 0.4 = drjl ! min distance between jet and lepton
 0.0 = drab ! min distance between gamma and b
 0.0 = drbl ! min distance between b and lepton
 0.4 = dral ! min distance between gamma and lepton
 -1.0 = drjjmax ! max distance between jets
 -1.0 = drbbmax ! max distance between b's
 -1.0 = drllmax ! max distance between leptons
 -1.0 = draamax ! max distance between gammas
 -1.0 = drbjmax ! max distance between b and jet
 -1.0 = drajmax ! max distance between gamma and jet
 -1.0 = drjlmax ! max distance between jet and lepton
 -1.0 = drabmax ! max distance between gamma and b
 -1.0 = drblmax ! max distance between b and lepton
 -1.0 = dralmax ! maxdistance between gamma and lepton
#*********************************************************************
# Minimum and maximum invariant mass for pairs *
# WARNING: for four lepton final state mmll cut require to have *
# different lepton masses for each flavor! *
#*********************************************************************
 0.0 = mmjj ! min invariant mass of a jet pair
 0.0 = mmbb ! min invariant mass of a b pair
 0.0 = mmaa ! min invariant mass of gamma gamma pair
 0.0 = mmll ! min invariant mass of l+l- (same flavour) lepton pair
 -1.0 = mmjjmax ! max invariant mass of a jet pair
 -1.0 = mmbbmax ! max invariant mass of a b pair
 -1.0 = mmaamax ! max invariant mass of gamma gamma pair
 -1.0 = mmllmax ! max invariant mass of l+l- (same flavour) lepton pair
#*********************************************************************
# Minimum and maximum invariant mass for all letpons *
#*********************************************************************
 0.0 = mmnl ! min invariant mass for all letpons (l+- and vl)
 -1.0 = mmnlmax ! max invariant mass for all letpons (l+- and vl)
#*********************************************************************
# Minimum and maximum pt for 4-momenta sum of leptons *
#*********************************************************************
 0.0 = ptllmin ! Minimum pt for 4-momenta sum of leptons(l and vl)
 -1.0 = ptllmax ! Maximum pt for 4-momenta sum of leptons(l and vl)
#*********************************************************************
# Inclusive cuts *
#*********************************************************************
 0.0 = xptj ! minimum pt for at least one jet
 0.0 = xptb ! minimum pt for at least one b
 0.0 = xpta ! minimum pt for at least one photon
 0.0 = xptl ! minimum pt for at least one charged lepton
#*********************************************************************
# Control the pt's of the jets sorted by pt *
#*********************************************************************
 0.0 = ptj1min ! minimum pt for the leading jet in pt
 0.0 = ptj2min ! minimum pt for the second jet in pt
 0.0 = ptj3min ! minimum pt for the third jet in pt
 0.0 = ptj4min ! minimum pt for the fourth jet in pt
 -1.0 = ptj1max ! maximum pt for the leading jet in pt
 -1.0 = ptj2max ! maximum pt for the second jet in pt
 -1.0 = ptj3max ! maximum pt for the third jet in pt
 -1.0 = ptj4max ! maximum pt for the fourth jet in pt
 0 = cutuse ! reject event if fails any (0) / all (1) jet pt cuts
#*********************************************************************
# Control the pt's of leptons sorted by pt *
#*********************************************************************
 0.0 = ptl1min ! minimum pt for the leading lepton in pt
 0.0 = ptl2min ! minimum pt for the second lepton in pt
 0.0 = ptl3min ! minimum pt for the third lepton in pt
 0.0 = ptl4min ! minimum pt for the fourth lepton in pt
 -1.0 = ptl1max ! maximum pt for the leading lepton in pt
 -1.0 = ptl2max ! maximum pt for the second lepton in pt
 -1.0 = ptl3max ! maximum pt for the third lepton in pt
 -1.0 = ptl4max ! maximum pt for the fourth lepton in pt
#*********************************************************************
# Control the Ht(k)=Sum of k leading jets *
#*********************************************************************
 0.0 = htjmin ! minimum jet HT=Sum(jet pt)
 -1.0 = htjmax ! maximum jet HT=Sum(jet pt)
 0.0 = ihtmin !inclusive Ht for all partons (including b)
 -1.0 = ihtmax !inclusive Ht for all partons (including b)
 0.0 = ht2min ! minimum Ht for the two leading jets
 0.0 = ht3min ! minimum Ht for the three leading jets
 0.0 = ht4min ! minimum Ht for the four leading jets
 -1.0 = ht2max ! maximum Ht for the two leading jets
 -1.0 = ht3max ! maximum Ht for the three leading jets
 -1.0 = ht4max ! maximum Ht for the four leading jets
#***********************************************************************
# Photon-isolation cuts, according to hep-ph/9801442 *
# When ptgmin=0, all the other parameters are ignored *
# When ptgmin>0, pta and draj are not going to be used *
#***********************************************************************
 0.0 = ptgmin ! Min photon transverse momentum
 0.4 = R0gamma ! Radius of isolation code
 1.0 = xn ! n parameter of eq.(3.4) in hep-ph/9801442
 1.0 = epsgamma ! epsilon_gamma parameter of eq.(3.4) in hep-ph/9801442
 True = isoEM ! isolate photons from EM energy (photons and leptons)
#*********************************************************************
# WBF cuts *
#*********************************************************************
 0.0 = xetamin ! minimum rapidity for two jets in the WBF case
 0.0 = deltaeta ! minimum rapidity for two jets in the WBF case
#***********************************************************************
# Turn on either the ktdurham or ptlund cut to activate *
# CKKW(L) merging with Pythia8 [arXiv:1410.3012, arXiv:1109.4829] *
#***********************************************************************
 -1.0 = ktdurham
 0.4 = dparameter
 -1.0 = ptlund
 1, 2, 3, 4, 5, 6, 21 = pdgs_for_merging_cut ! PDGs for two cuts above
#*********************************************************************
# maximal pdg code for quark to be considered as a light jet *
# (otherwise b cuts are applied) *
#*********************************************************************
 4 = maxjetflavor ! Maximum jet pdg code
#*********************************************************************
#
#*********************************************************************
# Store info for systematics studies *
# WARNING: Do not use for interference type of computation *
#*********************************************************************
   True = use_syst ! Enable systematics studies
#
#**************************************
# Parameter of the systematics study
# will be used by SysCalc (if installed)
#**************************************
#
0.5 1 2 = sys_scalefact # factorization/renormalization scale factor
None = sys_alpsfact # \alpha_s emission scale factors
auto = sys_matchscale # variation of merging scale
# PDF sets and number of members (0 or none for all members).
NNPDF23_lo_as_0130_qed = sys_pdf # list of pdf sets
# MSTW2008nlo68cl.LHgrid 1 = sys_pdf
#