support MT in ddsim

Author: wdconinc

DDG4/python/DDSim/DD4hepSimulation.py

@@ -71,8 +71,6 @@ def __init__(self):
     self.enableGun = False
     self.enableG4GPS = False
     self.enableG4Gun = False
-    self._g4gun = None
-    self._g4gps = None
     self.vertexSigma = [0.0, 0.0, 0.0, 0.0]
     self.vertexOffset = [0.0, 0.0, 0.0, 0.0]
     self.enableDetailedShowerMode = False
@@ -295,64 +293,14 @@ def getDetectorLists(self, detectorDescription):
 
 # ==================================================================================
 
-  def run(self):
-    """setup the geometry and dd4hep and geant4 and do what was asked to be done"""
-    import ROOT
-    ROOT.PyConfig.IgnoreCommandLineOptions = True
-
+  def __setupActions(self, kernel):
     import DDG4
-    import dd4hep
-
-    self.printLevel = getOutputLevel(self.printLevel)
-
-    kernel = DDG4.Kernel()
-    dd4hep.setPrintLevel(self.printLevel)
-
-    for compactFile in self.compactFile:
-      kernel.loadGeometry(str("file:" + os.path.abspath(compactFile)))
-    detectorDescription = kernel.detectorDescription()
-
-    DDG4.importConstants(detectorDescription)
-
-  # ----------------------------------------------------------------------------------
-
-    # simple = DDG4.Geant4( kernel, tracker='Geant4TrackerAction',calo='Geant4CalorimeterAction')
-    # geant4 = DDG4.Geant4( kernel, tracker='Geant4TrackerCombineAction',calo='Geant4ScintillatorCalorimeterAction')
-    geant4 = DDG4.Geant4(kernel, tracker=self.action.tracker, calo=self.action.calo)
-
-    geant4.printDetectors()
-
-    if self.runType == "vis":
-      uiaction = geant4.setupUI(typ="tcsh", vis=True, macro=self.macroFile)
-    elif self.runType == "qt":
-      uiaction = geant4.setupUI(typ="qt", vis=True, macro=self.macroFile)
-    elif self.runType == "run":
-      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=self.macroFile, ui=False)
-    elif self.runType == "shell":
-      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=None, ui=True)
-    elif self.runType == "batch":
-      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=None, ui=False)
-    else:
-      logger.error("unknown runType")
-      exit(1)
 
-    # User Configuration for the Geant4Phases
-    uiaction.ConfigureCommands = self.ui._commandsConfigure
-    uiaction.InitializeCommands = self.ui._commandsInitialize
-    uiaction.PostRunCommands = self.ui._commandsPostRun
-    uiaction.PreRunCommands = self.ui._commandsPreRun
-    uiaction.TerminateCommands = self.ui._commandsTerminate
+    # Configure default run action
+    run = DDG4.RunAction(kernel, 'Geant4TestRunAction/RunInit')
+    kernel.registerGlobalAction(run)
+    kernel.runAction().add(run)
 
-    kernel.NumEvents = self.numberOfEvents
-
-    # -----------------------------------------------------------------------------------
-    # setup the magnetic field:
-    self.__setMagneticFieldOptions(geant4)
-
-    # configure geometry creation
-    self.geometry.constructGeometry(kernel, geant4, self.output.geometry)
-
-    # ----------------------------------------------------------------------------------
     # Configure run, event, track, step, and stack actions, if present
     for action_list, DDG4_Action, kernel_Action in \
         [(self.action.run, DDG4.RunAction, kernel.runAction),
@@ -366,17 +314,10 @@ def run(self):
           setattr(action, parameter, value)
         kernel_Action().add(action)
 
-    # ----------------------------------------------------------------------------------
-    # Configure Run actions
-    run1 = DDG4.RunAction(kernel, 'Geant4TestRunAction/RunInit')
-    kernel.registerGlobalAction(run1)
-    kernel.runAction().add(run1)
-
-    # Configure the random seed, do it before the I/O because we might change the seed!
-    self.random.initialize(DDG4, kernel, self.output.random)
+    return 1
 
-    # Configure the output file format and plugin
-    self.outputConfig.initialize(dd4hepsimulation=self, geant4=geant4)
+  def __setupGeneratorActions(self, kernel, geant4):
+    import DDG4
 
     actionList = []
 
@@ -391,19 +332,19 @@ def run(self):
 
     if self.enableG4Gun:
       # GPS Create something
-      self._g4gun = DDG4.GeneratorAction(kernel, "Geant4GeneratorWrapper/Gun")
-      self._g4gun.Uses = 'G4ParticleGun'
-      self._g4gun.Mask = 2
+      g4gun = DDG4.GeneratorAction(kernel, "Geant4GeneratorWrapper/Gun")
+      g4gun.Uses = 'G4ParticleGun'
+      g4gun.Mask = 2
       logger.info("++++ Adding Geant4 Particle Gun ++++")
-      actionList.append(self._g4gun)
+      actionList.append(g4gun)
 
     if self.enableG4GPS:
       # GPS Create something
-      self._g4gps = DDG4.GeneratorAction(kernel, "Geant4GeneratorWrapper/GPS")
-      self._g4gps.Uses = 'G4GeneralParticleSource'
-      self._g4gps.Mask = 3
+      g4gps = DDG4.GeneratorAction(kernel, "Geant4GeneratorWrapper/GPS")
+      g4gps.Uses = 'G4GeneralParticleSource'
+      g4gps.Mask = 3
       logger.info("++++ Adding Geant4 General Particle Source ++++")
-      actionList.append(self._g4gps)
+      actionList.append(g4gps)
 
     start = 4
     for index, plugin in enumerate(self.inputConfig.userInputPlugin, start=start):
@@ -452,8 +393,6 @@ def run(self):
       self._buildInputStage(geant4, actionList, output_level=self.output.inputStage,
                             have_mctruth=self._enablePrimaryHandler())
 
-    # ================================================================================================
-
     # And handle the simulation particles.
     part = DDG4.GeneratorAction(kernel, "Geant4ParticleHandler/ParticleHandler")
     kernel.generatorAction().adopt(part)
@@ -472,31 +411,134 @@ def run(self):
 
     self.part.setupUserParticleHandler(part, kernel, DDG4)
 
-    # =================================================================================
+    return 1
+
+  def __setupSensitives(geant4, detectorDescription, sim):
+    kernel = geant4.kernel()
 
     # Setup global filters for use in sensitive detectors
     try:
-      self.filter.setupFilters(kernel)
+      sim.filter.setupFilters(kernel)
     except RuntimeError as e:
       logger.error("%s", e)
       exit(1)
 
-    # =================================================================================
     # get lists of trackers and calorimeters in detectorDescription
-
-    trk, cal, unk = self.getDetectorLists(detectorDescription)
+    trk, cal, unk = sim.getDetectorLists(detectorDescription)
     for detectors, function, defFilter, defAction, abort in \
-        [(trk, geant4.setupTracker, self.filter.tracker, self.action.tracker, False),
-         (cal, geant4.setupCalorimeter, self.filter.calo, self.action.calo, False),
+        [(trk, geant4.setupTracker, sim.filter.tracker, sim.action.tracker, False),
+         (cal, geant4.setupCalorimeter, sim.filter.calo, sim.action.calo, False),
          (unk, geant4.setupDetector, None, "No Default", True),
          ]:
       try:
-        self.__setupSensitiveDetectors(detectors, function, defFilter, defAction, abort)
+        sim.__setupSensitiveDetectors(detectors, function, defFilter, defAction, abort)
       except Exception as e:
         logger.error("Failed setting up sensitive detector %s", e)
         raise
 
-  # =================================================================================
+    return 1
+
+  def __setupWorker(geant4, sim):
+    logger.debug("Setting up worker")
+    kernel = geant4.kernel()
+    logger.debug("Setting up actions")
+    sim.__setupActions(kernel)
+    logger.debug("Setting up generator actions")
+    sim.__setupGeneratorActions(kernel, geant4)
+    return 1
+
+  def __setupMaster(geant4):
+    logger.debug("Setting up master")
+    kernel = geant4.master()
+    return 1
+
+  def run(self):
+    """setup the geometry and dd4hep and geant4 and do what was asked to be done"""
+    import ROOT
+    ROOT.PyConfig.IgnoreCommandLineOptions = True
+
+    import DDG4
+    import dd4hep
+
+    self.printLevel = getOutputLevel(self.printLevel)
+
+    kernel = DDG4.Kernel()
+    dd4hep.setPrintLevel(self.printLevel)
+
+    for compactFile in self.compactFile:
+      kernel.loadGeometry(str("file:" + os.path.abspath(compactFile)))
+    detectorDescription = kernel.detectorDescription()
+
+    DDG4.importConstants(detectorDescription)
+
+    # ----------------------------------------------------------------------------------
+
+    geant4 = DDG4.Geant4(kernel, tracker=self.action.tracker, calo=self.action.calo)
+
+    geant4.printDetectors()
+
+    if self.runType == "vis":
+      uiaction = geant4.setupUI(typ="tcsh", vis=True, macro=self.macroFile)
+    elif self.runType == "qt":
+      uiaction = geant4.setupUI(typ="qt", vis=True, macro=self.macroFile)
+    elif self.runType == "run":
+      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=self.macroFile, ui=False)
+    elif self.runType == "shell":
+      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=None, ui=True)
+    elif self.runType == "batch":
+      uiaction = geant4.setupUI(typ="tcsh", vis=False, macro=None, ui=False)
+    else:
+      logger.error("unknown runType")
+      exit(1)
+
+    # User Configuration for the Geant4Phases
+    uiaction.ConfigureCommands = self.ui._commandsConfigure
+    uiaction.InitializeCommands = self.ui._commandsInitialize
+    uiaction.PostRunCommands = self.ui._commandsPostRun
+    uiaction.PreRunCommands = self.ui._commandsPreRun
+    uiaction.TerminateCommands = self.ui._commandsTerminate
+
+    kernel.NumEvents = self.numberOfEvents
+
+    if self.numberOfThreads > 1:
+      logger.info("Multi-threaded with %d threads", self.numberOfThreads)
+      kernel.RunManagerType = "G4MTRunManager"
+      kernel.NumberOfThreads = self.numberOfThreads
+      geant4.addUserInitialization(
+        worker=DD4hepSimulation.__setupWorker, worker_args=(geant4, self),
+        master=DD4hepSimulation.__setupMaster, master_args=(geant4,))
+    else:
+      kernel.RunManagerType = "G4RunManager"
+      kernel.NumberOfThreads = 1
+      geant4.addUserInitialization(
+        worker=DD4hepSimulation.__setupWorker, worker_args=(geant4, self))
+
+    # -----------------------------------------------------------------------------------
+
+    logger.info("#  Configure G4 geometry setup")
+    seq, act = geant4.addDetectorConstruction("Geant4DetectorGeometryConstruction/ConstructGeo")
+
+    logger.info("# Configure G4 sensitive detectors: python setup callback")
+    seq, act = geant4.addDetectorConstruction(
+      "Geant4PythonDetectorConstruction/SetupSD",
+      sensitives=DD4hepSimulation.__setupSensitives,
+      sensitives_args=(geant4, detectorDescription, self))
+    logger.info("# Configure G4 sensitive detectors: atach'em to the sensitive volumes")
+    seq, act = geant4.addDetectorConstruction("Geant4DetectorSensitivesConstruction/ConstructSD")
+
+    # setup the magnetic field:
+    logger.info("Setting magnetic field")
+    self.__setMagneticFieldOptions(geant4)
+
+    # Configure the random seed, do it before the I/O because we might change the seed!
+    logger.info("Initializing random")
+    self.random.initialize(DDG4, kernel, self.output.random)
+
+    # Configure the output file format and plugin
+    logger.info("Initializing output")
+    self.outputConfig.initialize(dd4hepsimulation=self, geant4=geant4)
+
+    # =================================================================================
     # Now build the physics list:
     _phys = self.physics.setupPhysics(kernel, name=self.physicsList)
 
@@ -507,19 +549,9 @@ def run(self):
 
     dd4hep.setPrintLevel(self.printLevel)
 
-    kernel.configure()
-    kernel.initialize()
-
-    # GPS
-    if self._g4gun is not None:
-      self._g4gun.generator()
-    if self._g4gps is not None:
-      self._g4gps.generator()
-
     startUpTime, _sysTime, _cuTime, _csTime, _elapsedTime = os.times()
 
-    kernel.run()
-    kernel.terminate()
+    geant4.run()
 
     totalTimeUser, totalTimeSys, _cuTime, _csTime, _elapsedTime = os.times()
     if self.printLevel <= 3: