Skip to content

Commit c6e0fcd

Browse files
sawenzelclaude
andcommitted
Factor the primary request out of the sim worker kernel
This moves the request for a chunk of primaries out of O2SimDevice::Kernel into O2SimDevice::fetchChunk, without changing the behaviour. - fetchChunk sends the request, waits for the answer, backs off while the server is busy and decodes the chunk. - It reports whether there is work, nothing yet, the end-of-work marker, or nothing more to ask for, and Kernel acts on that as before. This lets a multithreaded Geant4 worker request its work the same way as a worker process does now. https://its.cern.ch/jira/browse/O2-7305 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent edc3fc1 commit c6e0fcd

2 files changed

Lines changed: 112 additions & 86 deletions

File tree

‎run/O2SimDevice.cxx‎

Lines changed: 96 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
#include <fairlogger/Logger.h>
2020
#include <DetectorsBase/Stack.h>
2121
#include <DetectorsBase/VMCSeederService.h>
22-
#include <SimulationDataFormat/PrimaryChunk.h>
2322
#include <TRandom.h>
2423
#include <SimConfig/SimConfig.h>
2524
#include <cstring>
@@ -130,7 +129,6 @@ bool O2SimDevice::isWorkAvailable(fair::mq::Channel& statuschannel, int workerID
130129

131130
bool O2SimDevice::Kernel(int workerID, fair::mq::Channel& requestchannel, fair::mq::Channel& dataoutchannel, fair::mq::Channel* statuschannel)
132131
{
133-
static int counter = 0;
134132
bool reproducibleSim = true;
135133
if (getenv("O2_DISABLE_REPRODUCIBLE_SIM")) {
136134
reproducibleSim = false;
@@ -158,9 +156,6 @@ bool O2SimDevice::Kernel(int workerID, fair::mq::Channel& requestchannel, fair::
158156
focus_on_part = std::atoi(p.second.c_str());
159157
}
160158

161-
fair::mq::MessagePtr request(requestchannel.NewSimpleMessage(PrimaryChunkRequest{workerID, -1, counter++})); // <-- don't need content; channel means -> give primaries
162-
fair::mq::Parts reply;
163-
164159
mVMCApp->setSimDataChannel(&dataoutchannel);
165160

166161
// we log info with workerID prepended
@@ -170,92 +165,107 @@ bool O2SimDevice::Kernel(int workerID, fair::mq::Channel& requestchannel, fair::
170165
return str.str();
171166
};
172167

168+
auto fetched = fetchChunk(workerID, requestchannel);
169+
if (fetched.status == ChunkStatus::Stop) {
170+
return false;
171+
}
172+
if (fetched.status != ChunkStatus::Work) {
173+
// EndOfWork deliberately keeps asking: the server answers the next request as idle, which is the Stop
174+
return true;
175+
}
176+
auto& chunk = fetched.chunk;
177+
178+
auto info = chunk->mSubEventInfo;
179+
LOG(info) << workerStr() << " Processing " << chunk->mParticles.size() << " primary particles "
180+
<< "for event " << info.eventID << "/" << info.maxEvents << " "
181+
<< "part " << info.part << "/" << info.nparts;
182+
183+
if (eventselection == nullptr || (focus_on_event == info.eventID && focus_on_part == info.part)) {
184+
mVMCApp->setPrimaries(chunk->mParticles);
185+
} else {
186+
// nothing to transport here
187+
mVMCApp->setPrimaries(std::vector<TParticle>{});
188+
LOG(info) << workerStr() << " This chunk will be skipped";
189+
}
190+
191+
mVMCApp->setSubEventInfo(&info);
192+
193+
if (reproducibleSim) {
194+
LOG(info) << workerStr() << " Setting seed for this sub-event to " << chunk->mSubEventInfo.seed;
195+
gRandom->SetSeed(chunk->mSubEventInfo.seed);
196+
o2::base::VMCSeederService::instance().setSeed();
197+
}
198+
199+
// Process one event
200+
auto& conf = o2::conf::SimConfig::Instance();
201+
if (strcmp(conf.getMCEngine().c_str(), "TGeant4") == 0 || strcmp(conf.getMCEngine().c_str(), "O2TrivialMCEngine") == 0) {
202+
// this is preferred and necessary for Geant4
203+
// since repeated "ProcessRun" might have significant overheads
204+
mVMC->ProcessEvent();
205+
} else {
206+
// for Geant3 calling ProcessEvent is not enough
207+
// as some hooks are not called
208+
mVMC->ProcessRun(1);
209+
}
210+
211+
FairSystemInfo sysinfo;
212+
LOG(info) << workerStr() << " TIME-STAMP " << mTimer.RealTime() << "\t";
213+
mTimer.Continue();
214+
LOG(info) << workerStr() << " MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " "
215+
<< sysinfo.GetMaxMemory() << " MB\n";
216+
return true;
217+
}
218+
219+
O2SimDevice::FetchedChunk O2SimDevice::fetchChunk(int workerID, fair::mq::Channel& requestchannel)
220+
{
221+
static int counter = 0;
222+
fair::mq::MessagePtr request(requestchannel.NewSimpleMessage(PrimaryChunkRequest{workerID, -1, counter++})); // <-- don't need content; channel means -> give primaries
223+
fair::mq::Parts reply;
224+
225+
auto workerStr = [workerID]() {
226+
std::stringstream str;
227+
str << "[W" << workerID << "]";
228+
return str.str();
229+
};
230+
173231
doLogInfo(workerID, "Requesting work chunk");
174232
int timeoutinMS = 2000;
175233
auto sendcode = requestchannel.Send(request, timeoutinMS);
176-
if (sendcode > 0) {
177-
doLogInfo(workerID, "Waiting for answer");
178-
// asking for primary generation
179-
180-
auto code = requestchannel.Receive(reply);
181-
if (code > 0) {
182-
doLogInfo(workerID, "Primary chunk received");
183-
auto rawmessage = std::move(reply.At(0));
184-
auto header = *(o2::PrimaryChunkAnswer*)(rawmessage->GetData());
185-
if (!header.payload_attached) {
186-
doLogInfo(workerID, "No payload; Server in stage " + std::string(PrimStateToString[(int)header.serverstate]));
187-
// if no payload attached we inspect the server state, to see what to do
188-
if (header.serverstate == O2PrimaryServerState::Initializing || header.serverstate == O2PrimaryServerState::WaitingEvent) {
189-
sleep(1); // back-off and retry
190-
return true;
191-
}
192-
// we need to decide what to do when the server is idle ---> if this happens immediately after a new batch request it means that the server might just lag a bit behind
193-
return false;
194-
} else {
195-
auto payload = std::move(reply.At(1));
196-
// wrap incoming bytes as a TMessageWrapper which offers "adoption" of a buffer
197-
auto message = new TMessageWrapper(payload->GetData(), payload->GetSize());
198-
auto chunk = static_cast<o2::data::PrimaryChunk*>(message->ReadObjectAny(message->GetClass()));
199-
200-
bool goon = true;
201-
// no particles and eventID == -1 --> indication for no more work
202-
if (chunk->mParticles.size() == 0 && chunk->mSubEventInfo.eventID == -1) {
203-
doLogInfo(workerID, "No particles in reply : quitting kernel");
204-
goon = false;
205-
}
206-
207-
if (goon) {
208-
209-
auto info = chunk->mSubEventInfo;
210-
LOG(info) << workerStr() << " Processing " << chunk->mParticles.size() << " primary particles "
211-
<< "for event " << info.eventID << "/" << info.maxEvents << " "
212-
<< "part " << info.part << "/" << info.nparts;
213-
214-
if (eventselection == nullptr || (focus_on_event == info.eventID && focus_on_part == info.part)) {
215-
mVMCApp->setPrimaries(chunk->mParticles);
216-
} else {
217-
// nothing to transport here
218-
mVMCApp->setPrimaries(std::vector<TParticle>{});
219-
LOG(info) << workerStr() << " This chunk will be skipped";
220-
}
221-
222-
mVMCApp->setSubEventInfo(&info);
223-
224-
if (reproducibleSim) {
225-
LOG(info) << workerStr() << " Setting seed for this sub-event to " << chunk->mSubEventInfo.seed;
226-
gRandom->SetSeed(chunk->mSubEventInfo.seed);
227-
o2::base::VMCSeederService::instance().setSeed();
228-
}
229-
230-
// Process one event
231-
auto& conf = o2::conf::SimConfig::Instance();
232-
if (strcmp(conf.getMCEngine().c_str(), "TGeant4") == 0 || strcmp(conf.getMCEngine().c_str(), "O2TrivialMCEngine") == 0) {
233-
// this is preferred and necessary for Geant4
234-
// since repeated "ProcessRun" might have significant overheads
235-
mVMC->ProcessEvent();
236-
} else {
237-
// for Geant3 calling ProcessEvent is not enough
238-
// as some hooks are not called
239-
mVMC->ProcessRun(1);
240-
}
241-
242-
FairSystemInfo sysinfo;
243-
LOG(info) << workerStr() << " TIME-STAMP " << mTimer.RealTime() << "\t";
244-
mTimer.Continue();
245-
LOG(info) << workerStr() << " MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " "
246-
<< sysinfo.GetMaxMemory() << " MB\n";
247-
}
248-
delete message;
249-
delete chunk;
250-
}
251-
} else {
252-
LOG(info) << workerStr() << " No primary answer received from server (within timeout). Return code " << code;
253-
}
254-
} else {
234+
if (sendcode <= 0) {
255235
LOG(info) << workerStr() << " Requesting work from server not possible. Return code " << sendcode;
256-
return false;
236+
return {ChunkStatus::Stop, nullptr};
257237
}
258-
return true;
238+
doLogInfo(workerID, "Waiting for answer");
239+
// asking for primary generation
240+
auto code = requestchannel.Receive(reply);
241+
if (code <= 0) {
242+
LOG(info) << workerStr() << " No primary answer received from server (within timeout). Return code " << code;
243+
return {ChunkStatus::Retry, nullptr};
244+
}
245+
doLogInfo(workerID, "Primary chunk received");
246+
auto rawmessage = std::move(reply.At(0));
247+
auto header = *(o2::PrimaryChunkAnswer*)(rawmessage->GetData());
248+
if (!header.payload_attached) {
249+
doLogInfo(workerID, "No payload; Server in stage " + std::string(PrimStateToString[(int)header.serverstate]));
250+
// if no payload attached we inspect the server state, to see what to do
251+
if (header.serverstate == O2PrimaryServerState::Initializing || header.serverstate == O2PrimaryServerState::WaitingEvent) {
252+
sleep(1); // back-off and retry
253+
return {ChunkStatus::Retry, nullptr};
254+
}
255+
// we need to decide what to do when the server is idle ---> if this happens immediately after a new batch request it means that the server might just lag a bit behind
256+
return {ChunkStatus::Stop, nullptr};
257+
}
258+
auto payload = std::move(reply.At(1));
259+
// wrap incoming bytes as a TMessageWrapper which offers "adoption" of a buffer
260+
TMessageWrapper message(payload->GetData(), payload->GetSize());
261+
std::unique_ptr<o2::data::PrimaryChunk> chunk(static_cast<o2::data::PrimaryChunk*>(message.ReadObjectAny(message.GetClass())));
262+
263+
// no particles and eventID == -1 --> indication for no more work
264+
if (chunk->mParticles.size() == 0 && chunk->mSubEventInfo.eventID == -1) {
265+
doLogInfo(workerID, "No particles in reply : quitting kernel");
266+
return {ChunkStatus::EndOfWork, nullptr};
267+
}
268+
return {ChunkStatus::Work, std::move(chunk)};
259269
}
260270

261271
bool O2SimDevice::ConditionalRun()

‎run/O2SimDevice.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <FairRunSim.h>
2121
#include <TStopwatch.h>
2222
#include "PrimaryServerState.h"
23+
#include <SimulationDataFormat/PrimaryChunk.h>
2324

2425
class TVirtualMC;
2526

@@ -60,6 +61,21 @@ class O2SimDevice final : public fair::mq::Device
6061

6162
bool Kernel(int workerID, fair::mq::Channel& requestchannel, fair::mq::Channel& dataoutchannel, fair::mq::Channel* statuschannel = nullptr);
6263

64+
/// outcome of asking the primary server for work
65+
enum class ChunkStatus {
66+
Work, ///< a chunk to transport
67+
Retry, ///< nothing yet (server busy or no answer); ask again
68+
EndOfWork, ///< the server sent the end-of-work marker
69+
Stop ///< the request could not be sent, or the server has nothing more to give; stop asking
70+
};
71+
struct FetchedChunk {
72+
ChunkStatus status;
73+
std::unique_ptr<o2::data::PrimaryChunk> chunk; ///< set for ChunkStatus::Work
74+
};
75+
76+
/// requests one chunk of primaries from the server on the given channel
77+
static FetchedChunk fetchChunk(int workerID, fair::mq::Channel& requestchannel);
78+
6379
protected:
6480
/// Overloads the ConditionalRun() method of fair::mq::Device
6581
bool ConditionalRun() final;

0 commit comments

Comments
 (0)