otsdaq_mu2e  v2_04_02
DTCFrontEndInterface_interface.cc
1 #include "otsdaq-mu2e/FEInterfaces/DTCFrontEndInterface.h"
2 #include "otsdaq/Macros/BinaryStringMacros.h"
3 #include "otsdaq/Macros/InterfacePluginMacros.h"
4 #include "otsdaq/PluginMakers/MakeInterface.h"
5 
6 using namespace ots;
7 
8 #undef __MF_SUBJECT__
9 #define __MF_SUBJECT__ "DTCFrontEndInterface"
10 // some global variables, probably a bad idea. But temporary
11 std::string RunDataFN = "";
12 std::fstream DataFile;
13 int FEWriteFile = 0;
14 bool artdaqMode_ = true;
15 //=========================================================================================
16 DTCFrontEndInterface::DTCFrontEndInterface(
17  const std::string& interfaceUID,
18  const ConfigurationTree& theXDAQContextConfigTree,
19  const std::string& interfaceConfigurationPath)
20  : CFOandDTCCoreVInterface(
21  interfaceUID, theXDAQContextConfigTree, interfaceConfigurationPath)
22  , thisDTC_(0)
23  , EmulatedCFO_(0)
24 {
25  __FE_COUT__ << "instantiate DTC... " << interfaceUID << " "
26  << theXDAQContextConfigTree << " " << interfaceConfigurationPath << __E__;
27 
28  // universalAddressSize_ = sizeof(dtc_address_t);
29  // universalDataSize_ = sizeof(dtc_data_t);
30  //
31  // configure_clock_ = getSelfNode().getNode("ConfigureClock").getValue<bool>();
32  emulate_cfo_ = getSelfNode().getNode("EmulateCFO").getValue<bool>();
33 
34  // label
35  // device_name_ = interfaceUID;
36 
37  // // linux file to communicate with
38  // dtc_ = getSelfNode().getNode("DeviceIndex").getValue<unsigned int>();
39  //
40  // try
41  // {
42  // emulatorMode_ = getSelfNode().getNode("EmulatorMode").getValue<bool>();
43  // }
44  // catch(...)
45  // {
46  // __FE_COUT__ << "Assuming NOT emulator mode." << __E__;
47  // emulatorMode_ = false;
48  // }
49 
50  if(emulatorMode_)
51  {
52  __FE_COUT__ << "Emulator DTC mode starting up..." << __E__;
53  createROCs();
54  registerFEMacros();
55  return;
56  }
57  // else not emulator mode
58 
59  // snprintf(devfile_, 11, "/dev/" MU2E_DEV_FILE, dtc_);
60  // fd_ = open(devfile_, O_RDONLY);
61 
62  unsigned dtc_class_roc_mask = 0;
63  // create roc mask for DTC
64  {
65  std::vector<std::pair<std::string, ConfigurationTree>> rocChildren =
66  Configurable::getSelfNode().getNode("LinkToROCGroupTable").getChildren();
67 
68  roc_mask_ = 0;
69 
70  for(auto& roc : rocChildren)
71  {
72  __FE_COUT__ << "roc uid " << roc.first << __E__;
73  bool enabled = roc.second.getNode("Status").getValue<bool>();
74  __FE_COUT__ << "roc enable " << enabled << __E__;
75 
76  if(enabled)
77  {
78  int linkID = roc.second.getNode("linkID").getValue<int>();
79  roc_mask_ |= (0x1 << linkID);
80  dtc_class_roc_mask |=
81  (0x1 << (linkID * 4)); // the DTC class instantiation expects each
82  // ROC has its own hex nibble
83  }
84  }
85 
86  __FE_COUT__ << "DTC roc_mask_ = 0x" << std::hex << roc_mask_ << std::dec << __E__;
87  __FE_COUT__ << "roc_mask to instantiate DTC class = 0x" << std::hex
88  << dtc_class_roc_mask << std::dec << __E__;
89 
90  } // end create roc mask
91 
92  // instantiate DTC with the appropriate ROCs enabled
93  std::string expectedDesignVersion = "";
94  auto mode = DTCLib::DTC_SimMode_NoCFO;
95 
96  __COUT__ << "DTC arguments..." << std::endl;
97  __COUTV__(dtc_);
98  __COUTV__(dtc_class_roc_mask);
99  __COUTV__(expectedDesignVersion);
100  __COUT__ << "END END DTC arguments..." << std::endl;
101 
102  thisDTC_ = new DTCLib::DTC(mode, dtc_, dtc_class_roc_mask, expectedDesignVersion);
103 
104  if(emulate_cfo_ == 1)
105  { // do NOT instantiate the DTCSoftwareCFO here, do it just when you need it
106 
107  // bool useCFOEmulator = true;
108  // uint16_t debugPacketCount = 0;
109  // auto debugType = DTCLib::DTC_DebugType_SpecialSequence;
110  // bool stickyDebugType = true;
111  // bool quiet = false;
112  // bool asyncRR = false;
113  // bool forceNoDebugMode = true;
114 
115  // std::cout << "DTCSoftwareCFO arguments..." << std::endl;
116  // std::cout << "useCFOEmulator = " << useCFOEmulator << std::endl;
117  // std::cout << "packetCount = " << debugPacketCount << std::endl;
118  // std::cout << "debugType = " << debugType << std::endl;
119  // std::cout << "stickyDebugType = " << stickyDebugType << std::endl;
120  // std::cout << "quiet = " << quiet << std::endl;
121  // std::cout << "asyncRR = " << asyncRR << std::endl;
122  // std::cout << "forceNoDebug = " << forceNoDebugMode << std::endl;
123  // std::cout << "END END DTCSoftwareCFO arguments..." << std::endl;
124 
125  // EmulatedCFO_ = new DTCLib::DTCSoftwareCFO(thisDTC_, useCFOEmulator,
126  // debugPacketCount, debugType, stickyDebugType, quiet, asyncRR,
127  // forceNoDebugMode);
128  }
129 
130  createROCs();
131  registerFEMacros();
132 
133  // DTC-specific info
134  dtc_location_in_chain_ =
135  getSelfNode().getNode("LocationInChain").getValue<unsigned int>();
136 
137  // check if any ROCs should be DTC-hardware emulated ROCs
138  {
139  std::vector<std::pair<std::string, ConfigurationTree>> rocChildren =
140  Configurable::getSelfNode().getNode("LinkToROCGroupTable").getChildren();
141 
142  int dtcHwEmulateROCmask = 0;
143  for(auto& roc : rocChildren)
144  {
145  bool enabled = roc.second.getNode("EmulateInDTCHardware").getValue<bool>();
146 
147  if(enabled)
148  {
149  int linkID = roc.second.getNode("linkID").getValue<int>();
150  __FE_COUT__ << "roc uid '" << roc.first << "' at link=" << linkID
151  << " is DTC-hardware emulated!" << __E__;
152  dtcHwEmulateROCmask |= (1 << linkID);
153  }
154  }
155 
156  __FE_COUT__ << "Writing DTC-hardware emulation mask: 0x" << std::hex
157  << dtcHwEmulateROCmask << std::dec << __E__;
158  registerWrite(0x9110, dtcHwEmulateROCmask);
159  __FE_COUT__ << "End check for DTC-hardware emulated ROCs." << __E__;
160  } // end check if any ROCs should be DTC-hardware emulated ROCs
161 
162  // done
163  __MCOUT_INFO__("DTCFrontEndInterface instantiated with name: "
164  << device_name_ << " dtc_location_in_chain_ = "
165  << dtc_location_in_chain_ << " talking to /dev/mu2e" << dtc_ << __E__);
166 } // end constructor()
167 
168 //==========================================================================================
169 DTCFrontEndInterface::~DTCFrontEndInterface(void)
170 {
171  // destroy ROCs before DTC destruction
172  rocs_.clear();
173 
174  if(thisDTC_)
175  delete thisDTC_;
176  // delete theFrontEndHardware_;
177  // delete theFrontEndFirmware_;
178 
179  __FE_COUT__ << "Destructed." << __E__;
180 } // end destructor()
181 
182 //==============================================================================
183 void DTCFrontEndInterface::configureSlowControls(void)
184 {
185  __FE_COUT__ << "Configuring slow controls..." << __E__;
186 
187  // parent configure adds DTC slow controls channels
188  FEVInterface::configureSlowControls(); // also resets DTC-proper channels
189 
190  __FE_COUT__ << "DTC '" << getInterfaceUID()
191  << "' slow controls channel count (BEFORE considering ROCs): "
192  << mapOfSlowControlsChannels_.size() << __E__;
193 
194  mapOfROCSlowControlsChannels_.clear(); // reset ROC channels
195 
196  // now add ROC slow controls channels
197  ConfigurationTree ROCLink =
198  Configurable::getSelfNode().getNode("LinkToROCGroupTable");
199  if(!ROCLink.isDisconnected())
200  {
201  std::vector<std::pair<std::string, ConfigurationTree>> rocChildren =
202  ROCLink.getChildren();
203 
204  unsigned int initialChannelCount;
205 
206  for(auto& rocChildPair : rocChildren)
207  {
208  initialChannelCount = mapOfROCSlowControlsChannels_.size();
209 
210  FEVInterface::addSlowControlsChannels(
211  rocChildPair.second.getNode("LinkToSlowControlsChannelTable"),
212  "/" + rocChildPair.first /*subInterfaceID*/,
213  &mapOfROCSlowControlsChannels_);
214 
215  __FE_COUT__ << "ROC '" << getInterfaceUID() << "/" << rocChildPair.first
216  << "' slow controls channel count: "
217  << mapOfROCSlowControlsChannels_.size() - initialChannelCount
218  << __E__;
219 
220  } // end ROC children loop
221 
222  } // end ROC channel handling
223  else
224  __FE_COUT__ << "ROC link disconnected, assuming no ROCs" << __E__;
225 
226  __FE_COUT__ << "DTC '" << getInterfaceUID()
227  << "' slow controls channel count (AFTER considering ROCs): "
228  << mapOfSlowControlsChannels_.size() +
229  mapOfROCSlowControlsChannels_.size()
230  << __E__;
231 
232  __FE_COUT__ << "Done configuring slow controls." << __E__;
233 
234 } // end configureSlowControls()
235 
236 //==============================================================================
237 // virtual in case channels are handled in multiple maps, for example
238 void DTCFrontEndInterface::resetSlowControlsChannelIterator(void)
239 {
240  // call parent
241  FEVInterface::resetSlowControlsChannelIterator();
242 
243  currentChannelIsInROC_ = false;
244 } // end resetSlowControlsChannelIterator()
245 
246 //==============================================================================
247 // virtual in case channels are handled in multiple maps, for example
248 FESlowControlsChannel* DTCFrontEndInterface::getNextSlowControlsChannel(void)
249 {
250  // if finished with DTC slow controls channels, move on to ROC list
251  if(slowControlsChannelsIterator_ == mapOfSlowControlsChannels_.end())
252  {
253  slowControlsChannelsIterator_ = mapOfROCSlowControlsChannels_.begin();
254  currentChannelIsInROC_ = true; // switch to ROC mode
255  }
256 
257  // if finished with ROC list, then done
258  if(slowControlsChannelsIterator_ == mapOfROCSlowControlsChannels_.end())
259  return nullptr;
260 
261  if(currentChannelIsInROC_)
262  {
263  std::vector<std::string> uidParts;
264  StringMacros::getVectorFromString(
265  slowControlsChannelsIterator_->second.interfaceUID_,
266  uidParts,
267  {'/'} /*delimiters*/);
268  if(uidParts.size() != 2)
269  {
270  __FE_SS__ << "Illegal ROC slow controls channel name '"
271  << slowControlsChannelsIterator_->second.interfaceUID_
272  << ".' Format should be DTC/ROC." << __E__;
273  }
274  currentChannelROCUID_ =
275  uidParts[1]; // format DTC/ROC names, take 2nd part as ROC UID
276  }
277  return &(
278  (slowControlsChannelsIterator_++)->second); // return iterator, then increment
279 } // end getNextSlowControlsChannel()
280 
281 //==============================================================================
282 // virtual in case channels are handled in multiple maps, for example
283 unsigned int DTCFrontEndInterface::getSlowControlsChannelCount(void)
284 {
285  return mapOfSlowControlsChannels_.size() + mapOfROCSlowControlsChannels_.size();
286 } // end getSlowControlsChannelCount()
287 
288 //==============================================================================
289 // virtual in case read should be different than universalread
290 void DTCFrontEndInterface::getSlowControlsValue(FESlowControlsChannel& channel,
291  std::string& readValue)
292 {
293  __FE_COUTV__(currentChannelIsInROC_);
294  __FE_COUTV__(currentChannelROCUID_);
295  __FE_COUTV__(universalDataSize_);
296  if(!currentChannelIsInROC_)
297  {
298  readValue.resize(universalDataSize_);
299  universalRead(channel.getUniversalAddress(), &readValue[0]);
300  }
301  else
302  {
303  auto rocIt = rocs_.find(currentChannelROCUID_);
304  if(rocIt == rocs_.end())
305  {
306  __FE_SS__ << "ROC UID '" << currentChannelROCUID_
307  << "' was not found in ROC map." << __E__;
308  ss << "Here are the existing ROCs: ";
309  bool first = true;
310  for(auto& rocPair : rocs_)
311  if(!first)
312  ss << ", " << rocPair.first;
313  else
314  {
315  ss << rocPair.first;
316  first = false;
317  }
318  ss << __E__;
319  __FE_SS_THROW__;
320  }
321  readValue.resize(universalDataSize_);
322  *((uint16_t*)(&readValue[0])) =
323  rocIt->second->readRegister(*((uint16_t*)channel.getUniversalAddress()));
324  }
325 
326  __FE_COUTV__(readValue.size());
327 } // end getNextSlowControlsChannel()
328 
329 //==============================================================================
330 void DTCFrontEndInterface::registerFEMacros(void)
331 {
332  mapOfFEMacroFunctions_.clear();
333 
334  // clang-format off
335  registerFEMacroFunction(
336  "ROC_WriteBlock", // feMacroName
337  static_cast<FEVInterface::frontEndMacroFunction_t>(
338  &DTCFrontEndInterface::WriteROCBlock), // feMacroFunction
339  std::vector<std::string>{"rocLinkIndex", "block", "address", "writeData"},
340  std::vector<std::string>{}, // namesOfOutputArgs
341  1); // requiredUserPermissions
342 
343  registerFEMacroFunction("ROC_MultipleRead",
344  static_cast<FEVInterface::frontEndMacroFunction_t>(
345  &DTCFrontEndInterface::ReadROCBlock),
346  std::vector<std::string>{"rocLinkIndex", "numberOfWords", "address", "incrementAddress"},
347  std::vector<std::string>{"readData"},
348  1); // requiredUserPermissions
349 
350  registerFEMacroFunction("ROC_ReadBlock",
351  static_cast<FEVInterface::frontEndMacroFunction_t>(
352  &DTCFrontEndInterface::BlockReadROC),
353  std::vector<std::string>{"rocLinkIndex", "block", "address"},
354  std::vector<std::string>{"readData"},
355  1); // requiredUserPermissions
356 
357  // registration of FEMacro 'DTCStatus' generated, Oct-22-2018 03:16:46, by
358  // 'admin' using MacroMaker.
359  registerFEMacroFunction(
360  "ROC_Write", // feMacroName
361  static_cast<FEVInterface::frontEndMacroFunction_t>(
362  &DTCFrontEndInterface::WriteROC), // feMacroFunction
363  std::vector<std::string>{"rocLinkIndex", "address", "writeData"},
364  std::vector<std::string>{}, // namesOfOutput
365  1); // requiredUserPermissions
366 
367  registerFEMacroFunction(
368  "ROC_Read",
369  static_cast<FEVInterface::frontEndMacroFunction_t>(
370  &DTCFrontEndInterface::ReadROC), // feMacroFunction
371  std::vector<std::string>{"rocLinkIndex", "address"}, // namesOfInputArgs
372  std::vector<std::string>{"readData"},
373  1); // requiredUserPermissions
374 
375  registerFEMacroFunction("DTC_Reset",
376  static_cast<FEVInterface::frontEndMacroFunction_t>(
377  &DTCFrontEndInterface::DTCReset),
378  std::vector<std::string>{},
379  std::vector<std::string>{},
380  1); // requiredUserPermissions
381 
382  registerFEMacroFunction("DTC_HighRate_DCS_Check",
383  static_cast<FEVInterface::frontEndMacroFunction_t>(
384  &DTCFrontEndInterface::DTCHighRateDCSCheck),
385  std::vector<std::string>{"rocLinkIndex","loops","baseAddress",
386  "correctRegisterValue0","correctRegisterValue1"},
387  std::vector<std::string>{},
388  1); // requiredUserPermissions
389 
390  registerFEMacroFunction("DTC_HighRate_DCS_Block_Check",
391  static_cast<FEVInterface::frontEndMacroFunction_t>(
392  &DTCFrontEndInterface::DTCHighRateBlockCheck),
393  std::vector<std::string>{"rocLinkIndex","loops","baseAddress",
394  "correctRegisterValue0","correctRegisterValue1"},
395  std::vector<std::string>{},
396  1); // requiredUserPermissions
397 
398  registerFEMacroFunction("DTC_SendHeartbeatAndDataRequest",
399  static_cast<FEVInterface::frontEndMacroFunction_t>(
400  &DTCFrontEndInterface::DTCSendHeartbeatAndDataRequest),
401  std::vector<std::string>{"numberOfRequests","timestampStart"},
402  std::vector<std::string>{"readData"},
403  1); // requiredUserPermissions
404 
405 
406  { //add ROC FE Macros
407  __FE_COUT__ << "Getting children ROC FEMacros..." << __E__;
408  rocFEMacroMap_.clear();
409  for(auto& roc : rocs_)
410  {
411  auto feMacros = roc.second->getMapOfFEMacroFunctions();
412  for(auto& feMacro:feMacros)
413  {
414  __FE_COUT__ << roc.first << "::" << feMacro.first << __E__;
415 
416  //make DTC FEMacro forwarding to ROC FEMacro
417  std::string macroName =
418  "Link" +
419  std::to_string(roc.second->getLinkID()) +
420  "_" + roc.first + "_" +
421  feMacro.first;
422  __FE_COUTV__(macroName);
423 
424  std::vector<std::string> inputArgs,outputArgs;
425 
426  //inputParams.push_back("ROC_UID");
427  //inputParams.push_back("ROC_FEMacroName");
428  for(auto& inArg: feMacro.second.namesOfInputArguments_)
429  inputArgs.push_back(inArg);
430  for(auto& outArg: feMacro.second.namesOfOutputArguments_)
431  outputArgs.push_back(outArg);
432 
433  __FE_COUTV__(StringMacros::vectorToString(inputArgs));
434  __FE_COUTV__(StringMacros::vectorToString(outputArgs));
435 
436  rocFEMacroMap_.emplace(std::make_pair(macroName,
437  std::make_pair(roc.first,feMacro.first)));
438 
439  registerFEMacroFunction(macroName,
440  static_cast<FEVInterface::frontEndMacroFunction_t>(
441  &DTCFrontEndInterface::RunROCFEMacro),
442  inputArgs,
443  outputArgs,
444  1); // requiredUserPermissions
445  }
446  }
447  } //end add ROC FE Macros
448 
449  // clang-format on
450 
451 } // end registerFEMacros()
452 
453 //==============================================================================
454 void DTCFrontEndInterface::createROCs(void)
455 {
456  rocs_.clear();
457 
458  std::vector<std::pair<std::string, ConfigurationTree>> rocChildren =
459  Configurable::getSelfNode().getNode("LinkToROCGroupTable").getChildren();
460 
461  // instantiate vector of ROCs
462  for(auto& roc : rocChildren)
463  if(roc.second.getNode("Status").getValue<bool>())
464  {
465  __FE_COUT__
466  << "ROC Plugin Name: "
467  << roc.second.getNode("ROCInterfacePluginName").getValue<std::string>()
468  << std::endl;
469  __FE_COUT__ << "ROC Name: " << roc.first << std::endl;
470 
471  try
472  {
473  __COUTV__(theXDAQContextConfigTree_.getValueAsString());
474  __COUTV__(
475  roc.second.getNode("ROCInterfacePluginName").getValue<std::string>());
476 
477  // Note: FEVInterface makeInterface returns a unique_ptr
478  // and we want to verify that ROCCoreVInterface functionality
479  // is there, so we do an intermediate dynamic_cast to check
480  // before placing in a new unique_ptr of type ROCCoreVInterface.
481  std::unique_ptr<FEVInterface> tmpVFE = makeInterface(
482  roc.second.getNode("ROCInterfacePluginName").getValue<std::string>(),
483  roc.first,
484  theXDAQContextConfigTree_,
485  (theConfigurationPath_ + "/LinkToROCGroupTable/" + roc.first));
486 
487  // setup parent supervisor of FEVinterface (for backwards compatibility,
488  // left out of constructor)
489  tmpVFE->parentSupervisor_ = parentSupervisor_;
490 
491  ROCCoreVInterface& tmpRoc = dynamic_cast<ROCCoreVInterface&>(
492  *tmpVFE); // dynamic_cast<ROCCoreVInterface*>(tmpRoc.get());
493 
494  // setup other members of ROCCore (for interface plug-in compatibility,
495  // left out of constructor)
496 
497  __COUTV__(tmpRoc.emulatorMode_);
498  tmpRoc.emulatorMode_ = emulatorMode_;
499  __COUTV__(tmpRoc.emulatorMode_);
500 
501  if(emulatorMode_)
502  {
503  __FE_COUT__ << "Creating ROC in emulator mode..." << __E__;
504 
505  // try
506  {
507  // all ROCs support emulator mode
508 
509  // // verify ROCCoreVEmulator class
510  // functionality with dynamic_cast
511  // ROCCoreVEmulator& tmpEmulator =
512  // dynamic_cast<ROCCoreVEmulator&>(
513  // tmpRoc); //
514  // dynamic_cast<ROCCoreVInterface*>(tmpRoc.get());
515 
516  // start emulator thread
517  std::thread(
518  [](ROCCoreVInterface* rocEmulator) {
519  __COUT__ << "Starting ROC emulator thread..." << __E__;
520  ROCCoreVInterface::emulatorThread(rocEmulator);
521  },
522  &tmpRoc)
523  .detach();
524  }
525  // catch(const std::bad_cast& e)
526  // {
527  // __SS__ << "Cast to ROCCoreVEmulator failed!
528  // Verify the emulator " "plugin
529  // inherits
530  // from ROCCoreVEmulator."
531  // << __E__;
532  // ss << "Failed to instantiate plugin named '"
533  //<< roc.first
534  // << "' of type '"
535  // <<
536  // roc.second.getNode("ROCInterfacePluginName")
537  // .getValue<std::string>()
538  // << "' due to the following error: \n"
539  // << e.what() << __E__;
540  //
541  // __SS_THROW__;
542  // }
543  }
544  else
545  {
546  tmpRoc.thisDTC_ = thisDTC_;
547  }
548 
549  rocs_.emplace(std::pair<std::string, std::unique_ptr<ROCCoreVInterface>>(
550  roc.first, &tmpRoc));
551  tmpVFE.release(); // release the FEVInterface unique_ptr, so we are left
552  // with just one
553 
554  __COUTV__(rocs_[roc.first]->emulatorMode_);
555  }
556  catch(const cet::exception& e)
557  {
558  __SS__ << "Failed to instantiate plugin named '" << roc.first
559  << "' of type '"
560  << roc.second.getNode("ROCInterfacePluginName")
561  .getValue<std::string>()
562  << "' due to the following error: \n"
563  << e.what() << __E__;
564  __FE_COUT_ERR__ << ss.str();
565  __MOUT_ERR__ << ss.str();
566  __SS_ONLY_THROW__;
567  }
568  catch(const std::bad_cast& e)
569  {
570  __SS__ << "Cast to ROCCoreVInterface failed! Verify the plugin inherits "
571  "from ROCCoreVInterface."
572  << __E__;
573  ss << "Failed to instantiate plugin named '" << roc.first << "' of type '"
574  << roc.second.getNode("ROCInterfacePluginName").getValue<std::string>()
575  << "' due to the following error: \n"
576  << e.what() << __E__;
577 
578  __FE_COUT_ERR__ << ss.str();
579  __MOUT_ERR__ << ss.str();
580 
581  __SS_ONLY_THROW__;
582  }
583  }
584 
585  __FE_COUT__ << "Done creating " << rocs_.size() << " ROC(s)" << std::endl;
586 
587 } // end createROCs
588 
589 //==================================================================================================
590 void DTCFrontEndInterface::readStatus(void)
591 {
592  __FE_COUT__ << device_name_ << " firmware version (0x9004) = 0x" << std::hex
593  << registerRead(0x9004) << __E__;
594 
595  printVoltages();
596 
597  __FE_COUT__ << device_name_ << " temperature = " << readTemperature() << " degC"
598  << __E__;
599 
600  __FE_COUT__ << device_name_ << " SERDES reset........ (0x9118) = 0x" << std::hex
601  << registerRead(0x9118) << __E__;
602  __FE_COUT__ << device_name_ << " SERDES disparity err (0x911c) = 0x" << std::hex
603  << registerRead(0x9118) << __E__;
604  __FE_COUT__ << device_name_ << " SERDES unlock error. (0x9124) = 0x" << std::hex
605  << registerRead(0x9124) << __E__;
606  __FE_COUT__ << device_name_ << " PLL locked.......... (0x9128) = 0x" << std::hex
607  << registerRead(0x9128) << __E__;
608  __FE_COUT__ << device_name_ << " SERDES Rx status.... (0x9134) = 0x" << std::hex
609  << registerRead(0x9134) << __E__;
610  __FE_COUT__ << device_name_ << " SERDES reset done... (0x9138) = 0x" << std::hex
611  << registerRead(0x9138) << __E__;
612  __FE_COUT__ << device_name_ << " link status......... (0x9140) = 0x" << std::hex
613  << registerRead(0x9140) << __E__;
614  __FE_COUT__ << device_name_ << " SERDES ref clk freq. (0x915c) = 0x" << std::hex
615  << registerRead(0x915c) << " = " << std::dec << registerRead(0x915c)
616  << __E__;
617  __FE_COUT__ << device_name_ << " control............. (0x9100) = 0x" << std::hex
618  << registerRead(0x9100) << __E__;
619  __FE_COUT__ << __E__;
620 
621  return;
622 }
623 
624 //==================================================================================================
625 int DTCFrontEndInterface::getROCLinkStatus(int ROC_link)
626 {
627  int overall_link_status = registerRead(0x9140);
628 
629  int ROC_link_status = (overall_link_status >> ROC_link) & 0x1;
630 
631  return ROC_link_status;
632 }
633 
634 int DTCFrontEndInterface::getCFOLinkStatus()
635 {
636  int overall_link_status = registerRead(0x9140);
637 
638  int CFO_link_status = (overall_link_status >> 6) & 0x1;
639 
640  return CFO_link_status;
641 }
642 
643 int DTCFrontEndInterface::checkLinkStatus()
644 {
645  int ROCs_OK = 1;
646 
647  for(int i = 0; i < 8; i++)
648  {
649  //__FE_COUT__ << " check link " << i << " ";
650 
651  if(ROCActive(i))
652  {
653  //__FE_COUT__ << " active... status = " << getROCLinkStatus(i) << __E__;
654  ROCs_OK &= getROCLinkStatus(i);
655  }
656  }
657 
658  if((getCFOLinkStatus() == 1) && ROCs_OK == 1)
659  {
660  // __FE_COUT__ << "DTC links OK = 0x" << std::hex << registerRead(0x9140)
661  //<< std::dec << __E__;
662  // __MOUT__ << "DTC links OK = 0x" << std::hex << registerRead(0x9140) <<
663  // std::dec << __E__;
664 
665  return 1;
666  }
667  else
668  {
669  // __FE_COUT__ << "DTC links not OK = 0x" << std::hex <<
670  // registerRead(0x9140) << std::dec << __E__;
671  // __MOUT__ << "DTC links not OK = 0x" << std::hex << registerRead(0x9140)
672  //<< std::dec << __E__;
673 
674  return 0;
675  }
676 }
677 
678 bool DTCFrontEndInterface::ROCActive(unsigned ROC_link)
679 {
680  // __FE_COUTV__(roc_mask_);
681  // __FE_COUTV__(ROC_link);
682 
683  if(((roc_mask_ >> ROC_link) & 0x01) == 1)
684  {
685  return true;
686  }
687  else
688  {
689  return false;
690  }
691 }
692 
693 //==================================================================================================
694 void DTCFrontEndInterface::configure(void) try
695 {
696  __FE_COUTV__(getIterationIndex());
697  __FE_COUTV__(getSubIterationIndex());
698 
699  if(emulatorMode_)
700  {
701  __FE_COUT__ << "Emulator DTC configuring... # of ROCs = " << rocs_.size()
702  << __E__;
703  for(auto& roc : rocs_)
704  roc.second->configure();
705  return;
706  }
707  __FE_COUT__ << "DTC configuring... # of ROCs = " << rocs_.size() << __E__;
708 
709  // NOTE: otsdaq/xdaq has a soap reply timeout for state transitions.
710  // Therefore, break up configuration into several steps so as to reply before
711  // the time out As well, there is a specific order in which to configure the
712  // links in the chain of CFO->DTC0->DTC1->...DTCN
713 
714  const int number_of_system_configs =
715  -1; // if < 0, keep trying until links are OK.
716  // If > 0, go through configuration steps this many times
717 
718  const int reset_fpga = 1; // 1 = yes, 0 = no
719  const bool config_clock = configure_clock_; // 1 = yes, 0 = no
720  const bool config_jitter_attenuator = configure_clock_; // 1 = yes, 0 = no
721  const int reset_rx = 0; // 1 = yes, 0 = no
722 
723  const int number_of_dtc_config_steps = 7;
724 
725  const int max_number_of_tries = 3; // max number to wait for links OK
726 
727  int number_of_total_config_steps =
728  number_of_system_configs * number_of_dtc_config_steps;
729 
730  int config_step = getIterationIndex();
731  int config_substep = getSubIterationIndex();
732 
733  if(number_of_system_configs > 0)
734  {
735  if(config_step >= number_of_total_config_steps) // done, exit system config
736  return;
737  }
738 
739  // waiting for link loop
740  if(config_substep > 0 && config_substep < max_number_of_tries)
741  { // wait a maximum of 30 seconds
742 
743  const int number_of_link_checks = 10;
744 
745  int link_ok = 0;
746 
747  for(int i = 0; i < number_of_link_checks; i++)
748  {
749  if(checkLinkStatus() == 1)
750  {
751  // links OK, continue with the rest of the configuration
752  __FE_COUT__ << device_name_ << " Link Status is OK = 0x" << std::hex
753  << registerRead(0x9140) << std::dec << __E__;
754 
755  indicateIterationWork();
756  turnOffLED();
757  return;
758  }
759  else if(getCFOLinkStatus() == 0)
760  {
761  // in this case, links will never get to be OK, stop waiting for them
762 
763  __FE_COUT__ << device_name_ << " CFO Link Status is bad = 0x" << std::hex
764  << registerRead(0x9140) << std::dec << __E__;
765  sleep(1);
766 
767  indicateIterationWork();
768  turnOffLED();
769  return;
770  }
771  else
772  {
773  // links still not OK, keep checking...
774 
775  __FE_COUT__ << "Waiting for DTC Link Status = 0x" << std::hex
776  << registerRead(0x9140) << std::dec << __E__;
777  sleep(1);
778  }
779  }
780 
781  indicateSubIterationWork();
782  return;
783  }
784  else if(config_substep > max_number_of_tries)
785  {
786  // wait a maximum of 30 seconds, then stop waiting for them
787 
788  __FE_COUT__ << "Links still bad = 0x" << std::hex << registerRead(0x9140)
789  << std::dec << "... continue" << __E__;
790  indicateIterationWork();
791  turnOffLED();
792  return;
793  }
794 
795  turnOnLED();
796 
797  if((config_step % number_of_dtc_config_steps) == 0)
798  {
799  if(reset_fpga == 1 && config_step < number_of_dtc_config_steps)
800  {
801  // only reset the FPGA the first time through
802 
803  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
804  << " reset FPGA...");
805 
806  int dataInReg = registerRead(0x9100);
807  int dataToWrite = dataInReg | 0x80000000; // bit 31 = DTC Reset FPGA
808  registerWrite(0x9100, dataToWrite);
809  sleep(3);
810 
811  __MCOUT_INFO__("............. firmware version "
812  << std::hex << registerRead(0x9004) << std::dec << __E__);
813 
814  DTCReset();
815  }
816  }
817  else if((config_step % number_of_dtc_config_steps) == 1)
818  {
819  if((config_clock == 1 || emulate_cfo_ == 1) &&
820  config_step < number_of_dtc_config_steps)
821  {
822  // only configure the clock/crystal the first loop through...
823 
824  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
825  << " reset clock..." << __E__);
826 
827  __FE_COUT__ << "DTC - set crystal frequency to 156.25 MHz" << __E__;
828  registerWrite(0x915c, 0x09502F90);
829 
830  // set RST_REG bit
831  registerWrite(0x9168, 0x5d870100);
832  registerWrite(0x916c, 0x00000001);
833 
834  sleep(5);
835 
836  int targetFrequency = 200000000;
837 
838  //--- begin code snippet pulled from: mu2eUtil program_clock -C 2 -F
839  // 200000000
840 
841  // auto oscillator = DTCLib::DTC_OscillatorType_SERDES; //-C 0 = CFO (main
842  // board SERDES clock) auto oscillator = DTCLib::DTC_OscillatorType_DDR;
843  // //-C 1 (DDR clock)
844  auto oscillator =
845  DTCLib::DTC_OscillatorType_Timing; //-C 2 = DTC (with timing card)
846 
847  __FE_COUT__ << "DTC - set oscillator frequency to " << std::dec
848  << targetFrequency << " MHz" << __E__;
849 
850  thisDTC_->SetNewOscillatorFrequency(oscillator, targetFrequency);
851 
852  //--- end code snippet pulled from: mu2eUtil program_clock -C 2 -F
853  // 200000000
854 
855  sleep(5);
856  }
857  else
858  {
859  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
860  << " do NOT reset clock..." << __E__);
861  }
862  }
863  else if((config_step % number_of_dtc_config_steps) == 2)
864  {
865  // configure Jitter Attenuator to recover clock
866 
867  if((config_jitter_attenuator == 1 || emulate_cfo_ == 1) &&
868  config_step < number_of_dtc_config_steps)
869  {
870  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
871  << " configure Jitter Attenuator..." << __E__);
872 
873  configureJitterAttenuator();
874 
875  sleep(5);
876  }
877  else
878  {
879  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
880  << " do NOT configure Jitter Attenuator..." << __E__);
881  }
882  }
883  else if((config_step % number_of_dtc_config_steps) == 3)
884  {
885  // reset CFO links, first what is received from upstream, then what is
886  // transmitted downstream
887 
888  if(emulate_cfo_ == 1)
889  {
890  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
891  << " enable CFO emulation and internal clock");
892  int dataInReg = registerRead(0x9100);
893  int dataToWrite =
894  dataInReg | 0x40808404; // new incantation from Rick K. 12/18/2019m
895  registerWrite(0x9100, dataToWrite);
896 
897  __FE_COUT__ << "....... CFO emulation: turn off Event Windows" << __E__;
898  registerWrite(0x91f0, 0x1000);
899 
900  __FE_COUT__ << "....... CFO emulation: turn off 40MHz marker interval"
901  << __E__;
902  registerWrite(0x91f4, 0x00000000);
903 
904  __FE_COUT__ << "....... CFO emulation: enable heartbeats" << __E__;
905  registerWrite(0x91a8, 0x15000);
906  }
907  else
908  {
909  int dataInReg = registerRead(0x9100);
910  int dataToWrite =
911  dataInReg &
912  0xbfff7fff; // bit 30 = CFO emulation enable; bit 15 CFO emulation mode
913  registerWrite(0x9100, dataToWrite);
914  }
915 
916  // THIS SHOULD NOT BE NECESSARY with firmware version 20181024
917  if(reset_rx == 1)
918  {
919  __FE_COUT__ << "DTC reset CFO link CPLL" << __E__;
920  registerWrite(0x9118, 0x00004000);
921  registerWrite(0x9118, 0x00000000);
922 
923  sleep(3);
924 
925  __FE_COUT__ << "DTC reset CFO link RX" << __E__;
926  registerWrite(0x9118, 0x00400000);
927  registerWrite(0x9118, 0x00000000);
928 
929  sleep(3);
930  }
931  else
932  {
933  __FE_COUT__ << "DTC do NOT reset PLL and CFO RX" << __E__;
934  }
935  //
936  // __FE_COUT__ << "DTC reset CFO link TX" << __E__;
937  // registerWrite(0x9118,0x40000000);
938  // registerWrite(0x9118,0x00000000);
939  //
940  // sleep(6);
941  }
942  else if((config_step % number_of_dtc_config_steps) == 4)
943  {
944  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
945  << " wait for links..." << __E__);
946 
947  // reset ROC links, first what is sent out, then what is received, then
948  // check links
949 
950  // RESETTING THE LINKS SHOULD NOT BE NECESSARY with firmware version
951  // 20181024, however, we DO want to confirm that the links are OK...
952 
953  if(emulate_cfo_ == 1)
954  {
955  __FE_COUT__ << "DTC reset ROC link SERDES CPLLs" << __E__;
956  registerWrite(0x9118, 0x00003f00);
957  registerWrite(0x9118, 0x00000000);
958 
959  sleep(3);
960 
961  __FE_COUT__ << "DTC reset ROC link SERDES TX" << __E__;
962  registerWrite(0x9118, 0x3f000000);
963  registerWrite(0x9118, 0x00000000);
964  }
965  // // WILL NEED TO CONFIGURE THE ROC LINKS HERE BEFORE RESETTING WHAT IS
966  // RECEIVED
967  //
968  // __FE_COUT__ << "DTC reset ROC link SERDES RX" << __E__;
969  // registerWrite(0x9118,0x003f0000);
970  // registerWrite(0x9118,0x00000000);
971  //
972  // sleep(6);
973 
974  indicateSubIterationWork(); // tell state machine to stay in configure state
975  // ("come back to me")
976 
977  return;
978  }
979  else if((config_step % number_of_dtc_config_steps) == 5)
980  {
981  __MCOUT_INFO__("Step " << config_step << ": " << device_name_
982  << " enable markers, Tx, Rx" << __E__);
983 
984  // enable markers, tx and rx
985 
986  int data_to_write = (roc_mask_ << 8) | roc_mask_;
987  __FE_COUT__ << "DTC enable markers - enabled ROC links 0x" << std::hex
988  << data_to_write << std::dec << __E__;
989  registerWrite(0x91f8, data_to_write);
990 
991  data_to_write = 0x4040 | (roc_mask_ << 8) | roc_mask_;
992  __FE_COUT__ << "DTC enable tx and rx - CFO and enabled ROC links 0x" << std::hex
993  << data_to_write << std::dec << __E__;
994  registerWrite(0x9114, data_to_write);
995 
996  // put DTC CFO link output into loopback mode
997  __FE_COUT__ << "DTC set CFO link output loopback mode ENABLE" << __E__;
998 
999  int dataInReg = registerRead(0x9100);
1000  int dataToWrite = dataInReg & 0xefffffff; // bit 28 = 0
1001  registerWrite(0x9100, dataToWrite);
1002 
1003  __MCOUT_INFO__("Step " << config_step << ": " << device_name_ << " configure ROCs"
1004  << __E__);
1005 
1006  bool doConfigureROCs = false;
1007  try
1008  {
1009  doConfigureROCs = Configurable::getSelfNode()
1010  .getNode("EnableROCConfigureStep")
1011  .getValue<bool>();
1012  }
1013  catch(...)
1014  {
1015  } // ignore missing field
1016  if(doConfigureROCs)
1017  for(auto& roc : rocs_)
1018  roc.second->configure();
1019 
1020  sleep(1);
1021  }
1022  else if((config_step % number_of_dtc_config_steps) == 6)
1023  {
1024  if(emulate_cfo_ == 1)
1025  {
1026  __MCOUT_INFO__("Step " << config_step
1027  << ": CFO emulation enable Event start characters "
1028  "and event window interval"
1029  << __E__);
1030 
1031  __FE_COUT__ << "CFO emulation: set Event Window interval" << __E__;
1032  // registerWrite(0x91f0, cfo_emulate_event_window_interval_);
1033  // registerWrite(0x91f0,0x154); //1.7us
1034  // registerWrite(0x91f0, 0x1f40); // 40us
1035  registerWrite(0x91f0, 0x00000000); // for NO markers, write these
1036  // values
1037 
1038  __FE_COUT__ << "CFO emulation: set 40MHz marker interval" << __E__;
1039  // registerWrite(0x91f4, 0x0800);
1040  registerWrite(0x91f4, 0x00000000); // for NO markers, write these
1041  // values
1042 
1043  __FE_COUT__ << "CFO emulation: set heartbeat interval " << __E__;
1044  // registerWrite(0x91a8, 0x0800);
1045  registerWrite(0x91a8, 0x00000000); // for NO markers, write these
1046  // values
1047  }
1048  __MCOUT_INFO__("Step " << config_step << ": " << device_name_ << " configured"
1049  << __E__);
1050 
1051  if(checkLinkStatus() == 1)
1052  {
1053  __MCOUT_INFO__(device_name_ << " links OK 0x" << std::hex
1054  << registerRead(0x9140) << std::dec << __E__);
1055 
1056  sleep(1);
1057  turnOffLED();
1058 
1059  if(number_of_system_configs < 0)
1060  {
1061  return; // links OK, kick out of configure
1062  }
1063  }
1064  else if(config_step > max_number_of_tries)
1065  {
1066  __MCOUT_INFO__(device_name_ << " links not OK 0x" << std::hex
1067  << registerRead(0x9140) << std::dec << __E__);
1068  return;
1069  }
1070  else
1071  {
1072  __MCOUT_INFO__(device_name_ << " links not OK 0x" << std::hex
1073  << registerRead(0x9140) << std::dec << __E__);
1074  }
1075  }
1076 
1077  readStatus(); // spit out link status at every step
1078 
1079  indicateIterationWork(); // otherwise, tell state machine to stay in configure
1080  // state ("come back to me")
1081 
1082  turnOffLED();
1083  return;
1084 } // end configure()
1085 catch(const std::runtime_error& e)
1086 {
1087  __FE_SS__ << "Error caught: " << e.what() << __E__;
1088  __FE_SS_THROW__;
1089 }
1090 catch(...)
1091 {
1092  __FE_SS__ << "Unknown error caught. Check the printouts!" << __E__;
1093  __FE_SS_THROW__;
1094 }
1095 
1096 //==============================================================================
1097 void DTCFrontEndInterface::halt(void)
1098 {
1099  __FE_COUT__ << "Halting..." << __E__;
1100 
1101  for(auto& roc : rocs_) // halt "as usual"
1102  {
1103  roc.second->halt();
1104  }
1105 
1106  rocs_.clear();
1107 
1108  __FE_COUT__ << "Halted." << __E__;
1109 
1110  // __FE_COUT__ << "HALT: DTC status" << __E__;
1111  // readStatus();
1112 } // end halt()
1113 
1114 //==============================================================================
1115 void DTCFrontEndInterface::pause(void)
1116 {
1117  __FE_COUT__ << "Pausing..." << __E__;
1118  for(auto& roc : rocs_) // pause "as usual"
1119  {
1120  roc.second->pause();
1121  }
1122 
1123  // __FE_COUT__ << "PAUSE: DTC status" << __E__;
1124  // readStatus();
1125 
1126  __FE_COUT__ << "Paused." << __E__;
1127 }
1128 
1129 //==============================================================================
1130 void DTCFrontEndInterface::resume(void)
1131 {
1132  __FE_COUT__ << "Resuming..." << __E__;
1133  for(auto& roc : rocs_) // resume "as usual"
1134  {
1135  roc.second->resume();
1136  }
1137 
1138  // __FE_COUT__ << "RESUME: DTC status" << __E__;
1139  // readStatus();
1140 
1141  __FE_COUT__ << "Resumed." << __E__;
1142 } // end resume()
1143 
1144 //==============================================================================
1145 void DTCFrontEndInterface::start(std::string runNumber)
1146 {
1147  if(emulatorMode_)
1148  {
1149  __FE_COUT__ << "Emulator DTC starting... # of ROCs = " << rocs_.size()
1150  << __E__;
1151  for(auto& roc : rocs_) {
1152  __FE_COUT__ << "Starting ROC ";
1153  roc.second->start(runNumber);
1154  __FE_COUT__ << "Done starting ROC";}
1155  return;
1156  }
1157 
1158 
1159  // open a file for this run number to write data to, if it hasn't been opened yet
1160  // define a data file if FEWRITE_RUNFILE environmental variable is not zero
1161  //
1162  // if (std::getenv("FEWRITE_RUNFILE" != NULL)
1163  __FE_COUT__ << " Trying to get the FEWRITE_RUNFILE info";
1164  FEWriteFile = std::atoi(std::getenv("FEWRITE_RUNFILE"));
1165  __FE_COUT__ << "FEWriteFile is " << FEWriteFile;
1166  FEWriteFile = 0;
1167  if (FEWriteFile) {
1168  char* dataPath(std::getenv("OTSDAQ_DATA"));
1169  RunDataFN = std::string(dataPath) + "/RunData_" + runNumber + ".dat";
1170 
1171  __FE_COUT__ << "Run data FN is: "<< RunDataFN;
1172  if (!DataFile.is_open()) {
1173  DataFile.open (RunDataFN, std::ios::out | std::ios::app);
1174 
1175  if (DataFile.fail()) {
1176  __FE_COUT__ << "FAILED to open data file RunData" << RunDataFN;
1177 
1178 
1179  }
1180  else {
1181  __FE_COUT__ << "opened data file RunData" << RunDataFN;
1182  }
1183  }
1184  }
1185 
1186  if(emulatorMode_)
1187  {
1188  __FE_COUT__ << "Emulator DTC starting..." << __E__;
1189  return;
1190  }
1191 
1192 
1193  int numberOfLoopbacks =
1194  getConfigurationManager()
1195  ->getNode("/Mu2eGlobalsTable/SyncDemoConfig/NumberOfLoopbacks")
1196  .getValue<unsigned int>();
1197 
1198  __FE_COUTV__(numberOfLoopbacks);
1199 
1200  int stopIndex = getIterationIndex();
1201 
1202  if(numberOfLoopbacks == 0)
1203  {
1204  for(auto& roc : rocs_) // start "as usual"
1205  {
1206  roc.second->start(runNumber);
1207  }
1208  return;
1209  }
1210 
1211  const int numberOfChains = 1;
1212  int link[numberOfChains] = {0};
1213 
1214  const int numberOfDTCsPerChain = 1;
1215 
1216  const int numberOfROCsPerDTC = 1; // assume these are ROC0 and ROC1
1217 
1218  // To do loopbacks on all CFOs, first have to setup all DTCs, then the CFO
1219  // (this method) work per iteration. Loop back done on all chains (in this
1220  // method), assuming the following order: i DTC0 DTC1 ... DTCN 0 ROC0
1221  // none ... none 1 ROC1 none ... none 2 none ROC0 ... none 3 none
1222  // ROC1 ... none
1223  // ...
1224  // N-1 none none ... ROC0
1225  // N none none ... ROC1
1226 
1227  int totalNumberOfMeasurements =
1228  numberOfChains * numberOfDTCsPerChain * numberOfROCsPerDTC;
1229 
1230  int loopbackIndex = getIterationIndex();
1231 
1232  if(loopbackIndex == 0) // start
1233  {
1234  initial_9100_ = registerRead(0x9100);
1235  initial_9114_ = registerRead(0x9114);
1236  indicateIterationWork();
1237  return;
1238  }
1239 
1240  if(loopbackIndex > totalNumberOfMeasurements) // finish
1241  {
1242  __MCOUT_INFO__(device_name_ << " loopback DONE" << __E__);
1243 
1244  if(checkLinkStatus() == 1)
1245  {
1246  // __MCOUT_INFO__(device_name_ << " links OK 0x" << std::hex <<
1247  // registerRead(0x9140) << std::dec << __E__);
1248  }
1249  else
1250  {
1251  // __MCOUT_INFO__(device_name_ << " links not OK 0x" << std::hex <<
1252  // registerRead(0x9140) << std::dec << __E__);
1253  }
1254 
1255  if(0)
1256  for(auto& roc : rocs_)
1257  {
1258  __MCOUT_INFO__(".... ROC" << roc.second->getLinkID() << "-DTC link lost "
1259  << roc.second->readDTCLinkLossCounter()
1260  << " times");
1261  }
1262 
1263  registerWrite(0x9100, initial_9100_);
1264  registerWrite(0x9114, initial_9114_);
1265 
1266  return;
1267  }
1268 
1269  //=========== Perform loopback=============
1270 
1271  // where are we in the procedure?
1272  unsigned int activeROC = (loopbackIndex - 1) % numberOfROCsPerDTC;
1273 
1274  int activeDTC = -1;
1275 
1276  for(int nDTC = 0; nDTC < numberOfDTCsPerChain; nDTC++)
1277  {
1278  if((loopbackIndex - 1) >= (nDTC * numberOfROCsPerDTC) &&
1279  (loopbackIndex - 1) < ((nDTC + 1) * numberOfROCsPerDTC))
1280  {
1281  activeDTC = nDTC;
1282  }
1283  }
1284 
1285  // __FE_COUT__ << "loopback index = " << loopbackIndex
1286  // << " activeDTC = " << activeDTC
1287  // << " activeROC = " << activeROC
1288  // << __E__;
1289 
1290  if(activeDTC == dtc_location_in_chain_)
1291  {
1292  __FE_COUT__ << "DTC" << activeDTC << "loopback mode ENABLE" << __E__;
1293  int dataInReg = registerRead(0x9100);
1294  int dataToWrite = dataInReg & 0xefffffff; // bit 28 = 0
1295  registerWrite(0x9100, dataToWrite);
1296  }
1297  else
1298  {
1299  // this DTC is lower in chain than the one being looped. Pass the loopback
1300  // signal through
1301  __FE_COUT__ << "active DTC = " << activeDTC
1302  << " is NOT this DTC = " << dtc_location_in_chain_
1303  << "... pass signal through" << __E__;
1304 
1305  int dataInReg = registerRead(0x9100);
1306  int dataToWrite = dataInReg | 0x10000000; // bit 28 = 1
1307  registerWrite(0x9100, dataToWrite);
1308  }
1309 
1310  int ROCToEnable =
1311  0x00004040 |
1312  (0x101 << activeROC); // enables TX and Rx to CFO (bit 6) and appropriate ROC
1313  __FE_COUT__ << "enable ROC " << activeROC << " --> 0x" << std::hex << ROCToEnable
1314  << std::dec << __E__;
1315 
1316  registerWrite(0x9114, ROCToEnable);
1317 
1318  indicateIterationWork(); // FIXME -- go back to including the ROC (could not 'read'
1319  // for some reason)
1320  return;
1321  // Re-align the links for the activeROC
1322  for(auto& roc : rocs_)
1323  {
1324  if(roc.second->getLinkID() == activeROC)
1325  {
1326  __FE_COUT__ << "... ROC realign link... " << __E__;
1327  roc.second->writeRegister(22, 0);
1328  roc.second->writeRegister(22, 1);
1329  }
1330  }
1331 
1332  indicateIterationWork();
1333  return;
1334 } // end start()
1335 
1336 //==============================================================================
1337 void DTCFrontEndInterface::stop(void)
1338 {
1339  // If using artdaq, all data goes through the BoardReader, not here!
1340  if(artdaqMode_){
1341  __FE_COUT__ << "Stopping in artdaqmode" << __E__;
1342  return;
1343  }
1344 
1345  if(emulatorMode_)
1346  {
1347  __FE_COUT__ << "Emulator DTC stopping... # of ROCs = " << rocs_.size()
1348  << __E__;
1349  for(auto& roc : rocs_)
1350  roc.second->stop();
1351  return;
1352  }
1353 
1354 
1355  if (DataFile.is_open()) {
1356  DataFile.close();
1357  __FE_COUT__ << "closed data file";
1358  }
1359 
1360  if(emulatorMode_)
1361  {
1362 
1363  __FE_COUT__ << "Emulator DTC stopping..." << __E__;
1364  return;
1365  }
1366 
1367  int numberOfCAPTANPulses =
1368  getConfigurationManager()
1369  ->getNode("/Mu2eGlobalsTable/SyncDemoConfig/NumberOfCAPTANPulses")
1370  .getValue<unsigned int>();
1371 
1372  __FE_COUTV__(numberOfCAPTANPulses);
1373 
1374  int stopIndex = getIterationIndex();
1375 
1376  if(numberOfCAPTANPulses == 0)
1377  {
1378  for(auto& roc : rocs_) // stop "as usual"
1379  {
1380  roc.second->stop();
1381  }
1382  return;
1383  }
1384 
1385  if(stopIndex == 0)
1386  {
1387  int i = 0;
1388  for(auto& roc : rocs_)
1389  {
1390  // re-align link
1391  roc.second->writeRegister(22, 0);
1392  roc.second->writeRegister(22, 1);
1393 
1394  std::stringstream filename;
1395  filename << "/home/mu2edaq/sync_demo/ots/" << device_name_ << "_ROC"
1396  << roc.second->getLinkID() << "data.txt";
1397  std::string filenamestring = filename.str();
1398  datafile_[i].open(filenamestring);
1399  i++;
1400  }
1401  }
1402 
1403  if(stopIndex > numberOfCAPTANPulses)
1404  {
1405  int i = 0;
1406  for(auto& roc : rocs_)
1407  {
1408  __MCOUT_INFO__(".... ROC" << roc.second->getLinkID() << "-DTC link lost "
1409  << roc.second->readDTCLinkLossCounter()
1410  << " times");
1411  datafile_[i].close();
1412  i++;
1413  }
1414  return;
1415  }
1416 
1417  int i = 0;
1418  __FE_COUT__ << "Entering read timestamp loop..." << __E__;
1419  for(auto& roc : rocs_)
1420  {
1421  int timestamp_data = roc.second->readTimestamp();
1422 
1423  __FE_COUT__ << "Read " << stopIndex << " -> " << device_name_ << " timestamp "
1424  << timestamp_data << __E__;
1425 
1426  datafile_[i] << stopIndex << " " << timestamp_data << std::endl;
1427  i++;
1428  }
1429 
1430  indicateIterationWork();
1431  return;
1432 } // end stop()
1433 
1434 //==============================================================================
1435 //return true to keep running
1436 bool DTCFrontEndInterface::running(void)
1437 {
1438  if(artdaqMode_) {
1439  __FE_COUT__ << "Running in artdaqmode" << __E__;
1440  return true;
1441  }
1442  if(emulatorMode_)
1443  {
1444  __FE_COUT__ << "Emulator DTC running... # of ROCs = " << rocs_.size()
1445  << __E__;
1446  bool stillRunning = false;
1447  for(auto& roc : rocs_)
1448  stillRunning = stillRunning || roc.second->running();
1449 
1450  return stillRunning;
1451  }
1452  // first setup DTC and CFO. This is stolen from "getheartbeatanddatarequest"
1453 
1454  // auto start = DTCLib::DTC_Timestamp(static_cast<uint64_t>(timestampStart));
1455 
1456  std::time_t current_time;
1457  bool incrementTimestamp = true;
1458 
1459  uint32_t cfodelay = 10000; // have no idea what this is, but 1000 didn't work (don't
1460  // know if 10000 works, either)
1461  int requestsAhead = 0;
1462  unsigned int number = -1; // largest number of events?
1463  unsigned int timestampStart = 0;
1464 
1465  auto device = thisDTC_->GetDevice();
1466  auto initTime = device->GetDeviceTime();
1467  device->ResetDeviceTime();
1468  auto afterInit = std::chrono::steady_clock::now();
1469 
1470 
1471  if(emulate_cfo_ == 1)
1472  {
1473  registerWrite(0x9100, 0x40008404); // bit 30 = CFO emulation enable, bit 15 = CFO
1474  // emulation mode, bit 2 = DCS enable
1475  // bit 10 turns off retry which isn't working right now
1476  sleep(1);
1477 
1478  // set number of null heartbeats
1479  // registerWrite(0x91BC, 0x0);
1480  registerWrite(0x91BC, 0x10); // new incantaton from Rick K. 12/18/2019
1481  // sleep(1);
1482 
1483  //# Send data
1484  //#disable 40mhz marker
1485  registerWrite(0x91f4, 0x0);
1486  // sleep(1);
1487 
1488  //#set num dtcs
1489  registerWrite(0x9158, 0x1);
1490  // sleep(1);
1491 
1492  bool useCFOEmulator = true;
1493  uint16_t debugPacketCount = 0;
1494  auto debugType = DTCLib::DTC_DebugType_SpecialSequence;
1495  bool stickyDebugType = true;
1496  bool quiet = false;
1497  bool asyncRR = false;
1498  bool forceNoDebugMode = true;
1499 
1500  DTCLib::DTCSoftwareCFO* EmulatedCFO_ =
1501  new DTCLib::DTCSoftwareCFO(thisDTC_,
1502  useCFOEmulator,
1503  debugPacketCount,
1504  debugType,
1505  stickyDebugType,
1506  quiet,
1507  asyncRR,
1508  forceNoDebugMode);
1509 
1510  EmulatedCFO_->SendRequestsForRange(
1511  number,
1512  DTCLib::DTC_Timestamp(static_cast<uint64_t>(timestampStart)),
1513  incrementTimestamp,
1514  cfodelay,
1515  requestsAhead);
1516 
1517  auto readoutRequestTime = device->GetDeviceTime();
1518  device->ResetDeviceTime();
1519  auto afterRequests = std::chrono::steady_clock::now();
1520 
1521  }
1522 
1523  while(WorkLoop::continueWorkLoop_)
1524  {
1525  for(auto& roc : rocs_)
1526  {
1527  roc.second->running();
1528  }
1529 
1530  // print out stuff
1531  unsigned quietCount = 20;
1532  bool quiet = false;
1533 
1534  std::stringstream ostr;
1535  ostr << std::endl;
1536 
1537 
1538  // std::cout << "Buffer Read " << std::dec << ii << std::endl;
1539  mu2e_databuff_t* buffer;
1540  auto tmo_ms = 1500;
1541  __FE_COUT__ << "util - before read for DAQ in running";
1542  auto sts = device->read_data(
1543  DTC_DMA_Engine_DAQ, reinterpret_cast<void**>(&buffer), tmo_ms);
1544  __FE_COUT__ << "util - after read for DAQ in running " << " sts=" << sts
1545  << ", buffer=" << (void*)buffer;
1546 
1547  if(sts > 0)
1548  {
1549  void* readPtr = &buffer[0];
1550  auto bufSize = static_cast<uint16_t>(*static_cast<uint64_t*>(readPtr));
1551  readPtr = static_cast<uint8_t*>(readPtr) + 8;
1552 
1553  __FE_COUT__ << "Buffer reports DMA size of " << std::dec << bufSize
1554  << " bytes. Device driver reports read of " << sts << " bytes,"
1555  << std::endl;
1556 
1557  __FE_COUT__ << "util - bufSize is " << bufSize;
1558  outputStream.write(static_cast<char*>(readPtr), sts - 8);
1559  auto maxLine = static_cast<unsigned>(ceil((sts - 8) / 16.0));
1560  __FE_COUT__ << "maxLine " << maxLine;
1561  for(unsigned line = 0; line < maxLine; ++line)
1562  {
1563  ostr << "0x" << std::hex << std::setw(5) << std::setfill('0') << line
1564  << "0: ";
1565  for(unsigned byte = 0; byte < 8; ++byte)
1566  {
1567  if(line * 16 + 2 * byte < sts - 8u)
1568  {
1569  auto thisWord =
1570  reinterpret_cast<uint16_t*>(buffer)[4 + line * 8 + byte];
1571  ostr << std::setw(4) << static_cast<int>(thisWord) << " ";
1572  }
1573  }
1574 
1575  ostr << std::endl;
1576  // std::cout << ostr.str();
1577 
1578  // __SET_ARG_OUT__("readData", ostr.str()); // write to data file
1579 
1580  if (FEWriteFile) { // overkill. If the file isn't open, won't try to write.
1581  __FE_COUT__ << "writing to DataFile";
1582  if (DataFile.bad())
1583  __FE_COUT__ << " something bad happened when writing to datafile? \n";
1584  if (!DataFile.is_open())
1585  __FE_COUT__ << "trying to write to the data file but it isnt open. \n";
1586  DataFile << ostr.str();
1587  }
1588  // don't write data to the log file, only the data file
1589  // __FE_COUT__ << ostr.str();
1590 
1591  if(maxLine > quietCount * 2 && quiet && line == (quietCount - 1))
1592  {
1593  line = static_cast<unsigned>(ceil((sts - 8) / 16.0)) -
1594  (1 + quietCount);
1595  }
1596  }
1597  }
1598  device->read_release(DTC_DMA_Engine_DAQ, 1);
1599 
1600  ostr << std::endl;
1601 
1602  if (FEWriteFile) {
1603  DataFile << ostr.str();
1604  DataFile.flush(); // flush to disk
1605  }
1606  //__FE_COUT__ << ostr.str();
1607 
1608  delete EmulatedCFO_;
1609 
1610  break;
1611  }
1612  return true;
1613 }
1614 
1615 //==============================================================================
1616 // rocRead
1617 void DTCFrontEndInterface::ReadROC(__ARGS__)
1618 {
1619  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1620  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1621  for(auto& argIn : argsIn)
1622  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1623 
1624  DTCLib::DTC_Link_ID rocLinkIndex =
1625  DTCLib::DTC_Link_ID(__GET_ARG_IN__("rocLinkIndex", uint8_t));
1626  uint8_t address = __GET_ARG_IN__("address", uint8_t);
1627  __FE_COUTV__(rocLinkIndex);
1628  __FE_COUTV__((unsigned int)address);
1629 
1630  // DTCLib::DTC* tmpDTC = new
1631  // DTCLib::DTC(DTCLib::DTC_SimMode_NoCFO,dtc_,roc_mask_,"");
1632 
1633  uint16_t readData = -999;
1634 
1635  for(auto& roc : rocs_)
1636  {
1637  __FE_COUT__ << "Found link ID " << roc.second->getLinkID() << " looking for "
1638  << rocLinkIndex << __E__;
1639 
1640  if(rocLinkIndex == roc.second->getLinkID())
1641  {
1642  readData = roc.second->readRegister(address);
1643 
1644  // uint16_t readData = thisDTC_->ReadROCRegister(rocLinkIndex, address);
1645  // delete tmpDTC;
1646 
1647  char readDataStr[100];
1648  sprintf(readDataStr,"0x%X",readData);
1649  __SET_ARG_OUT__("readData",readDataStr);
1650  //__SET_ARG_OUT__("readData", readData);
1651 
1652  // for(auto &argOut:argsOut)
1653  __FE_COUT__ << "readData"
1654  << ": " << std::hex << readData << std::dec << __E__;
1655  __FE_COUT__ << "End of Data";
1656  return;
1657  }
1658  }
1659 
1660  __FE_SS__ << "ROC link ID " << rocLinkIndex << " not found!" << __E__;
1661  __FE_SS_THROW__;
1662 
1663 } // end ReadROC()
1664 
1665 //==============================================================================
1666 // DTCStatus
1667 // FEMacro 'DTCStatus' generated, Oct-22-2018 03:16:46, by 'admin' using
1668 // MacroMaker. Macro Notes:
1669 void DTCFrontEndInterface::WriteROC(__ARGS__)
1670 {
1671  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1672  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1673  for(auto& argIn : argsIn)
1674  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1675 
1676  DTCLib::DTC_Link_ID rocLinkIndex =
1677  DTCLib::DTC_Link_ID(__GET_ARG_IN__("rocLinkIndex", uint8_t));
1678  uint8_t address = __GET_ARG_IN__("address", uint8_t);
1679  uint16_t writeData = __GET_ARG_IN__("writeData", uint16_t);
1680 
1681  __FE_COUTV__(rocLinkIndex);
1682  __FE_COUTV__((unsigned int)address);
1683  __FE_COUTV__(writeData);
1684 
1685  __FE_COUT__ << "ROCs size = " << rocs_.size() << __E__;
1686 
1687  for(auto& roc : rocs_)
1688  {
1689  __FE_COUT__ << "Found link ID " << roc.second->getLinkID() << " looking for "
1690  << rocLinkIndex << __E__;
1691 
1692  if(rocLinkIndex == roc.second->getLinkID())
1693  {
1694  roc.second->writeRegister(address, writeData);
1695 
1696  for(auto& argOut : argsOut)
1697  __FE_COUT__ << argOut.first << ": " << argOut.second << __E__;
1698 
1699  return;
1700  }
1701  }
1702 
1703  __FE_SS__ << "ROC link ID " << rocLinkIndex << " not found!" << __E__;
1704  __FE_SS_THROW__;
1705 }
1706 
1707 //==============================================================================
1708 void DTCFrontEndInterface::WriteROCBlock(__ARGS__)
1709 {
1710  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1711  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1712  for(auto& argIn : argsIn)
1713  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1714 
1715  // macro commands section
1716 
1717  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1718  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1719 
1720  for(auto& argIn : argsIn)
1721  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1722 
1723  DTCLib::DTC_Link_ID rocLinkIndex =
1724  DTCLib::DTC_Link_ID(__GET_ARG_IN__("rocLinkIndex", uint8_t));
1725  uint8_t address = __GET_ARG_IN__("address", uint8_t);
1726  uint16_t writeData = __GET_ARG_IN__("writeData", uint16_t);
1727  uint8_t block = __GET_ARG_IN__("block", uint8_t);
1728  __FE_COUTV__(rocLinkIndex);
1729  __FE_COUT__ << "block = " << std::dec << (unsigned int)block << __E__;
1730  __FE_COUT__ << "address = 0x" << std::hex << (unsigned int)address << std::dec
1731  << __E__;
1732  __FE_COUT__ << "writeData = 0x" << std::hex << writeData << std::dec << __E__;
1733 
1734  bool acknowledge_request = false;
1735 
1736  thisDTC_->WriteExtROCRegister(
1737  rocLinkIndex, block, address, writeData, acknowledge_request);
1738 
1739  for(auto& argOut : argsOut)
1740  __FE_COUT__ << argOut.first << ": " << argOut.second << __E__;
1741 }
1742 
1743 //==============================================================================
1744 void DTCFrontEndInterface::BlockReadROC(__ARGS__)
1745 {
1746  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1747  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1748  for(auto& argIn : argsIn)
1749  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1750 
1751  DTCLib::DTC_Link_ID rocLinkIndex =
1752  DTCLib::DTC_Link_ID(__GET_ARG_IN__("rocLinkIndex", uint8_t));
1753  uint8_t address = __GET_ARG_IN__("address", uint8_t);
1754  uint8_t block = __GET_ARG_IN__("block", uint8_t);
1755  __FE_COUTV__(rocLinkIndex);
1756  __FE_COUT__ << "block = " << std::dec << (unsigned int)block << __E__;
1757  __FE_COUT__ << "address = 0x" << std::hex << (unsigned int)address << std::dec
1758  << __E__;
1759 
1760  bool acknowledge_request = false;
1761 
1762  for(auto& roc : rocs_)
1763  {
1764  __FE_COUT__ << "At ROC link ID " << roc.second->getLinkID() << ", looking for "
1765  << rocLinkIndex << __E__;
1766 
1767  if(rocLinkIndex == roc.second->getLinkID())
1768  {
1769  uint16_t readData;
1770 
1771  readData = thisDTC_->ReadExtROCRegister(rocLinkIndex, block, address);
1772 
1773  std::string readDataString = "";
1774  readDataString = BinaryStringMacros::binaryNumberToHexString(readData);
1775 
1776  // StringMacros::vectorToString(readData);
1777 
1778  __SET_ARG_OUT__("readData", readDataString);
1779 
1780  // for(auto &argOut:argsOut)
1781  __FE_COUT__ << "readData"
1782  << ": " << readDataString << __E__;
1783  return;
1784  }
1785  }
1786 
1787  __FE_SS__ << "ROC link ID " << rocLinkIndex << " not found!" << __E__;
1788  __FE_SS_THROW__;
1789 }
1790 
1791 //========================================================================
1792 void DTCFrontEndInterface::ReadROCBlock(__ARGS__)
1793 {
1794  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1795  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1796  for(auto& argIn : argsIn)
1797  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1798 
1799  // macro commands section
1800  __FE_COUT__ << "# of input args = " << argsIn.size() << __E__;
1801  __FE_COUT__ << "# of output args = " << argsOut.size() << __E__;
1802 
1803  for(auto& argIn : argsIn)
1804  __FE_COUT__ << argIn.first << ": " << argIn.second << __E__;
1805 
1806  DTCLib::DTC_Link_ID rocLinkIndex =
1807  DTCLib::DTC_Link_ID(__GET_ARG_IN__("rocLinkIndex", uint8_t));
1808  uint16_t address = __GET_ARG_IN__("address", uint16_t);
1809  uint16_t wordCount = __GET_ARG_IN__("numberOfWords", uint16_t);
1810  bool incrementAddress = __GET_ARG_IN__("incrementAddress", bool);
1811 
1812  __FE_COUTV__(rocLinkIndex);
1813  __FE_COUT__ << "address = 0x" << std::hex << (unsigned int)address << std::dec
1814  << __E__;
1815  __FE_COUT__ << "numberOfWords = " << std::dec << (unsigned int)wordCount << __E__;
1816  __FE_COUTV__(incrementAddress);
1817 
1818  for(auto& roc : rocs_)
1819  {
1820  __FE_COUT__ << "At ROC link ID " << roc.second->getLinkID() << ", looking for "
1821  << rocLinkIndex << __E__;
1822 
1823  if(rocLinkIndex == roc.second->getLinkID())
1824  {
1825  std::vector<uint16_t> readData;
1826 
1827  roc.second->readBlock(readData, address, wordCount, incrementAddress);
1828 
1829  std::string readDataString = "";
1830  {
1831  bool first = true;
1832  for(const auto& data : readData)
1833  {
1834  if(!first)
1835  readDataString += ", ";
1836  else
1837  first = false;
1838  readDataString += BinaryStringMacros::binaryNumberToHexString(data);
1839  }
1840  }
1841  // StringMacros::vectorToString(readData);
1842 
1843  __SET_ARG_OUT__("readData", readDataString);
1844 
1845  // for(auto &argOut:argsOut)
1846  __FE_COUT__ << "readData"
1847  << ": " << readDataString << __E__;
1848  return;
1849  }
1850  }
1851 
1852  __FE_SS__ << "ROC link ID " << rocLinkIndex << " not found!" << __E__;
1853  __FE_SS_THROW__;
1854 
1855 } // end ReadROCBlock()
1856 
1857 //========================================================================
1858 void DTCFrontEndInterface::DTCHighRateBlockCheck(__ARGS__)
1859 {
1860  unsigned int linkIndex = __GET_ARG_IN__("rocLinkIndex", unsigned int);
1861  unsigned int loops = __GET_ARG_IN__("loops", unsigned int);
1862  unsigned int baseAddress = __GET_ARG_IN__("baseAddress", unsigned int);
1863  unsigned int correctRegisterValue0 =
1864  __GET_ARG_IN__("correctRegisterValue0", unsigned int);
1865  unsigned int correctRegisterValue1 =
1866  __GET_ARG_IN__("correctRegisterValue1", unsigned int);
1867 
1868  __FE_COUTV__(linkIndex);
1869  __FE_COUTV__(loops);
1870  __FE_COUTV__(baseAddress);
1871  __FE_COUTV__(correctRegisterValue0);
1872  __FE_COUTV__(correctRegisterValue1);
1873 
1874  for(auto& roc : rocs_)
1875  if(roc.second->getLinkID() == linkIndex)
1876  {
1877  roc.second->highRateBlockCheck(
1878  loops, baseAddress, correctRegisterValue0, correctRegisterValue1);
1879  return;
1880  }
1881 
1882  __FE_SS__ << "Error! Could not find ROC at link index " << linkIndex << __E__;
1883  __FE_SS_THROW__;
1884 
1885 } // end DTCHighRateBlockCheck()
1886 
1887 //========================================================================
1888 void DTCFrontEndInterface::DTCHighRateDCSCheck(__ARGS__)
1889 {
1890  unsigned int linkIndex = __GET_ARG_IN__("rocLinkIndex", unsigned int);
1891  unsigned int loops = __GET_ARG_IN__("loops", unsigned int);
1892  unsigned int baseAddress = __GET_ARG_IN__("baseAddress", unsigned int);
1893  unsigned int correctRegisterValue0 =
1894  __GET_ARG_IN__("correctRegisterValue0", unsigned int);
1895  unsigned int correctRegisterValue1 =
1896  __GET_ARG_IN__("correctRegisterValue1", unsigned int);
1897 
1898  __FE_COUTV__(linkIndex);
1899  __FE_COUTV__(loops);
1900  __FE_COUTV__(baseAddress);
1901  __FE_COUTV__(correctRegisterValue0);
1902  __FE_COUTV__(correctRegisterValue1);
1903 
1904  for(auto& roc : rocs_)
1905  if(roc.second->getLinkID() == linkIndex)
1906  {
1907  roc.second->highRateCheck(
1908  loops, baseAddress, correctRegisterValue0, correctRegisterValue1);
1909  return;
1910  }
1911 
1912  __FE_SS__ << "Error! Could not find ROC at link index " << linkIndex << __E__;
1913  __FE_SS_THROW__;
1914 
1915 } // end DTCHighRateDCSCheck()
1916 
1917 //========================================================================
1918 void DTCFrontEndInterface::DTCSendHeartbeatAndDataRequest(__ARGS__)
1919 {
1920  unsigned int number = __GET_ARG_IN__("numberOfRequests", unsigned int);
1921  unsigned int timestampStart = __GET_ARG_IN__("timestampStart", unsigned int);
1922 
1923  // auto start = DTCLib::DTC_Timestamp(static_cast<uint64_t>(timestampStart));
1924 
1925  bool incrementTimestamp = true;
1926  uint32_t cfodelay = 10000; // have no idea what this is, but 1000 didn't work (don't
1927  // know if 10000 works, either)
1928  int requestsAhead = 0;
1929 
1930  __FE_COUTV__(number);
1931  __FE_COUTV__(timestampStart);
1932 
1933  auto device = thisDTC_->GetDevice();
1934 
1935  auto initTime = device->GetDeviceTime();
1936  device->ResetDeviceTime();
1937  auto afterInit = std::chrono::steady_clock::now();
1938 
1939  if(emulate_cfo_ == 1)
1940  {
1941  registerWrite(0x9100, 0x40008404); // bit 30 = CFO emulation enable, bit 15 = CFO
1942  // emulation mode, bit 2 = DCS enable
1943  // bit 10 turns off retry which isn't working right now
1944  sleep(1);
1945 
1946  // set number of null heartbeats
1947  // registerWrite(0x91BC, 0x0);
1948  registerWrite(0x91BC, 0x10); // new incantaton from Rick K. 12/18/2019
1949  // sleep(1);
1950 
1951  //# Send data
1952  //#disable 40mhz marker
1953  registerWrite(0x91f4, 0x0);
1954  // sleep(1);
1955 
1956  //#set num dtcs
1957  registerWrite(0x9158, 0x1);
1958  // sleep(1);
1959 
1960  bool useCFOEmulator = true;
1961  uint16_t debugPacketCount = 0;
1962  auto debugType = DTCLib::DTC_DebugType_SpecialSequence;
1963  bool stickyDebugType = true;
1964  bool quiet = false;
1965  bool asyncRR = false;
1966  bool forceNoDebugMode = true;
1967 
1968  // std::cout << "DTCSoftwareCFO arguments..." << std::endl;
1969  // std::cout << "useCFOEmulator = " << useCFOEmulator << std::endl;
1970  // std::cout << "packetCount = " << debugPacketCount << std::endl;
1971  // std::cout << "debugType = " << debugType << std::endl;
1972  // std::cout << "stickyDebugType = " << stickyDebugType << std::endl;
1973  // std::cout << "quiet = " << quiet << std::endl;
1974  // std::cout << "asyncRR = " << asyncRR << std::endl;
1975  // std::cout << "forceNoDebug = " << forceNoDebugMode << std::endl;
1976  // std::cout << "END END DTCSoftwareCFO arguments..." << std::endl;
1977 
1978  DTCLib::DTCSoftwareCFO* EmulatedCFO_ =
1979  new DTCLib::DTCSoftwareCFO(thisDTC_,
1980  useCFOEmulator,
1981  debugPacketCount,
1982  debugType,
1983  stickyDebugType,
1984  quiet,
1985  asyncRR,
1986  forceNoDebugMode);
1987 
1988  // std::cout << "SendRequestsForRange arguments..." << std::endl;
1989  // std::cout << "number = " << number << std::endl;
1990  // std::cout << "timestampOffset = " << timestampStart << std::endl;
1991  // std::cout << "incrementTimestamp = " << incrementTimestamp << std::endl;
1992  // std::cout << "cfodelay = " << cfodelay << std::endl;
1993  // std::cout << "requestsAhead = " << requestsAhead << std::endl;
1994  // std::cout << "END END SendRequestsForRange arguments..." << std::endl;
1995 
1996  EmulatedCFO_->SendRequestsForRange(
1997  number,
1998  DTCLib::DTC_Timestamp(static_cast<uint64_t>(timestampStart)),
1999  incrementTimestamp,
2000  cfodelay,
2001  requestsAhead);
2002 
2003  // delete EmulatedCFO_;
2004  // moved to later (after reads)
2005 
2006  auto readoutRequestTime = device->GetDeviceTime();
2007  device->ResetDeviceTime();
2008  auto afterRequests = std::chrono::steady_clock::now();
2009 
2010  // print out stuff
2011  unsigned quietCount = 20;
2012  quiet = false;
2013 
2014  std::stringstream ostr;
2015  ostr << std::endl;
2016 
2017  for(unsigned ii = 0; ii < number; ++ii)
2018  {
2019  __FE_COUT__ << "Buffer Read " << std::dec << ii << std::endl;
2020  mu2e_databuff_t* buffer;
2021  auto tmo_ms = 1500;
2022  __FE_COUT__ << "util - before read for DAQ - ii=" << ii;
2023  auto sts = device->read_data(
2024  DTC_DMA_Engine_DAQ, reinterpret_cast<void**>(&buffer), tmo_ms);
2025  __FE_COUT__ << "util - after read for DAQ - ii=" << ii << ", sts=" << sts
2026  << ", buffer=" << (void*)buffer;
2027 
2028  if(sts > 0)
2029  {
2030  void* readPtr = &buffer[0];
2031  auto bufSize = static_cast<uint16_t>(*static_cast<uint64_t*>(readPtr));
2032  readPtr = static_cast<uint8_t*>(readPtr) + 8;
2033 
2034  std::cout << "Buffer reports DMA size of " << std::dec << bufSize
2035  << " bytes. Device driver reports read of " << sts << " bytes,"
2036  << std::endl;
2037 
2038  std::cout << "util - bufSize is " << bufSize;
2039  // __SET_ARG_OUT__("bufSize", bufSize);
2040 
2041  // __FE_COUT__ << "bufSize" << bufSize;
2042  outputStream.write(static_cast<char*>(readPtr), sts - 8);
2043 
2044  auto maxLine = static_cast<unsigned>(ceil((sts - 8) / 16.0));
2045  std::cout << "maxLine " << maxLine;
2046  for(unsigned line = 0; line < maxLine; ++line)
2047  {
2048  ostr << "0x" << std::hex << std::setw(5) << std::setfill('0') << line
2049  << "0: ";
2050  for(unsigned byte = 0; byte < 8; ++byte)
2051  {
2052  if(line * 16 + 2 * byte < sts - 8u)
2053  {
2054  auto thisWord =
2055  reinterpret_cast<uint16_t*>(buffer)[4 + line * 8 + byte];
2056  ostr << std::setw(4) << static_cast<int>(thisWord) << " ";
2057  }
2058  }
2059 
2060  ostr << std::endl;
2061  // std::cout << ostr.str();
2062 
2063  __FE_COUT__ << ostr.str(); // write to log file
2064 
2065  if(maxLine > quietCount * 2 && quiet && line == (quietCount - 1))
2066  {
2067  line = static_cast<unsigned>(ceil((sts - 8) / 16.0)) -
2068  (1 + quietCount);
2069  }
2070  }
2071  }
2072  device->read_release(DTC_DMA_Engine_DAQ, 1);
2073 
2074  }
2075  ostr << std::endl;
2076 
2077  __SET_ARG_OUT__("readData", ostr.str()); // write to data file
2078 
2079  __FE_COUT__ << ostr.str(); // write to log file
2080 
2081  delete EmulatedCFO_;
2082 
2083 
2084  }
2085  else
2086  {
2087  __FE_SS__ << "Error! DTC must be in CFOEmulate mode" << __E__;
2088  __FE_SS_THROW__;
2089  }
2090 
2091 
2092 } // end DTCSendHeartbeatAndDataRequest()
2093 
2094 //========================================================================
2095 void DTCFrontEndInterface::DTCReset(__ARGS__) { DTCReset(); }
2096 
2097 //========================================================================
2098 void DTCFrontEndInterface::DTCReset()
2099 {
2100  {
2101  char* address = new char[universalAddressSize_]{
2102  0}; //create address buffer of interface size and init to all 0
2103  char* data = new char[universalDataSize_]{
2104  0}; //create data buffer of interface size and init to all 0
2105  uint64_t macroAddress; // create macro address buffer (size 8 bytes)
2106  uint64_t macroData; // create macro address buffer (size 8 bytes)
2107  std::map<std::string /*arg name*/, uint64_t /*arg val*/>
2108  macroArgs; // create map from arg name to 64-bit number
2109 
2110  // command-#0: Write(0x9100 /*address*/,0xa0000000 /*data*/);
2111  macroAddress = 0x9100;
2112  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2113  macroData = 0xa0000000;
2114  memcpy(data, &macroData, 8); // copy macro data to buffer
2115  universalWrite(address, data);
2116 
2117  // command-#1: Write(0x9118 /*address*/,0x0000003f /*data*/);
2118  macroAddress = 0x9118;
2119  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2120  macroData = 0x0000003f;
2121  memcpy(data, &macroData, 8); // copy macro data to buffer
2122  universalWrite(address, data);
2123 
2124  // command-#2: Write(0x9100 /*address*/,0x00000000 /*data*/);
2125  macroAddress = 0x9100;
2126  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2127  macroData = 0x00000000;
2128  memcpy(data, &macroData, 8); // copy macro data to buffer
2129  universalWrite(address, data);
2130 
2131  // command-#3: Write(0x9100 /*address*/,0x10000000 /*data*/);
2132  macroAddress = 0x9100;
2133  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2134  macroData = 0x10000000;
2135  memcpy(data, &macroData, 8); // copy macro data to buffer
2136  universalWrite(address, data);
2137 
2138  // command-#4: Write(0x9100 /*address*/,0x30000000 /*data*/);
2139  macroAddress = 0x9100;
2140  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2141  macroData = 0x30000000;
2142  memcpy(data, &macroData, 8); // copy macro data to buffer
2143  universalWrite(address, data);
2144 
2145  // command-#5: Write(0x9100 /*address*/,0x10000000 /*data*/);
2146  macroAddress = 0x9100;
2147  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2148  macroData = 0x10000000;
2149  memcpy(data, &macroData, 8); // copy macro data to buffer
2150  universalWrite(address, data);
2151 
2152  // command-#6: Write(0x9118 /*address*/,0x00000000 /*data*/);
2153  macroAddress = 0x9118;
2154  memcpy(address, &macroAddress, 8); // copy macro address to buffer
2155  macroData = 0x00000000;
2156  memcpy(data, &macroData, 8); // copy macro data to buffer
2157  universalWrite(address, data);
2158 
2159  delete[] address; // free the memory
2160  delete[] data; // free the memory
2161  }
2162 }
2163 
2164 //========================================================================
2165 void DTCFrontEndInterface::RunROCFEMacro(__ARGS__)
2166 {
2167  // std::string feMacroName = __GET_ARG_IN__("ROC_FEMacroName", std::string);
2168  // std::string rocUID = __GET_ARG_IN__("ROC_UID", std::string);
2169  //
2170  // __FE_COUTV__(feMacroName);
2171  // __FE_COUTV__(rocUID);
2172 
2173  auto feMacroIt = rocFEMacroMap_.find(feMacroStruct.feMacroName_);
2174  if(feMacroIt == rocFEMacroMap_.end())
2175  {
2176  __FE_SS__ << "Fatal error - ROC FE Macro name '" << feMacroStruct.feMacroName_
2177  << "' not found in DTC's map!" << __E__;
2178  __FE_SS_THROW__;
2179  }
2180 
2181  const std::string& rocUID = feMacroIt->second.first;
2182  const std::string& rocFEMacroName = feMacroIt->second.second;
2183 
2184  __FE_COUTV__(rocUID);
2185  __FE_COUTV__(rocFEMacroName);
2186 
2187  auto rocIt = rocs_.find(rocUID);
2188  if(rocIt == rocs_.end())
2189  {
2190  __FE_SS__ << "Fatal error - ROC name '" << rocUID << "' not found in DTC's map!"
2191  << __E__;
2192  __FE_SS_THROW__;
2193  }
2194 
2195  rocIt->second->runSelfFrontEndMacro(rocFEMacroName, argsIn, argsOut);
2196 
2197 } // end RunROCFEMacro()
2198 
2199 DEFINE_OTS_INTERFACE(DTCFrontEndInterface)