tdaq-develop-2025-02-12
GatewaySupervisor.cc
1 #include "otsdaq/GatewaySupervisor/GatewaySupervisor.h"
2 #include "otsdaq/CgiDataUtilities/CgiDataUtilities.h"
3 #include "otsdaq/Macros/CoutMacros.h"
4 #include "otsdaq/MessageFacility/ITRACEController.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/SOAPUtilities/SOAPCommand.h"
7 #include "otsdaq/SOAPUtilities/SOAPUtilities.h"
8 #include "otsdaq/XmlUtilities/HttpXmlDocument.h"
9 
10 #include "otsdaq/ConfigurationInterface/ConfigurationManager.h"
11 #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
12 #include "otsdaq/TablePlugins/XDAQContextTable/XDAQContextTable.h"
13 #include "otsdaq/WorkLoopManager/WorkLoopManager.h"
14 
15 #include "otsdaq/FiniteStateMachine/MakeRunInfo.h" // for Run Info plugin macro
16 #include "otsdaq/FiniteStateMachine/RunInfoVInterface.h" // for Run Info plugins
17 
18 #pragma GCC diagnostic push
19 #pragma GCC diagnostic ignored "-Wunknown-pragmas"
20 #include <cgicc/HTMLClasses.h>
21 #include <cgicc/HTMLDoctype.h>
22 #include <cgicc/HTTPCookie.h>
23 #include <cgicc/HTTPHeader.h>
24 #include <xgi/Utils.h>
25 #pragma GCC diagnostic pop
26 
27 #include <toolbox/fsm/FailedEvent.h>
28 #include <toolbox/task/WorkLoopFactory.h>
29 #include <xdaq/NamespaceURI.h>
30 #include <xoap/Method.h>
31 
32 #include <sys/stat.h> // for mkdir
33 #include <chrono> // std::chrono::seconds
34 #include <fstream>
35 #include <thread> // std::this_thread::sleep_for
36 
37 using namespace ots;
38 
39 #define RUN_NUMBER_PATH std::string(__ENV__("SERVICE_DATA_PATH")) + "/RunNumber/"
40 #define RUN_NUMBER_FILE_NAME "NextRunNumber.txt"
41 #define LOG_ENTRY_PATH std::string(__ENV__("SERVICE_DATA_PATH")) + "/FSM_LastLogEntry/"
42 #define LOG_ENTRY_FILE_NAME "LastLogEntry.txt"
43 #define FSM_LAST_GROUP_ALIAS_FILE_START std::string("FSMLastGroupAlias-")
44 #define FSM_USERS_PREFERENCES_FILETYPE "pref"
45 
46 #define REMOTE_SUBSYSTEM_SETTINGS_FILE_NAME "RemoteSubsystems.txt"
47 
48 #undef __MF_SUBJECT__
49 #define __MF_SUBJECT__ "GatewaySupervisor"
50 
51 XDAQ_INSTANTIATOR_IMPL(GatewaySupervisor)
52 
53 WebUsers GatewaySupervisor::theWebUsers_ = WebUsers();
54 std::vector<std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct>>
55  GatewaySupervisor::broadcastThreadStructs_;
56 
57 //==============================================================================
58 GatewaySupervisor::GatewaySupervisor(xdaq::ApplicationStub* s)
59  : xdaq::Application(s)
60  , SOAPMessenger(this)
61  , RunControlStateMachine("GatewaySupervisor")
62  , CorePropertySupervisorBase(this)
63  , stateMachineWorkLoopManager_(toolbox::task::bind(
64  this, &GatewaySupervisor::stateMachineThread, "StateMachine"))
65  , stateMachineSemaphore_(toolbox::BSem::FULL)
66  , activeStateMachineName_("")
67  , theIterator_(this)
68  , broadcastCommandMessageIndex_(0)
69  , broadcastIterationBreakpoint_(
70  -1) // for standard transitions, ignore the breakpoint
71 {
72  INIT_MF("." /*directory used is USER_DATA/LOG/.*/);
73 
74  __COUT__ << "Constructing" << __E__;
75 
76  if(0) // to test xdaq exception what
77  {
78  std::stringstream ss;
79  ss << "This is a test" << std::endl;
80  try
81  {
82  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
83  }
84  catch(const toolbox::fsm::exception::Exception& e)
85  {
86  // std::cout << "1Message: " << e.rbegin()->at("message") << std::endl;
87  // std::cout << "2Message: " << e.message() << std::endl;
88  // std::cout << "3Message: " << e.what() << std::endl;
89  // std::string what = e.what();
90  // std::cout << "4Message: " << what << std::endl;
91  // if(what != e.message())
92  {
93  std::cout << "Mismatch!" << std::endl;
94  throw;
95  }
96  }
97  }
98 
99  // attempt to make directory structure (just in case)
100  mkdir((std::string(__ENV__("SERVICE_DATA_PATH"))).c_str(), 0755);
101 
102  // make table group history directory here and at ConfigurationManagerRW (just in case)
103  mkdir((ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH).c_str(), 0755);
104  mkdir((RUN_NUMBER_PATH).c_str(), 0755);
105  mkdir((LOG_ENTRY_PATH).c_str(), 0755);
106 
107  securityType_ = GatewaySupervisor::theWebUsers_.getSecurity();
108 
109  __COUT__ << "Security: " << securityType_ << __E__;
110 
111  xgi::bind(this, &GatewaySupervisor::Default, "Default");
112  xgi::bind(this, &GatewaySupervisor::loginRequest, "LoginRequest");
113  xgi::bind(this, &GatewaySupervisor::request, "Request");
114  xgi::bind(this, &GatewaySupervisor::stateMachineXgiHandler, "StateMachineXgiHandler");
115  xgi::bind(this,
116  &GatewaySupervisor::stateMachineIterationBreakpoint,
117  "StateMachineIterationBreakpoint");
118  xgi::bind(this, &GatewaySupervisor::tooltipRequest, "TooltipRequest");
119  xgi::bind(this, &GatewaySupervisor::XGI_Turtle, "XGI_Turtle");
120 
121  xoap::bind(this,
122  &GatewaySupervisor::supervisorCookieCheck,
123  "SupervisorCookieCheck",
124  XDAQ_NS_URI);
125  xoap::bind(this,
126  &GatewaySupervisor::supervisorGetActiveUsers,
127  "SupervisorGetActiveUsers",
128  XDAQ_NS_URI);
129  xoap::bind(this,
130  &GatewaySupervisor::supervisorSystemMessage,
131  "SupervisorSystemMessage",
132  XDAQ_NS_URI);
133  xoap::bind(this,
134  &GatewaySupervisor::supervisorSystemLogbookEntry,
135  "SupervisorSystemLogbookEntry",
136  XDAQ_NS_URI);
137  xoap::bind(this,
138  &GatewaySupervisor::supervisorLastTableGroupRequest,
139  "SupervisorLastTableGroupRequest",
140  XDAQ_NS_URI);
141  xoap::bind(this,
142  &GatewaySupervisor::TRACESupervisorRequest,
143  "TRACESupervisorRequest",
144  XDAQ_NS_URI);
145 
146  init();
147 
148  __SUP_COUT__ << "Constructed. getpid()=" << getpid() << " gettid()=" << gettid()
149  << __E__;
150 } // end constructor
151 
152 //==============================================================================
155 GatewaySupervisor::~GatewaySupervisor(void)
156 {
157  delete CorePropertySupervisorBase::theConfigurationManager_;
158 
159  bool doLog = false;
160  try
161  {
162  doLog = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
163  }
164  catch(...)
165  { /* ignore errors */
166  ;
167  }
168 
169  if(doLog)
170  makeSystemLogEntry("ots shutdown.");
171 } // end destructor
172 
173 //==============================================================================
175 void GatewaySupervisor::indicateOtsAlive(const CorePropertySupervisorBase* properties)
176 {
177  CorePropertySupervisorBase::indicateOtsAlive(properties);
178 }
179 
180 //==============================================================================
181 void GatewaySupervisor::init(void)
182 {
183  supervisorGuiHasBeenLoaded_ = false;
184 
185  //Initialize the variable used by the RunInfo plugin
186  conditionID_ = (unsigned int)-1;
187 
188  // setting up thread for UDP thread to drive state machine
189  {
190  bool enableStateChanges = false;
191  bool enableLoginVerify = false;
192  try
193  {
194  enableStateChanges = CorePropertySupervisorBase::getSupervisorTableNode()
195  .getNode("EnableStateChangesOverUDP")
196  .getValue<bool>();
197  enableLoginVerify = CorePropertySupervisorBase::getSupervisorTableNode()
198  .getNode("EnableAckForStateChangesOverUDP")
199  .getValue<bool>();
200  }
201  catch(...)
202  {
203  ;
204  } // ignore errors
205 
206  if(enableStateChanges || enableLoginVerify)
207  {
208  if(enableStateChanges)
209  __COUT_INFO__ << "Enabling state changes over UDP..." << __E__;
210  else if(enableLoginVerify)
211  {
212  __COUT_INFO__ << "Enabling this Gateway as source of primary login "
213  "verification over UDP..."
214  << __E__;
215 
216  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
217  loadRemoteGatewaySettings(remoteGatewayApps_);
218  }
219 
220  // start state changer UDP listener thread
221  std::thread(
222  [](GatewaySupervisor* s) { GatewaySupervisor::StateChangerWorkLoop(s); },
223  this)
224  .detach();
225  }
226  else
227  __COUT__ << "State changes over UDP are disabled." << __E__;
228  } // end setting up thread for UDP drive of state machine
229 
230  // setting up checking of App Status
231  {
232  bool checkAppStatus = false;
233  try
234  {
235  checkAppStatus = CorePropertySupervisorBase::getSupervisorTableNode()
236  .getNode("EnableApplicationStatusMonitoring")
237  .getValue<bool>();
238  }
239  catch(...)
240  {
241  ;
242  } // ignore errors
243 
244  if(checkAppStatus)
245  {
246  __COUT__ << "Enabling App Status checking..." << __E__;
247 
248  std::thread(
249  [](GatewaySupervisor* s) { GatewaySupervisor::AppStatusWorkLoop(s); },
250  this)
251  .detach();
252  }
253  else
254  {
255  __COUT__ << "App Status checking is disabled." << __E__;
256 
257  // set all app status to "Not Monitored" so that FSM changes ignore missing app status
258  for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
259  {
260  auto appInfo = it.second;
261  allSupervisorInfo_.setSupervisorStatus(
262  appInfo,
263  SupervisorInfo::APP_STATUS_NOT_MONITORED,
264  0 /* progressInteger */,
265  "" /* detail */);
266  }
267  }
268 
269  } // end checking of Application Status
270 
271 } // end init()
272 
273 //==============================================================================
276 void GatewaySupervisor::AppStatusWorkLoop(GatewaySupervisor* theSupervisor)
277 {
278  sleep(5); // wait for apps to get started
279 
280  bool firstError = true;
281  std::string status, progress, detail, appName;
282  std::vector<SupervisorInfo::SubappInfo> subapps;
283  int progressInteger;
284  bool oneStatusReqHasFailed = false;
285  size_t loopCount = -1; //first time through loop will be 0
286 
287  std::map<std::string /* appName */, bool /* lastStatusGood */> appLastStatusGood;
288 
289  std::unique_ptr<TransceiverSocket>
290  remoteGatewaySocket; //use to get remote gateway status
291  bool resetRemoteGatewayApps = false;
292  bool commandingRemoteGatewayApps = false;
293  size_t commandRemoteIdleCount = 0;
294  int portForReverseLoginOverUDP = 0; //if 0, then not reverse login not enabled
295  std::string ipAddressForStateChangesOverUDP = ""; //if "", then not enabled
296 
297  while(1)
298  {
299  ++loopCount;
300  sleep(1);
301 
302  // workloop procedure
303  // Loop through all Apps and request status
304  // sleep
305 
306  oneStatusReqHasFailed = false;
307  // __COUT__ << "Just debugging App status checking" << __E__;
308  for(const auto& it : theSupervisor->allSupervisorInfo_.getAllSupervisorInfo())
309  {
310  auto appInfo = it.second;
311  appName = appInfo.getName();
312  // __COUT__ << "Getting Status "
313  // << " Supervisor instance = '" << appName
314  // << "' [LID=" << appInfo.getId() << "] in Context '"
315  // << appInfo.getContextName() << "' [URL=" <<
316  // appInfo.getURL()
317  // << "].\n\n";
318 
319  // if the application is the gateway supervisor, we do not send a SOAP message
320  if(appInfo.isGatewaySupervisor()) // get gateway status
321  {
322  try
323  {
324  // send back status and progress parameters
325  const std::string& err =
326  theSupervisor->theStateMachine_.getErrorMessage();
327  try
328  {
329  __COUTVS__(39, theSupervisor->theStateMachine_.isInTransition());
330  if(theSupervisor->theStateMachine_.isInTransition())
331  __COUTVS__(39,
332  theSupervisor->theStateMachine_
333  .getCurrentTransitionName());
334  __COUTVS__(
335  39, theSupervisor->theStateMachine_.getProvenanceStateName());
336  __COUTVS__(39,
337  theSupervisor->theStateMachine_.getCurrentStateName());
338  }
339  catch(...)
340  {
341  ;
342  }
343 
344  if(err == "")
345  {
346  if(theSupervisor->theStateMachine_
347  .isInTransition()) // || theSupervisor->theProgressBar_.read() < 100)
348  {
349  // attempt to get transition name, otherwise give provenance state
350  try
351  {
352  status = theSupervisor->theStateMachine_
353  .getCurrentTransitionName();
354  }
355  catch(...)
356  {
357  status = theSupervisor->theStateMachine_
358  .getProvenanceStateName();
359  }
360  progress =
361  theSupervisor->theProgressBar_.readPercentageString();
362  }
363  else
364  {
365  status =
366  theSupervisor->theStateMachine_.getCurrentStateName();
367  progress = "100"; //if not in transition, then 100
368  }
369  }
370  else
371  {
372  status = (theSupervisor->theStateMachine_.getCurrentStateName() ==
373  RunControlStateMachine::PAUSED_STATE_NAME
374  ? "Soft-Error:::"
375  : "Failed:::") +
376  err;
377  progress = theSupervisor->theProgressBar_.readPercentageString();
378  }
379 
380  __COUTVS__(39, status);
381  __COUTVS__(39, progress);
382 
383  try
384  {
385  detail =
386  (theSupervisor->theStateMachine_.isInTransition()
387  ? theSupervisor->theStateMachine_
388  .getCurrentTransitionName(
389  theSupervisor->stateMachineLastCommandInput_)
390  : (std::string("Uptime: ") +
391  StringMacros::encodeURIComponent(
392  StringMacros::getTimeDurationString(
393  theSupervisor->CorePropertySupervisorBase:: getSupervisorUptime())) +
394  ", Time-in-state: " +
395  StringMacros::encodeURIComponent(
396  StringMacros::getTimeDurationString(
397  theSupervisor->theStateMachine_
398  .getTimeInState()))));
399  // make sure broadcast message status is not being updated
400  std::lock_guard<std::mutex> lock(
401  theSupervisor->broadcastCommandStatusUpdateMutex_);
402  if(detail != "" && theSupervisor->broadcastCommandStatus_ != "")
403  detail += " - " + theSupervisor->broadcastCommandStatus_;
404 
405  if(!theSupervisor->theStateMachine_.isInTransition() &&
406  (theSupervisor->theStateMachine_.getCurrentStateName() ==
407  RunControlStateMachine::CONFIGURED_STATE_NAME ||
408  theSupervisor->theStateMachine_.getCurrentStateName() ==
409  RunControlStateMachine::RUNNING_STATE_NAME ||
410  theSupervisor->theStateMachine_.getCurrentStateName() ==
411  RunControlStateMachine::PAUSED_STATE_NAME))
412  {
413  //add Configuration details
414  detail +=
415  " - Configured with System Configuration Alias '" +
416  theSupervisor->activeStateMachineConfigurationAlias_ +
417  "' which translates to " +
418  theSupervisor->theConfigurationTableGroup_.first + "(" +
419  theSupervisor->theConfigurationTableGroup_.second.str() +
420  "). Active Context Group " +
421  theSupervisor
422  ->CorePropertySupervisorBase::theConfigurationManager_
423  ->getActiveGroupName(
424  ConfigurationManager::GroupType::CONTEXT_TYPE) +
425  "(" +
426  theSupervisor
427  ->CorePropertySupervisorBase::theConfigurationManager_
428  ->getActiveGroupKey(
429  ConfigurationManager::GroupType::CONTEXT_TYPE)
430  .str() +
431  ").";
432  }
433  }
434  catch(...)
435  {
436  detail = "";
437  }
438 
439  std::vector<GatewaySupervisor::RemoteGatewayInfo>
440  remoteApps; //local copy to avoid long mutex lock
441  { //lock for remainder of scope
442  std::lock_guard<std::mutex> lock(
443  theSupervisor->remoteGatewayAppsMutex_);
444  remoteApps = theSupervisor->remoteGatewayApps_;
445 
446  //check for commands
447  if(!commandingRemoteGatewayApps)
448  {
449  for(const auto& remoteApp : remoteApps)
450  {
451  if(remoteApp.command != "")
452  {
453  //latch until all remote apps are stable
454  commandingRemoteGatewayApps = true;
455  break;
456  }
457  }
458  }
459  }
460  __COUTVS__(38, commandingRemoteGatewayApps);
461 
462  //Add sub-apps for each Remote Gateway specified as a Remote Desktop Icon
463  if((!commandingRemoteGatewayApps && loopCount % 20 == 0) ||
464  loopCount ==
465  0) //periodically refresh Remote Gateway list based on icon list
466  {
467  // use latest context always from temporary configuration manager,
468  // to get updated icons every time...
469  //(so icon changes do no require an ots restart)
470  ConfigurationManager
471  tmpCfgMgr; // Creating new temporary instance so that constructor will activate latest context, note: not using member CorePropertySupervisorBase::theConfigurationManager_
472  const DesktopIconTable* iconTable =
473  tmpCfgMgr.__GET_CONFIG__(DesktopIconTable);
474  const std::vector<DesktopIconTable::DesktopIcon>& icons =
475  iconTable->getAllDesktopIcons();
476 
477  for(auto& remoteGatewayApp : remoteApps)
478  remoteGatewayApp.appInfo.status =
479  ""; //clear status, to be used to remove remote gateways no longer targeted
480 
481  resetRemoteGatewayApps = true;
482 
483  for(const auto& icon : icons)
484  {
485  __COUTTV__(icon.windowContentURL_);
486  if(icon.windowContentURL_.size() > 4 &&
487  icon.windowContentURL_[0] == 'o' &&
488  icon.windowContentURL_[1] == 't' &&
489  icon.windowContentURL_[2] == 's' &&
490  icon.windowContentURL_[3] == ':')
491  {
492  __COUT__ << "Found '" << icon.recordUID_
493  << "' remote gateway icons url: "
494  << icon.windowContentURL_ << __E__;
495 
496  GatewaySupervisor::RemoteGatewayInfo thisInfo;
497 
498  std::string remoteURL = icon.windowContentURL_;
499  std::string remoteLandingPage = "";
500  std::string remoteSetupType = "";
501  //remote ? parameters from remoteURL
502  if(remoteURL.find('?') != std::string::npos)
503  {
504  __COUT__
505  << "Extracting GET ? parameters from remote url."
506  << __E__;
507  std::vector<std::string> urlSplit =
508  StringMacros::getVectorFromString(remoteURL,
509  {'?'});
510  if(urlSplit.size() > 0)
511  remoteURL = urlSplit[0];
512  if(urlSplit.size() > 1)
513  {
514  //look for 'LandingPage' parameter
515  std::vector<std::string> parameterPairs =
516  StringMacros::getVectorFromString(urlSplit[1],
517  {'&'});
518  for(const auto& parameterPair : parameterPairs)
519  {
520  std::vector<std::string> parameterPairSplit =
521  StringMacros::getVectorFromString(
522  parameterPair, {'='});
523  if(parameterPairSplit.size() == 2)
524  {
525  __COUT__ << "Found remote URL parameter "
526  << parameterPairSplit[0] << ", "
527  << parameterPairSplit[1]
528  << __E__;
529  if(parameterPairSplit[0] == "LandingPage")
530  {
531  remoteLandingPage =
532  StringMacros::decodeURIComponent(
533  parameterPairSplit[1]);
534  if(remoteLandingPage.find(
535  icon.folderPath_) != 0)
536  remoteLandingPage =
537  icon.folderPath_ + "/" +
538  remoteLandingPage;
539  __COUT__ << "Found landing page "
540  << remoteLandingPage
541  << " for " << icon.recordUID_
542  << __E__;
543  }
544  if(parameterPairSplit[0] == "SetupType")
545  {
546  remoteSetupType =
547  StringMacros::decodeURIComponent(
548  parameterPairSplit[1]);
549 
550  __COUT__ << "Found setup_ots.sh type "
551  << remoteSetupType << " for "
552  << icon.recordUID_ << __E__;
553  }
554  }
555  }
556  }
557  } //end remote URL parameter handling
558 
559  thisInfo.appInfo.name = icon.recordUID_;
560  thisInfo.appInfo.status =
561  SupervisorInfo::APP_STATUS_UNKNOWN;
562  thisInfo.appInfo.progress = 0;
563  thisInfo.appInfo.detail = "";
564  thisInfo.appInfo.url =
565  remoteURL; //icon.windowContentURL_;
566  thisInfo.appInfo.class_name = "Remote Gateway";
567  thisInfo.appInfo.lastStatusTime = time(0);
568 
569  thisInfo.user_data_path_record = icon.alternateText_;
570  thisInfo.parentIconFolderPath = icon.folderPath_;
571  thisInfo.permissionThresholdString =
572  icon.permissionThresholdString_;
573  thisInfo.landingPage = remoteLandingPage;
574  thisInfo.setupType = remoteSetupType;
575 
576  try
577  {
578  tmpCfgMgr.getOtherSubsystemInstanceInfo(
579  thisInfo.user_data_path_record,
580  &thisInfo.instancePath,
581  &thisInfo.instanceHost,
582  &thisInfo.instanceUser,
583  &thisInfo.fullName);
584  }
585  catch(...)
586  {
587  ;
588  } //ignore any errors getting full name and instance info
589 
590  //replace or add to local copy of supervisor remote gateway list (control info will be protected later, after status update, with final copy to real list)
591  bool found = false;
592  size_t i = 0;
593  for(; i < remoteApps.size(); ++i)
594  {
595  if(thisInfo.appInfo.name ==
596  remoteApps[i].appInfo.name)
597  {
598  found = true;
599 
600  //overwrite with refreshed info
601  remoteApps[i].appInfo = thisInfo.appInfo;
602  remoteApps[i].user_data_path_record =
603  thisInfo.user_data_path_record;
604  remoteApps[i].parentIconFolderPath =
605  thisInfo.parentIconFolderPath;
606  remoteApps[i].permissionThresholdString =
607  thisInfo.permissionThresholdString;
608  remoteApps[i].landingPage = thisInfo.landingPage;
609  remoteApps[i].setupType = thisInfo.setupType;
610  remoteApps[i].fullName = thisInfo.fullName;
611  remoteApps[i].instancePath =
612  thisInfo.instancePath;
613  remoteApps[i].instanceHost =
614  thisInfo.instanceHost;
615  remoteApps[i].instanceUser =
616  thisInfo.instanceUser;
617 
618  break;
619  }
620  }
621 
622  if(!found) //add
623  remoteApps.push_back(thisInfo);
624 
625  //if possible, get CSV list of potential config_aliases (for dropdown)
626  try
627  {
628  __COUTTV__(remoteApps[i].fsmName);
629  remoteApps[i].config_aliases =
630  tmpCfgMgr.getOtherSubsystemConfigAliases(
631  remoteApps[i]
632  .user_data_path_record); //getOtherSubsystemFilteredConfigAliases(remoteApps[i].user_data_path_record, remoteApps[i].fsmName);
633  __COUTV__(StringMacros::setToString(
634  thisInfo.config_aliases));
635  if(remoteApps[i]
636  .config_aliases
637  .size()) //initialize selected alias in case it is needed (selected alias will be verified at final copy)
638  remoteApps[i].selected_config_alias =
639  *(remoteApps[i].config_aliases.begin());
640  }
641  catch(const std::runtime_error& e)
642  {
643  __SS__
644  << "Failed to retrieve the list of Configuration "
645  "Aliases for Remote Subsystem '"
646  << remoteApps[i].appInfo.name
647  << ".' Remote Subsystems are specified through "
648  "their Desktop Icon record. "
649  "Please specify a valid User Data Path record "
650  "as the Desktop Icon AlternateText field, "
651  "targeting a UID in the "
652  "SubsystemUserDataPathsTable."
653  << __E__;
654  ss << "\n\nHere was the error: " << e.what() << __E__;
655  __COUT__ << ss.str();
656  remoteApps[i].error = ss.str();
657 
658  //give feedback immediately to user!!
659  {
660  __COUTV__(remoteApps[i].error);
661  //lock for remainder of scope
662  std::lock_guard<std::mutex> lock(
663  theSupervisor->remoteGatewayAppsMutex_);
664  for(size_t i = 0;
665  i < theSupervisor->remoteGatewayApps_.size();
666  ++i)
667  if(remoteApps[i].appInfo.name ==
668  theSupervisor->remoteGatewayApps_[i]
669  .appInfo.name)
670  {
671  theSupervisor->remoteGatewayApps_[i]
672  .error = remoteApps[i].error;
673  break;
674  }
675  }
676  }
677 
678  } //end remote icon handling
679 
680  } //end icon loop
681 
682  //clean up stale remoteGatewayApps with blank status
683  bool remoteAppsExist = false;
684  for(size_t i = 0; i < remoteApps.size(); ++i)
685  {
686  if(remoteApps[i].appInfo.status == "")
687  {
688  //rewind and erase
689  remoteApps.erase(remoteApps.begin() + i);
690  --i;
691  }
692  }
693  remoteAppsExist = remoteApps.size();
694 
695  if(remoteAppsExist &&
696  !remoteGatewaySocket) //instantiate socket first time there are remote apps
697  {
698  __COUT_INFO__
699  << "Instantiating Remote Gateway App Status Socket!"
700  << __E__;
701  ConfigurationTree configLinkNode =
702  theSupervisor->CorePropertySupervisorBase:: getSupervisorTableNode();
703  ipAddressForStateChangesOverUDP =
704  configLinkNode.getNode("IPAddressForStateChangesOverUDP")
705  .getValue<std::string>();
706  __COUTTV__(ipAddressForStateChangesOverUDP);
707 
708  //check if allowing reverse login verification from remote Gateways to this Gateway
709  if(theSupervisor->theWebUsers_.getSecurity() ==
710  WebUsers::SECURITY_TYPE_DIGEST_ACCESS)
711  {
712  bool enableLoginVerify =
713  configLinkNode
714  .getNode("EnableAckForStateChangesOverUDP")
715  .getValue<bool>();
716 
717  if(enableLoginVerify)
718  {
719  portForReverseLoginOverUDP =
720  configLinkNode
721  .getNode("PortForStateChangesOverUDP")
722  .getValue<int>();
723  if(portForReverseLoginOverUDP)
724  __COUT_INFO__
725  << "Enabling reverse login verification for "
726  "Remote Gateways at "
727  << ipAddressForStateChangesOverUDP << ":"
728  << portForReverseLoginOverUDP << __E__;
729  }
730  else
731  {
732  __COUT_WARN__
733  << "Remote login verification at this Gateway, "
734  "for other Remote Gateways, is not enabled "
735  "unless the GatewaySupervisor has "
736  "'EnableStateChangesOverUDP' enabled."
737  << __E__;
738  }
739  }
740  remoteGatewaySocket = std::make_unique<TransceiverSocket>(
741  ipAddressForStateChangesOverUDP);
742  remoteGatewaySocket->initialize();
743  }
744 
745  } //end periodic Remote Gateway refresh
746 
747  //for each remote gateway, request app status with "GetRemoteAppStatus"
748  if(loopCount % 3 == 0 ||
749  resetRemoteGatewayApps || //a little less frequently
750  commandingRemoteGatewayApps)
751  {
752  if(theSupervisor->remoteGatewayApps_.size())
753  __COUTVS__(38, theSupervisor->remoteGatewayApps_[0].error);
754 
755  //check for commands first
756  bool commandSent = false;
757 
758  for(auto& remoteGatewayApp : remoteApps)
759  if(remoteGatewayApp.command != "")
760  {
761  GatewaySupervisor::SendRemoteGatewayCommand(
762  remoteGatewayApp, remoteGatewaySocket);
763  if(remoteGatewayApp.error == "")
764  {
765  remoteGatewayApp.ignoreStatusCount =
766  0; //if non-zero, do not ask for status
767  commandSent = true;
768  }
769 
770  //give feedback immediately to user!!
771  {
772  __COUT__ << "remoteGatewayApp "
773  << remoteGatewayApp.appInfo.name
774  << " error: " << remoteGatewayApp.error
775  << __E__;
776  //lock for remainder of scope
777  std::lock_guard<std::mutex> lock(
778  theSupervisor->remoteGatewayAppsMutex_);
779  for(size_t i = 0;
780  i < theSupervisor->remoteGatewayApps_.size();
781  ++i)
782  if(remoteGatewayApp.appInfo.name ==
783  theSupervisor->remoteGatewayApps_[i]
784  .appInfo.name)
785  {
786  theSupervisor->remoteGatewayApps_[i].error =
787  remoteGatewayApp.error;
788  break;
789  }
790  }
791  }
792 
793  if(commandSent)
794  {
795  commandRemoteIdleCount = 0; //reset
796  sleep(1); //gives some time for command to sink in
797  }
798 
799  if(theSupervisor->remoteGatewayApps_.size())
800  __COUTVS__(38, theSupervisor->remoteGatewayApps_[0].error);
801 
802  //then get status
803  bool allAppsAreIdle = true;
804  bool allApssAreUnknown = true;
805  for(auto& remoteGatewayApp : remoteApps)
806  {
807  GatewaySupervisor::CheckRemoteGatewayStatus(
808  remoteGatewayApp,
809  remoteGatewaySocket,
810  ipAddressForStateChangesOverUDP,
811  portForReverseLoginOverUDP);
812  if(remoteGatewayApp.appInfo.status !=
813  SupervisorInfo::APP_STATUS_UNKNOWN)
814  {
815  allApssAreUnknown = false;
816  if(!appLastStatusGood[remoteGatewayApp.appInfo.url +
817  remoteGatewayApp.appInfo.name])
818  {
819  __COUT_INFO__
820  << "First good status from "
821  << " Remote subapp = '"
822  << remoteGatewayApp.appInfo.name
823  << "' [URL=" << remoteGatewayApp.appInfo.url
824  << "].\n\n";
825  }
826  appLastStatusGood[remoteGatewayApp.appInfo.url +
827  remoteGatewayApp.appInfo.name] = true;
828 
829  if(!(remoteGatewayApp.appInfo.progress ==
830  0 || //if !(idle)
831  remoteGatewayApp.appInfo.progress == 100 ||
832  remoteGatewayApp.appInfo.status.find("Error") !=
833  std::string::
834  npos || // case "Failed", "Error", "Soft-Error"
835  remoteGatewayApp.appInfo.status.find("Fail") !=
836  std::string::npos))
837  {
838  __COUT__
839  << remoteGatewayApp.appInfo.name << " not idle: "
840  << remoteGatewayApp.appInfo.status
841  << " progress: "
842  << remoteGatewayApp.appInfo.progress << __E__;
843  allAppsAreIdle = false;
844  }
845  }
846  else //skip absent subsystems for a while
847  remoteGatewayApp.ignoreStatusCount =
848  3; //if non-zero, do not ask for status
849  } //end remote app status update loop
850 
851  if(allApssAreUnknown) //then remove ignore status, and give user feedback faster
852  {
853  for(auto& remoteGatewayApp : remoteApps)
854  remoteGatewayApp.ignoreStatusCount =
855  0; //if non-zero, do not ask for status
856  }
857 
858  if(commandingRemoteGatewayApps && allAppsAreIdle)
859  {
860  ++commandRemoteIdleCount;
861  if(commandRemoteIdleCount >= 3)
862  {
863  __COUTT__ << "Back to idle statusing" << __E__;
864  commandingRemoteGatewayApps = false;
865  }
866  }
867 
868  __COUT_TYPE__(TLVL_DEBUG + 38)
869  << __COUT_HDR__ << "commandRemoteIdleCount "
870  << commandRemoteIdleCount << " " << allAppsAreIdle << " "
871  << commandingRemoteGatewayApps << __E__;
872 
873  } //end remote app status update
874 
875  //if possible, get remote icon list for desktop from each remote app
876  if(resetRemoteGatewayApps)
877  {
878  __COUT_TYPE__(TLVL_DEBUG + 35)
879  << __COUT_HDR__
880  << "Attempting to get Remote Desktop Icons... size="
881  << remoteApps.size() << __E__;
882 
883  for(auto& remoteGatewayApp : remoteApps)
884  {
885  __COUTVS__(35, remoteGatewayApp.appInfo.name);
886  __COUTVS__(35, remoteGatewayApp.command);
887  if(remoteGatewayApp.command != "")
888  continue; //skip if command to be sent
889 
890  __COUT_TYPE__(TLVL_DEBUG + 14)
891  << __COUT_HDR__ << remoteGatewayApp.appInfo.name << ": "
892  << remoteGatewayApp.appInfo.status << __E__;
893  if(remoteGatewayApp.appInfo.status ==
894  SupervisorInfo::APP_STATUS_UNKNOWN)
895  continue; //skip if no status yet
896 
897  //clear any previous icon error
898  if(remoteGatewayApp.error.find("desktop icons") !=
899  std::string::npos)
900  {
901  __COUTV__(remoteGatewayApp.error);
902  //lock for remainder of scope
903  std::lock_guard<std::mutex> lock(
904  theSupervisor->remoteGatewayAppsMutex_);
905  for(size_t i = 0;
906  i < theSupervisor->remoteGatewayApps_.size();
907  ++i)
908  if(remoteGatewayApp.appInfo.name ==
909  theSupervisor->remoteGatewayApps_[i].appInfo.name)
910  {
911  theSupervisor->remoteGatewayApps_[i].error = "";
912  __COUTV__(
913  theSupervisor->remoteGatewayApps_[i].error);
914  break;
915  }
916  remoteGatewayApp.error = "";
917  }
918 
919  if(remoteGatewayApp.error ==
920  "") //only request icons if no errors
921  {
922  GatewaySupervisor::GetRemoteGatewayIcons(
923  remoteGatewayApp, remoteGatewaySocket);
924  if(remoteGatewayApp.error !=
925  "") //give feedback immediately to user!!
926  {
927  __COUTV__(remoteGatewayApp.error);
928  //lock for remainder of scope
929  std::lock_guard<std::mutex> lock(
930  theSupervisor->remoteGatewayAppsMutex_);
931  for(size_t i = 0;
932  i < theSupervisor->remoteGatewayApps_.size();
933  ++i)
934  if(remoteGatewayApp.appInfo.name ==
935  theSupervisor->remoteGatewayApps_[i]
936  .appInfo.name)
937  {
938  theSupervisor->remoteGatewayApps_[i].error =
939  remoteGatewayApp.error;
940  break;
941  }
942  }
943  }
944  }
945 
946  } //end remote desktop icon gathering
947 
948  //for each remote gateway, copy info to Gateway supervisor remote gateway structure
949  if(loopCount % 3 == 0 ||
950  resetRemoteGatewayApps || //a little less frequently
951  commandingRemoteGatewayApps)
952  {
953  if(theSupervisor->remoteGatewayApps_.size())
954  __COUTVS__(37, theSupervisor->remoteGatewayApps_[0].error);
955 
956  //replace info in supervisor remote gateway list
957  {
958  //lock for remainder of scope
959  std::lock_guard<std::mutex> lock(
960  theSupervisor->remoteGatewayAppsMutex_);
961  for(size_t i = 0;
962  !commandingRemoteGatewayApps &&
963  i < theSupervisor->remoteGatewayApps_.size();
964  ++i)
965  {
966  __COUTVS__(50,
967  theSupervisor->remoteGatewayApps_[i].command);
968  if(theSupervisor->remoteGatewayApps_[i].command ==
969  "") //make sure not mid-command
970  theSupervisor->remoteGatewayApps_[i].appInfo.status =
971  ""; //clear status as indicator to be erased
972  }
973 
974  for(auto& remoteGatewayApp : remoteApps)
975  {
976  bool found = false;
977  for(size_t i = 0;
978  i < theSupervisor->remoteGatewayApps_.size();
979  ++i)
980  {
981  if(remoteGatewayApp.appInfo.name ==
982  theSupervisor->remoteGatewayApps_[i].appInfo.name)
983  {
984  found = true;
985  //copy over updated status (but not control info, which may be have been changed while mutex was dropped)
986 
987  if(remoteGatewayApp.command ==
988  "") //if there is action on command, then error is being set (request()) or cleared (send) somewhere else
989  theSupervisor->remoteGatewayApps_[i].error =
990  remoteGatewayApp.error;
991 
992  theSupervisor->remoteGatewayApps_[i]
993  .ignoreStatusCount =
994  remoteGatewayApp.ignoreStatusCount;
995  theSupervisor->remoteGatewayApps_[i]
996  .consoleErrCount =
997  remoteGatewayApp.consoleErrCount;
998  theSupervisor->remoteGatewayApps_[i]
999  .consoleWarnCount =
1000  remoteGatewayApp.consoleWarnCount;
1001 
1002  theSupervisor->remoteGatewayApps_[i]
1003  .usernameWithLock =
1004  remoteGatewayApp.usernameWithLock;
1005 
1006  theSupervisor->remoteGatewayApps_[i].config_dump =
1007  remoteGatewayApp.config_dump;
1008 
1009  theSupervisor->remoteGatewayApps_[i]
1010  .user_data_path_record =
1011  remoteGatewayApp.user_data_path_record;
1012  theSupervisor->remoteGatewayApps_[i].iconString =
1013  remoteGatewayApp.iconString;
1014  theSupervisor->remoteGatewayApps_[i]
1015  .parentIconFolderPath =
1016  remoteGatewayApp.parentIconFolderPath;
1017  theSupervisor->remoteGatewayApps_[i]
1018  .permissionThresholdString =
1019  remoteGatewayApp.permissionThresholdString;
1020  theSupervisor->remoteGatewayApps_[i].landingPage =
1021  remoteGatewayApp.landingPage;
1022  theSupervisor->remoteGatewayApps_[i].setupType =
1023  remoteGatewayApp.setupType;
1024  theSupervisor->remoteGatewayApps_[i].fullName =
1025  remoteGatewayApp.fullName;
1026  theSupervisor->remoteGatewayApps_[i]
1027  .instancePath = remoteGatewayApp.instancePath;
1028  theSupervisor->remoteGatewayApps_[i]
1029  .instanceHost = remoteGatewayApp.instanceHost;
1030  theSupervisor->remoteGatewayApps_[i]
1031  .instanceUser = remoteGatewayApp.instanceUser;
1032 
1033  //fix config_aliases and selected_config_alias
1034  theSupervisor->remoteGatewayApps_[i]
1035  .config_aliases =
1036  remoteGatewayApp.config_aliases;
1037  //if invalid selected_config_alias, renitialize for user
1038  if(remoteGatewayApp.config_aliases.find(
1039  theSupervisor->remoteGatewayApps_[i]
1040  .selected_config_alias) ==
1041  remoteGatewayApp.config_aliases.end())
1042  theSupervisor->remoteGatewayApps_[i]
1043  .selected_config_alias =
1044  remoteGatewayApp.selected_config_alias;
1045 
1046  __COUTTV__(theSupervisor->remoteGatewayApps_[i]
1047  .selected_config_alias);
1048 
1049  __COUTT__
1050  << remoteGatewayApp.appInfo.name
1051  << " -- Command: " << remoteGatewayApp.command
1052  << " Command-old: "
1053  << theSupervisor->remoteGatewayApps_[i]
1054  .command
1055  << " Status: "
1056  << remoteGatewayApp.appInfo.status
1057  << " Status_old: "
1058  << theSupervisor->remoteGatewayApps_[i]
1059  .appInfo.status
1060  << " Error: " << remoteGatewayApp.error
1061  << " progress: "
1062  << remoteGatewayApp.appInfo.progress << __E__;
1063 
1064  if(remoteGatewayApp.command ==
1065  "Sent") //apply command clear
1066  theSupervisor->remoteGatewayApps_[i].command =
1067  "";
1068 
1069  if(theSupervisor->remoteGatewayApps_[i].command !=
1070  "" ||
1071  (commandingRemoteGatewayApps &&
1072  theSupervisor->remoteGatewayApps_[i]
1073  .appInfo.status.find("Launching") ==
1074  0 &&
1075  remoteGatewayApp.appInfo.progress ==
1076  100)) //dont trust done progress while still 'commanding'
1077  __COUT__ << "Ignoring '"
1078  << remoteGatewayApp.appInfo.name
1079  << "' assumed stale status: "
1080  << remoteGatewayApp.appInfo.status
1081  << __E__;
1082  else
1083  theSupervisor->remoteGatewayApps_[i].appInfo =
1084  remoteGatewayApp.appInfo;
1085 
1086  theSupervisor->remoteGatewayApps_[i].subapps =
1087  remoteGatewayApp.subapps;
1088  break;
1089  }
1090  }
1091  if(!found) //add
1092  theSupervisor->remoteGatewayApps_.push_back(
1093  remoteGatewayApp);
1094  }
1095 
1096  //cleanup unused remoteGatewayApps_
1097  for(size_t i = 0;
1098  i < theSupervisor->remoteGatewayApps_.size();
1099  ++i)
1100  {
1101  if(theSupervisor->remoteGatewayApps_[i].appInfo.status ==
1102  "")
1103  {
1104  //rewind and erase
1105  theSupervisor->remoteGatewayApps_.erase(
1106  theSupervisor->remoteGatewayApps_.begin() + i);
1107  --i;
1108  }
1109  }
1110  }
1111 
1112  if(theSupervisor->remoteGatewayApps_.size())
1113  __COUTVS__(38, theSupervisor->remoteGatewayApps_[0].error);
1114  }
1115 
1116  //copy to subapps for display of primary Gateway
1117  {
1118  std::lock_guard<std::mutex> lock(
1119  theSupervisor->remoteGatewayAppsMutex_);
1120  for(const auto& remoteGatewayApp :
1121  theSupervisor->remoteGatewayApps_)
1122  subapps.push_back(remoteGatewayApp.appInfo);
1123  }
1124 
1125  resetRemoteGatewayApps = false; //reset
1126  }
1127  catch(const std::runtime_error& e)
1128  {
1129  status = SupervisorInfo::APP_STATUS_UNKNOWN;
1130  progress = "0";
1131  detail = "SOAP Message Error";
1132  oneStatusReqHasFailed = true;
1133 
1134  __COUT__ << "Failed getting status from Gateway"
1135  << " Supervisor instance = '" << appName
1136  << "' [LID=" << appInfo.getId() << "] in Context '"
1137  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
1138  << "].\n\n";
1139  __COUT_WARN__ << "Failed to retrieve Gateway Supervisor status. Here "
1140  "is the error: "
1141  << e.what() << __E__;
1142  }
1143  catch(...)
1144  {
1145  status = SupervisorInfo::APP_STATUS_UNKNOWN;
1146  progress = "0";
1147  detail = "Unknown SOAP Message Error";
1148  oneStatusReqHasFailed = true;
1149 
1150  __COUT__ << "Failed getting status from Gateway "
1151  << " Supervisor instance = '" << appName
1152  << "' [LID=" << appInfo.getId() << "] in Context '"
1153  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
1154  << "].\n\n";
1155  __COUT_WARN__ << "Failed to retrieve Gateway Supervisor status to "
1156  "unknown error."
1157  << __E__;
1158  }
1159  }
1160  else // get non-gateway status
1161  {
1162  // pass the application as a parameter to tempMessage
1163  SOAPParameters appPointer;
1164  appPointer.addParameter("ApplicationPointer");
1165 
1166  xoap::MessageReference tempMessage =
1167  SOAPUtilities::makeSOAPMessageReference("ApplicationStatusRequest");
1168 
1169  __COUT_TYPE__(TLVL_DEBUG + 39)
1170  << __COUT_HDR__ << "tempMessage... "
1171  << SOAPUtilities::translate(tempMessage) << std::endl;
1172 
1173  try
1174  {
1175  xoap::MessageReference statusMessage =
1176  theSupervisor->sendWithSOAPReply(appInfo.getDescriptor(),
1177  tempMessage);
1178 
1179  if("ContextARTDAQ" == appInfo.getContextName())
1180  __COUT_TYPE__(TLVL_DEBUG + 41)
1181  << __COUT_HDR__ << " Supervisor instance = '" << appName
1182  << "' [LID=" << appInfo.getId() << "] in Context '"
1183  << appInfo.getContextName() << " statusMessage... "
1184  << SOAPUtilities::translate(statusMessage) << std::endl;
1185  else
1186  __COUT_TYPE__(TLVL_DEBUG + 40)
1187  << __COUT_HDR__ << " Supervisor instance = '" << appName
1188  << "' [LID=" << appInfo.getId() << "] in Context '"
1189  << appInfo.getContextName() << " statusMessage... "
1190  << SOAPUtilities::translate(statusMessage) << std::endl;
1191 
1192  SOAPParameters parameters;
1193  parameters.addParameter("Status");
1194  parameters.addParameter("Progress");
1195  parameters.addParameter("Detail");
1196  parameters.addParameter("Subapps");
1197  SOAPUtilities::receive(statusMessage, parameters);
1198 
1199  status = parameters.getValue("Status");
1200  if(status.empty())
1201  status = SupervisorInfo::APP_STATUS_UNKNOWN;
1202 
1203  progress = parameters.getValue("Progress");
1204  if(progress.empty())
1205  progress = "100";
1206 
1207  detail = parameters.getValue("Detail");
1208  if(appInfo.isTypeConsoleSupervisor())
1209  {
1210  //parse detail
1211 
1212  //Note: do not printout detail, because custom counts will fire recursively
1213  // std::cout << __COUT_HDR__ << (detail);
1214 
1215  //Console Supervisor status detatil format is (from otsdaq-utilities/otsdaq-utilities/Console/ConsoleSupervisor.cc:1722):
1216  // uptime, Err count, Warn count, Last Error msg, Last Warn msg
1217  std::vector<std::string> parseDetail =
1218  StringMacros::getVectorFromString(detail, {','});
1219 
1220  __COUTVS__(50, detail);
1221  __COUTVS__(50, parseDetail.size());
1222  __COUTVS__(50, StringMacros::vectorToString(parseDetail));
1223 
1224  std::lock_guard<std::mutex> lock(
1225  theSupervisor->systemStatusMutex_); //lock for rest of scope
1226  if(parseDetail.size() > 1)
1227  theSupervisor->systemConsoleErrCount_ =
1228  atoi(parseDetail[1]
1229  .substr(parseDetail[1].find(':') + 1)
1230  .c_str());
1231  if(parseDetail.size() > 2)
1232  theSupervisor->systemConsoleWarnCount_ =
1233  atoi(parseDetail[2]
1234  .substr(parseDetail[2].find(':') + 1)
1235  .c_str());
1236  __COUTVS__(36, theSupervisor->systemConsoleErrCount_);
1237  __COUTVS__(36, theSupervisor->systemConsoleWarnCount_);
1238  if(parseDetail.size() >
1239  3) //e.g. Last Err (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1240  {
1241  size_t closeTimePos = parseDetail[3].find(')');
1242  __COUTVS__(36, closeTimePos);
1243  theSupervisor->lastConsoleErr_ =
1244  parseDetail[3].substr(closeTimePos + 2);
1245  size_t openTimePos = parseDetail[3].find('(');
1246  __COUTVS__(36, openTimePos);
1247  theSupervisor->lastConsoleErrTime_ = parseDetail[3].substr(
1248  openTimePos, closeTimePos - openTimePos + 1);
1249  __COUTVS__(36, theSupervisor->lastConsoleErrTime_);
1250  }
1251  if(parseDetail.size() >
1252  4) //e.g. Last Warn (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1253  {
1254  size_t closeTimePos = parseDetail[4].find(')');
1255  theSupervisor->lastConsoleWarn_ =
1256  parseDetail[4].substr(closeTimePos + 2);
1257  size_t openTimePos = parseDetail[4].find('(');
1258  theSupervisor->lastConsoleWarnTime_ = parseDetail[4].substr(
1259  openTimePos, closeTimePos - openTimePos + 1);
1260  __COUTVS__(36, theSupervisor->lastConsoleWarnTime_);
1261  }
1262  if(parseDetail.size() >
1263  5) //e.g. Last Info (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1264  {
1265  size_t closeTimePos = parseDetail[5].find(')');
1266  theSupervisor->lastConsoleInfo_ =
1267  parseDetail[5].substr(closeTimePos + 2);
1268  size_t openTimePos = parseDetail[5].find('(');
1269  theSupervisor->lastConsoleInfoTime_ = parseDetail[5].substr(
1270  openTimePos, closeTimePos - openTimePos + 1);
1271  __COUTVS__(36, theSupervisor->lastConsoleInfoTime_);
1272  }
1273  if(parseDetail.size() > 6)
1274  theSupervisor->systemConsoleInfoCount_ =
1275  atoi(parseDetail[6]
1276  .substr(parseDetail[6].find(':') + 1)
1277  .c_str());
1278 
1279  if(parseDetail.size() >
1280  7) //e.g. First Err (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1281  {
1282  size_t closeTimePos = parseDetail[7].find(')');
1283  theSupervisor->firstConsoleErr_ =
1284  parseDetail[7].substr(closeTimePos + 2);
1285  size_t openTimePos = parseDetail[7].find('(');
1286  theSupervisor->firstConsoleErrTime_ = parseDetail[7].substr(
1287  openTimePos, closeTimePos - openTimePos + 1);
1288  __COUTVS__(36, theSupervisor->firstConsoleErrTime_);
1289  }
1290  if(parseDetail.size() >
1291  8) //e.g. First Warn (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1292  {
1293  size_t closeTimePos = parseDetail[8].find(')');
1294  theSupervisor->firstConsoleWarn_ =
1295  parseDetail[8].substr(closeTimePos + 2);
1296  size_t openTimePos = parseDetail[8].find('(');
1297  theSupervisor->firstConsoleWarnTime_ = parseDetail[8].substr(
1298  openTimePos, closeTimePos - openTimePos + 1);
1299  __COUTVS__(36, theSupervisor->firstConsoleWarnTime_);
1300  }
1301  if(parseDetail.size() >
1302  9) //e.g. First Info (Mon Sep 30 14:38:20 2024 CDT): Remote%20lo
1303  {
1304  size_t closeTimePos = parseDetail[9].find(')');
1305  theSupervisor->firstConsoleInfo_ =
1306  parseDetail[9].substr(closeTimePos + 2);
1307  size_t openTimePos = parseDetail[9].find('(');
1308  theSupervisor->firstConsoleInfoTime_ = parseDetail[9].substr(
1309  openTimePos, closeTimePos - openTimePos + 1);
1310  __COUTVS__(36, theSupervisor->firstConsoleInfoTime_);
1311  }
1312  }
1313 
1314  subapps = SupervisorInfo::deserializeSubappInfos(
1315  parameters.getValue("Subapps"));
1316 
1317  if(!appLastStatusGood[appName])
1318  {
1319  __COUT_INFO__ << "First good status from "
1320  << " Supervisor instance = '" << appName
1321  << "' [LID=" << appInfo.getId() << "] in Context '"
1322  << appInfo.getContextName()
1323  << "' [URL=" << appInfo.getURL() << "].\n\n";
1324  __COUTTV__(SOAPUtilities::translate(tempMessage));
1325  }
1326  appLastStatusGood[appName] = true;
1327  }
1328  catch(const xdaq::exception::Exception& e)
1329  {
1330  status = SupervisorInfo::APP_STATUS_UNKNOWN;
1331  progress = "0";
1332  detail = "SOAP Message Error";
1333  oneStatusReqHasFailed = true;
1334  if(firstError) // first error, give some more time for apps to boot
1335  {
1336  firstError = false;
1337  break;
1338  }
1339  if(appLastStatusGood[appName])
1340  {
1341  __COUT__ << "Failed getting status from "
1342  << " Supervisor instance = '" << appName
1343  << "' [LID=" << appInfo.getId() << "] in Context '"
1344  << appInfo.getContextName()
1345  << "' [URL=" << appInfo.getURL() << "].\n\n";
1346  __COUTTV__(SOAPUtilities::translate(tempMessage));
1347  __COUT_WARN__ << "Failed to send getStatus SOAP Message - will "
1348  "suppress repeat errors: "
1349  << e.what() << __E__;
1350  } // else quiet repeat error messages
1351  else //check if should throw state machine error
1352  {
1353  std::lock_guard<std::mutex> lock(
1354  theSupervisor->stateMachineAccessMutex_);
1355 
1356  std::string currentState =
1357  theSupervisor->theStateMachine_.getCurrentStateName();
1358  if(currentState != RunControlStateMachine::FAILED_STATE_NAME &&
1359  currentState != RunControlStateMachine::HALTED_STATE_NAME &&
1360  currentState != RunControlStateMachine::INITIAL_STATE_NAME)
1361  {
1362  __SS__
1363  << "\nDid a supervisor crash? Failed getting status from "
1364  << " Supervisor instance = '" << appName
1365  << "' [LID=" << appInfo.getId() << "] in Context '"
1366  << appInfo.getContextName()
1367  << "' [URL=" << appInfo.getURL() << "]." << __E__;
1368  __COUT_ERR__ << "\n" << ss.str();
1369 
1370  //Prevent stopping state machine from failed status (for now)
1371  // Seeing Console Crash and bring down state machine at the moment
1372  if(!appInfo.isTypeConsoleSupervisor())
1373  {
1374  theSupervisor->theStateMachine_.setErrorMessage(ss.str());
1375  try
1376  {
1377  theSupervisor->runControlMessageHandler(
1378  SOAPUtilities::makeSOAPMessageReference(
1379  RunControlStateMachine::
1380  ERROR_TRANSITION_NAME));
1381  }
1382  catch(...)
1383  {
1384  } //ignore any errors
1385  }
1386  else
1387  __COUT__
1388  << "Ignoring that Console type supervisor crashed."
1389  << __E__;
1390 
1391  break; //only send one Error, then restart status loop
1392  }
1393  }
1394  appLastStatusGood[appName] = false;
1395  }
1396  catch(...)
1397  {
1398  try
1399  {
1400  throw;
1401  } //one more try to printout extra info
1402  catch(const std::runtime_error& e)
1403  {
1404  __COUT_ERR__ << "Exception of type runtime_error message: "
1405  << e.what();
1406  }
1407  catch(const std::exception& e)
1408  {
1409  __COUT_ERR__ << "Exception of type " << typeid(e).name()
1410  << " message: " << e.what();
1411  }
1412  catch(...)
1413  {
1414  }
1415 
1416  status = SupervisorInfo::APP_STATUS_UNKNOWN;
1417  progress = "0";
1418  detail = "Unknown SOAP Message Error";
1419  oneStatusReqHasFailed = true;
1420  if(firstError) // first error, give some more time for apps to boot
1421  {
1422  firstError = false;
1423  break;
1424  }
1425  if(appLastStatusGood[appName])
1426  {
1427  __COUT__ << "Failed getting status from "
1428  << " Supervisor instance = '" << appName
1429  << "' [LID=" << appInfo.getId() << "] in Context '"
1430  << appInfo.getContextName()
1431  << "' [URL=" << appInfo.getURL() << "].\n\n";
1432  __COUTV__(SOAPUtilities::translate(tempMessage));
1433  __COUT_WARN__ << "Failed to send getStatus SOAP Message due to "
1434  "unknown error. Will suppress repeat errors "
1435  "from Supervisor instance = '"
1436  << appName << "' [LID=" << appInfo.getId()
1437  << "] in Context '" << appInfo.getContextName()
1438  << "' [URL=" << appInfo.getURL() << "]." << __E__;
1439  } // else quiet repeat error messages
1440  else //check if should throw state machine error
1441  {
1442  std::lock_guard<std::mutex> lock(
1443  theSupervisor->stateMachineAccessMutex_);
1444 
1445  std::string currentState =
1446  theSupervisor->theStateMachine_.getCurrentStateName();
1447  if(currentState != RunControlStateMachine::FAILED_STATE_NAME &&
1448  currentState != RunControlStateMachine::HALTED_STATE_NAME &&
1449  currentState != RunControlStateMachine::INITIAL_STATE_NAME)
1450  {
1451  __SS__ << "\nDid a supervisor crash? Failed getting Status "
1452  << " Supervisor instance = '" << appName
1453  << "' [LID=" << appInfo.getId() << "] in Context '"
1454  << appInfo.getContextName()
1455  << "' [URL=" << appInfo.getURL() << "]." << __E__;
1456  __COUT_ERR__ << "\n" << ss.str();
1457 
1458  theSupervisor->theStateMachine_.setErrorMessage(ss.str());
1459  try
1460  {
1461  theSupervisor->runControlMessageHandler(
1462  SOAPUtilities::makeSOAPMessageReference(
1463  RunControlStateMachine::ERROR_TRANSITION_NAME));
1464  }
1465  catch(...)
1466  {
1467  } //ignore any errors
1468 
1469  break; //only send one Error, then restart status loop
1470  }
1471  }
1472  appLastStatusGood[appName] = false;
1473  }
1474  } // end with non-gateway status request handling
1475 
1476  __COUTVS__(38, status);
1477  __COUTVS__(38, progress);
1478 
1479  // set status and progress
1480  // convert the progress string into an integer in order to call
1481  // appInfo.setProgress() function
1482  std::istringstream ssProgress(progress);
1483  ssProgress >> progressInteger;
1484 
1485  if("ContextARTDAQ" == appInfo.getContextName())
1486  __COUTVS__(41, progressInteger);
1487  else
1488  __COUTVS__(40, progressInteger);
1489 
1490  theSupervisor->allSupervisorInfo_.setSupervisorStatus(
1491  appInfo, status, progressInteger, detail, subapps);
1492 
1493  } // end of app loop
1494 
1495  if(oneStatusReqHasFailed)
1496  {
1497  __COUTT__ << "oneStatusReqHasFailed" << __E__;
1498  sleep(5); // sleep to not overwhelm server with errors
1499  }
1500 
1501  } // end of infinite status checking loop
1502 } // end AppStatusWorkLoop()
1503 
1504 //==============================================================================
1507 void GatewaySupervisor::GetRemoteGatewayIcons(
1508  GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp,
1509  const std::unique_ptr<TransceiverSocket>& /* not transferring ownership */
1510  remoteGatewaySocket)
1511 {
1512  // comma-separated icon string, 7 fields:
1513  // 0 - caption = text below icon
1514  // 1 - altText = text icon if no image given
1515  // 2 - uniqueWin = if true, only one window is allowed,
1516  // else multiple instances of window
1517  // 3 - permissions = security level needed to see icon
1518  // 4 - picfn = icon image filename
1519  // 5 - linkurl = url of the window to open
1520  // 6 - folderPath = folder and subfolder location '/' separated
1521  // for example: State Machine,FSM,1,200,icon-Physics.gif,/WebPath/html/StateMachine.html?fsm_name=OtherRuns0,,Chat,CHAT,1,1,icon-Chat.png,/urn:xdaq-application:lid=250,,Visualizer,VIS,0,10,icon-Visualizer.png,/WebPath/html/Visualization.html?urn=270,,Configure,CFG,0,10,icon-Configure.png,/urn:xdaq-application:lid=281,,Front-ends,CFG,0,15,icon-Configure.png,/WebPath/html/ConfigurationGUI_subset.html?urn=281&subsetBasePath=FEInterfaceTable&groupingFieldList=Status%2CFEInterfacePluginName&recordAlias=Front%2Dends&editableFieldList=%21%2ACommentDescription%2C%21SlowControls%2A,Config Subsets
1522 
1523  std::string iconString = "";
1524 
1525  std::string command = "GetRemoteDesktopIcons";
1526 
1527  __COUTT__ << "Sending remote gateway command '" << command << "' to target '"
1528  << remoteGatewayApp.appInfo.name
1529  << "' at url: " << remoteGatewayApp.appInfo.url << __E__;
1530 
1531  try
1532  {
1533  std::vector<std::string> parsedFields =
1534  StringMacros::getVectorFromString(remoteGatewayApp.appInfo.url, {':'});
1535  __COUTVS__(10, StringMacros::vectorToString(parsedFields));
1536  __COUTVS__(10, command);
1537 
1538  Socket gatewayRemoteSocket(parsedFields[1], atoi(parsedFields[2].c_str()));
1539  std::string remoteIconString = remoteGatewaySocket->sendAndReceive(
1540  gatewayRemoteSocket, command, 10 /*timeoutSeconds*/);
1541  __COUTVS__(10, remoteIconString);
1542 
1543  bool firstIcon = true;
1544 
1545  //now have remote icon string, append icons to list
1546  std::vector<std::string> remoteIconsCSV = StringMacros::getVectorFromString(
1547  remoteIconString + ",", //add 1 just in case last folder string is empty
1548  {','});
1549  const size_t numOfIconFields = 7;
1550  for(size_t i = 0; i + numOfIconFields < remoteIconsCSV.size();
1551  i += numOfIconFields)
1552  {
1553  if(firstIcon)
1554  firstIcon = false;
1555  else
1556  iconString += ",";
1557 
1558  __COUTVS__(10, remoteIconsCSV[i + 0]);
1559  if(remoteGatewayApp.parentIconFolderPath ==
1560  "") //icon.folderPath_ == "") //if not in folder, distinguish remote icon somehow
1561  iconString +=
1562  remoteGatewayApp.user_data_path_record //icon.alternateText_
1563  + " " + remoteIconsCSV[i + 0]; //icon.caption_;
1564  else
1565  iconString += remoteIconsCSV[i + 0]; //icon.caption_;
1566  iconString += "," + remoteIconsCSV[i + 1]; //icon.alternateText_;
1567  iconString +=
1568  "," +
1569  remoteIconsCSV
1570  [i + 2]; //std::string(icon.enforceOneWindowInstance_ ? "1" : "0");
1571  iconString += "," + std::string("1"); // set permission to 1 so the
1572  // desktop shows every icon that the
1573  // server allows (i.e., trust server
1574  // security, ignore client security)
1575  iconString += "," + remoteIconsCSV[i + 4]; //icon.imageURL_;
1576  iconString += "," + remoteIconsCSV[i + 5]; //icon.windowContentURL_;
1577 
1578  iconString += "," + remoteGatewayApp.parentIconFolderPath //icon.folderPath_
1579  + "/" + remoteIconsCSV[i + 6];
1580 
1581  } //end append remote icons
1582 
1583  } //end GetRemoteGatewayIcons()
1584  catch(const std::runtime_error& e)
1585  {
1586  __SS__ << "Failure gathering Remote Gateway desktop icons with command '"
1587  << command << "' from target '" << remoteGatewayApp.appInfo.name
1588  << "' at url: " << remoteGatewayApp.appInfo.url
1589  << " due to error: " << e.what() << __E__;
1590  __COUT_ERR__ << ss.str();
1591  remoteGatewayApp.error = ss.str();
1592  return;
1593  } //end GetRemoteGatewayIcons() catch
1594 
1595  __COUTV__(iconString);
1596  remoteGatewayApp.iconString = iconString;
1597 } //end GetRemoteGatewayIcons()
1598 
1599 //==============================================================================
1603 void GatewaySupervisor::SendRemoteGatewayCommand(
1604  GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp,
1605  const std::unique_ptr<TransceiverSocket>& /* not transferring ownership */
1606  remoteGatewaySocket)
1607 {
1608  remoteGatewayApp.error = ""; //clear error for new command
1609 
1610  __COUT__ << "Sending remote gateway command '" << remoteGatewayApp.command
1611  << "' to target '" << remoteGatewayApp.appInfo.name
1612  << "' at url: " << remoteGatewayApp.appInfo.url << __E__;
1613 
1614  std::string tmpCommand = remoteGatewayApp.command;
1615  try
1616  {
1617  std::string command;
1618 
1619  //for non-FSM commands, do not use fsmName
1620  if(remoteGatewayApp.command == "ResetConsoleCounts")
1621  command = "ResetConsoleCounts";
1622  else
1623  command = remoteGatewayApp.fsmName + "," + remoteGatewayApp.command;
1624 
1625  remoteGatewayApp.command = "Sent"; //Mark that send is being attempted
1626  if(tmpCommand == "Reboot") //do nothing for reboot command
1627  {
1628  __COUT__ << "Reboot command handled by ots script command." << __E__;
1629  sleep(5);
1630  return;
1631  }
1632 
1633  std::vector<std::string> parsedFields =
1634  StringMacros::getVectorFromString(remoteGatewayApp.appInfo.url, {':'});
1635  __COUTTV__(StringMacros::vectorToString(parsedFields));
1636  __COUT__ << "Sending to subsystem '" << remoteGatewayApp.appInfo.name
1637  << "' the command: " << command << __E__;
1638 
1639  Socket gatewayRemoteSocket(parsedFields[1], atoi(parsedFields[2].c_str()));
1640  std::string commandResponseString = remoteGatewaySocket->sendAndReceive(
1641  gatewayRemoteSocket, command, 10 /*timeoutSeconds*/);
1642  __COUT__ << "Response from subsystem '" << remoteGatewayApp.appInfo.name
1643  << "' received: " << commandResponseString << __E__;
1644 
1645  if(commandResponseString.find("Done") != 0) //then error
1646  {
1647  __SS__ << "Unsuccessful response received from Remote Gateway '"
1648  << remoteGatewayApp.appInfo.name + "' - here was the response: "
1649  << commandResponseString << __E__;
1650  __SS_THROW__;
1651  }
1652 
1653  if(commandResponseString.size() > strlen("Done") + 1)
1654  {
1655  //assume have config dump response!
1656  remoteGatewayApp.config_dump = "\n\n************************\n";
1657  remoteGatewayApp.config_dump +=
1658  "* Remote Subsystem Dump from '" + remoteGatewayApp.appInfo.name +
1659  "' at url: " + remoteGatewayApp.appInfo.url + "\n";
1660  remoteGatewayApp.config_dump += "* \n";
1661  remoteGatewayApp.config_dump += "\n\n";
1662  remoteGatewayApp.config_dump +=
1663  commandResponseString.substr(strlen("Done") + 1);
1664 
1665  __COUTTV__(remoteGatewayApp.config_dump);
1666  }
1667 
1668  } //end SendRemoteGatewayCommand()
1669  catch(const std::runtime_error& e)
1670  {
1671  __SS__ << "Failure sending Remote Gateway App '" << remoteGatewayApp.appInfo.name
1672  << "' the command '"
1673  << (tmpCommand.size() > 30 ? tmpCommand.substr(0, 30) : tmpCommand)
1674  << "' at url: " << remoteGatewayApp.appInfo.url
1675  << " due to error: " << e.what() << __E__;
1676  __COUT_ERR__ << ss.str();
1677  remoteGatewayApp.error = ss.str();
1678  } //end SendRemoteGatewayCommand() catch
1679 
1680 } //end SendRemoteGatewayCommand()
1681 
1682 //==============================================================================
1686 void GatewaySupervisor::CheckRemoteGatewayStatus(
1687  GatewaySupervisor::RemoteGatewayInfo& remoteGatewayApp,
1688  const std::unique_ptr<TransceiverSocket>& /* not transferring ownership */
1689  remoteGatewaySocket,
1690  const std::string& ipForReverseLoginOverUDP,
1691  int portForReverseLoginOverUDP)
1692 try
1693 {
1694  __COUTT__ << "Checking remote gateway status of '" << remoteGatewayApp.appInfo.name
1695  << "'" << __E__;
1696  std::vector<std::string> parsedFields =
1697  StringMacros::getVectorFromString(remoteGatewayApp.appInfo.url, {':'});
1698  __COUTTV__(StringMacros::vectorToString(parsedFields));
1699 
1700  if(parsedFields.size() == 3)
1701  {
1702  Socket gatewayRemoteSocket(parsedFields[1], atoi(parsedFields[2].c_str()));
1703  std::string requestString = "GetRemoteGatewayStatus";
1704  if(portForReverseLoginOverUDP)
1705  requestString += "," + ipForReverseLoginOverUDP + "," +
1706  std::to_string(portForReverseLoginOverUDP) + "," +
1707  remoteGatewayApp.appInfo.name;
1708  __COUT_TYPE__(TLVL_DEBUG + 24)
1709  << __COUT_HDR__ << "requestString = " << requestString << __E__;
1710  std::string remoteStatusString = remoteGatewaySocket->sendAndReceive(
1711  gatewayRemoteSocket, requestString, 2 /*timeoutSeconds*/);
1712  __COUT_TYPE__(TLVL_DEBUG + 24)
1713  << __COUT_HDR__ << "remoteStatusString = " << remoteStatusString << __E__;
1714 
1715  std::string value, name;
1716  bool foundGateway = false;
1717  size_t after = 0;
1718  while((name = StringMacros::extractXmlField(
1719  remoteStatusString, "name", 0, after, &after)) != "")
1720  {
1721  after += std::string("name").size(); //move beyond found pos
1722 
1723  //find class associated with record
1724  value = StringMacros::extractXmlField(remoteStatusString, "class", 0, after);
1725  if(value == XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
1726  {
1727  foundGateway = true;
1728 
1729  //found remote gateway
1730  __COUTVS__(25, remoteStatusString.size());
1731  __COUTVS__(25, after);
1732  __COUTVS__(25, value);
1733 
1734  //get gateway status
1735  value =
1736  StringMacros::extractXmlField(remoteStatusString, "status", 0, after);
1737  __COUTVS__(25, value);
1738  remoteGatewayApp.appInfo.status = value;
1739 
1740  value = StringMacros::extractXmlField(
1741  remoteStatusString, "progress", 0, after);
1742  __COUTVS__(25, value);
1743  remoteGatewayApp.appInfo.progress = atoi(value.c_str());
1744 
1745  value =
1746  StringMacros::extractXmlField(remoteStatusString, "detail", 0, after);
1747  __COUTVS__(25, value);
1748  remoteGatewayApp.appInfo.detail =
1749  value; //StringMacros::decodeURIComponent(value);
1750 
1751  value =
1752  StringMacros::extractXmlField(remoteStatusString, "time", 0, after);
1753  __COUTVS__(25, value);
1754  remoteGatewayApp.appInfo.lastStatusTime = atoi(value.c_str());
1755 
1756  value =
1757  StringMacros::extractXmlField(remoteStatusString, "url", 0, after);
1758  __COUTVS__(25, value);
1759  remoteGatewayApp.appInfo.parent_url = value;
1760 
1761  value = StringMacros::extractXmlField(remoteStatusString, "id", 0, after);
1762  __COUTVS__(25, value);
1763  remoteGatewayApp.appInfo.id = atoi(value.c_str());
1764 
1765  } //end found Remote Gateway status
1766  else //found remote subapp
1767  {
1768  //get remote subapp class name
1769  remoteGatewayApp.subapps[name].class_name = value;
1770  __COUTVS__(25, value);
1771 
1772  //get remote subapp status
1773  value =
1774  StringMacros::extractXmlField(remoteStatusString, "status", 0, after);
1775  __COUTVS__(25, value);
1776  remoteGatewayApp.subapps[name].status = value;
1777 
1778  value = StringMacros::extractXmlField(
1779  remoteStatusString, "progress", 0, after);
1780  __COUTVS__(25, value);
1781  remoteGatewayApp.subapps[name].progress = atoi(value.c_str());
1782 
1783  value =
1784  StringMacros::extractXmlField(remoteStatusString, "detail", 0, after);
1785  __COUTVS__(25, value);
1786  remoteGatewayApp.subapps[name].detail =
1787  value; //StringMacros::decodeURIComponent(value);
1788 
1789  value =
1790  StringMacros::extractXmlField(remoteStatusString, "time", 0, after);
1791  __COUTVS__(25, value);
1792  remoteGatewayApp.subapps[name].lastStatusTime = atoi(value.c_str());
1793 
1794  value =
1795  StringMacros::extractXmlField(remoteStatusString, "url", 0, after);
1796  __COUTVS__(25, value);
1797  remoteGatewayApp.subapps[name].parent_url = value;
1798 
1799  value = StringMacros::extractXmlField(remoteStatusString, "id", 0, after);
1800  __COUTVS__(25, value);
1801  remoteGatewayApp.subapps[name].id = atoi(value.c_str());
1802  }
1803  } //end primary loop
1804 
1805  if(!foundGateway)
1806  {
1807  __SS__ << "Failure encountered while checking remote gateway status of '"
1808  << remoteGatewayApp.appInfo.name
1809  << "' - no Gateway app status reported!" << __E__;
1810  __SS_THROW__;
1811  }
1812 
1813  //get system messages
1814  value = StringMacros::extractXmlField(
1815  remoteStatusString, "systemMessages", 0, after, &after);
1816  __COUT_TYPE__(TLVL_DEBUG + 2)
1817  << __COUT_HDR__ << "Remote System Messages:" << value << __E__;
1818  std::vector<std::string> parsedSysMsgs;
1819  StringMacros::getVectorFromString(value, parsedSysMsgs, {'|'});
1820 
1821  //Format: targetUser | time | msg | targetUser | time | msg...etc
1822  for(size_t i = 0; i + 2 < parsedSysMsgs.size(); i += 3)
1823  {
1824  GatewaySupervisor::addSystemMessage(
1825  parsedSysMsgs[i],
1826  "Remote System Message from '" + remoteGatewayApp.appInfo.name +
1827  "' at url: " + remoteGatewayApp.appInfo.url + " ... " +
1828  StringMacros::decodeURIComponent(parsedSysMsgs[i + 2]));
1829  } //end System Message handling loop
1830 
1831  //get user with lock
1832  value = StringMacros::extractXmlField(
1833  remoteStatusString, "usernameWithLock", 0, after, &after);
1834  __COUT_TYPE__(TLVL_DEBUG + 2)
1835  << __COUT_HDR__ << "Remote User with Lock:" << value << __E__;
1836  remoteGatewayApp.usernameWithLock = value;
1837 
1838  //get Console err/warn count
1839  value = StringMacros::extractXmlField(
1840  remoteStatusString, "console_err_count", 0, after, &after);
1841  __COUTVS__(25, value);
1842  remoteGatewayApp.consoleErrCount = atoi(value.c_str());
1843 
1844  value = StringMacros::extractXmlField(
1845  remoteStatusString, "console_warn_count", 0, after);
1846  __COUTVS__(25, value);
1847  remoteGatewayApp.consoleWarnCount = atoi(value.c_str());
1848  }
1849  else
1850  __COUT_WARN__ << "Illegal Remote Gateawy App URL (must be ots:<IP>:<PORT>): "
1851  << remoteGatewayApp.appInfo.url << __E__;
1852 } //end CheckRemoteGatewayStatus()
1853 catch(const std::runtime_error& e)
1854 {
1855  __COUT_WARN__ << "Failure getting Remote Gateway App status of '"
1856  << remoteGatewayApp.appInfo.name
1857  << "' at url: " << remoteGatewayApp.appInfo.url
1858  << " due to error: " << e.what() << __E__;
1859 
1860  remoteGatewayApp.appInfo.status = SupervisorInfo::APP_STATUS_UNKNOWN;
1861  remoteGatewayApp.appInfo.progress = 0;
1862  remoteGatewayApp.appInfo.detail = "Unknown UDP Message Error";
1863  remoteGatewayApp.appInfo.lastStatusTime = time(0);
1864 } //end CheckRemoteGatewayStatus() catch
1865 
1866 //==============================================================================
1869 void GatewaySupervisor::StateChangerWorkLoop(GatewaySupervisor* theSupervisor)
1870 {
1871  ConfigurationTree configLinkNode =
1872  theSupervisor->CorePropertySupervisorBase::getSupervisorTableNode();
1873 
1874  std::string ipAddressForStateChangesOverUDP =
1875  configLinkNode.getNode("IPAddressForStateChangesOverUDP").getValue<std::string>();
1876  int portForStateChangesOverUDP =
1877  configLinkNode.getNode("PortForStateChangesOverUDP").getValue<int>();
1878  bool acknowledgementEnabled =
1879  configLinkNode.getNode("EnableAckForStateChangesOverUDP").getValue<bool>();
1880  bool enableStateChanges =
1881  configLinkNode.getNode("EnableStateChangesOverUDP").getValue<bool>();
1882 
1883  __COUTV__(ipAddressForStateChangesOverUDP);
1884  __COUTV__(portForStateChangesOverUDP);
1885  __COUTV__(acknowledgementEnabled);
1886  __COUTV__(enableStateChanges);
1887 
1888  TransceiverSocket sock(ipAddressForStateChangesOverUDP,
1889  portForStateChangesOverUDP); // Take Port from Table
1890  try
1891  {
1892  sock.initialize();
1893  }
1894  catch(...)
1895  {
1896  // generate special message to indicate failed socket
1897  __SS__ << "FATAL Console error. Could not initialize socket at ip '"
1898  << ipAddressForStateChangesOverUDP << "' and port "
1899  << portForStateChangesOverUDP
1900  << ". Perhaps it is already in use? Exiting State Changer "
1901  "SOAPUtilities::receive loop."
1902  << __E__;
1903  __SS_THROW__;
1904  return;
1905  }
1906 
1907  std::size_t commaPosition;
1908  unsigned int commaCounter = 0;
1909  std::size_t begin = 0;
1910  std::string buffer;
1911  std::string errorStr;
1912  std::string fsmName;
1913  std::string command;
1914  std::vector<std::string> parameters;
1915  while(1)
1916  {
1917  // workloop procedure
1918  // if SOAPUtilities::receive a UDP command
1919  // execute command
1920  // else
1921  // sleep
1922 
1923  if(sock.receive(
1924  buffer, 0 /*timeoutSeconds*/, 1 /*timeoutUSeconds*/, false /*verbose*/) !=
1925  -1)
1926  {
1927  __COUT_TYPE__(TLVL_DEBUG + 9)
1928  << __COUT_HDR__ << "UDP State Changer packet received from ip:port "
1929  << sock.getLastIncomingIPAddress() << ":" << sock.getLastIncomingPort()
1930  << " of size = " << buffer.size() << __E__;
1931  __COUTVS__(11, buffer);
1932 
1933  try
1934  {
1935  bool remoteGatewayStatus = buffer.find("GetRemoteGatewayStatus") == 0;
1936  if(remoteGatewayStatus || buffer == "GetRemoteAppStatus")
1937  {
1938  __COUT_TYPE__(TLVL_DEBUG + 12)
1939  << "Giving app status to remote monitor..." << __E__;
1940 
1941  if(remoteGatewayStatus &&
1942  buffer.size() > strlen("GetRemoteGatewayStatus") + 1)
1943  {
1944  std::vector<std::string> params =
1945  StringMacros::getVectorFromString(buffer, {','});
1946  if(params.size() == 4)
1947  {
1948  //Parameters are "," + ipForReverseLoginOverUDP +
1949  // "," + std::to_string(portForReverseLoginOverUDP) +
1950  // "," + remoteGatewayApp.appInfo.name;
1951 
1952  __COUTVS__(23, StringMacros::vectorToString(params));
1953  std::string tmpIP = params[1];
1954  int tmpPort = atoi(params[2].c_str());
1955 
1956  if(!theSupervisor->theWebUsers_
1957  .remoteLoginVerificationEnabled_ ||
1958  theSupervisor->theWebUsers_.remoteLoginVerificationIP_ !=
1959  tmpIP ||
1960  theSupervisor->theWebUsers_.remoteLoginVerificationPort_ !=
1961  tmpPort)
1962  {
1963  theSupervisor->theWebUsers_.remoteLoginVerificationIP_ =
1964  tmpIP;
1965  theSupervisor->theWebUsers_.remoteLoginVerificationPort_ =
1966  tmpPort;
1967  theSupervisor->theWebUsers_.remoteGatewaySelfName_ =
1968  params[3];
1969  theSupervisor->theWebUsers_
1970  .remoteLoginVerificationEnabled_ =
1971  true; //mark as under remote control
1972  __COUT_INFO__
1973  << "This Gateway '"
1974  << theSupervisor->theWebUsers_.remoteGatewaySelfName_
1975  << "' is now under remote control and will validate "
1976  "logins through remote Gateway Supervisor at "
1977  << theSupervisor->theWebUsers_
1978  .remoteLoginVerificationIP_
1979  << ":"
1980  << theSupervisor->theWebUsers_
1981  .remoteLoginVerificationPort_
1982  << __E__;
1983  }
1984  }
1985  else
1986  __COUT_ERR__ << "Parameter count is not 4, it is "
1987  << params.size() << __E__;
1988  }
1989 
1990  HttpXmlDocument xmlOut;
1991  for(const auto& it :
1992  theSupervisor->allSupervisorInfo_.getAllSupervisorInfo())
1993  {
1994  const auto& appInfo = it.second;
1995  if(0 && //always return full status
1996  remoteGatewayStatus &&
1997  appInfo.getClass() !=
1998  XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
1999  continue; //only return Gateway status
2000 
2001  xmlOut.addTextElementToData(
2002  "name",
2003  appInfo.getName()); // get application name
2004  xmlOut.addTextElementToData(
2005  "id", std::to_string(appInfo.getId())); // get application id
2006  xmlOut.addTextElementToData("status",
2007  appInfo.getStatus()); // get status
2008  xmlOut.addTextElementToData(
2009  "time",
2010  std::to_string(
2011  appInfo
2012  .getLastStatusTime())); // ? StringMacros::getTimestampString(appInfo.getLastStatusTime()) : "0"); // get time stamp
2013  xmlOut.addTextElementToData(
2014  "stale",
2015  std::to_string(
2016  time(0) -
2017  appInfo.getLastStatusTime())); // time since update
2018  xmlOut.addTextElementToData(
2019  "progress",
2020  std::to_string(appInfo.getProgress())); // get progress
2021  xmlOut.addTextElementToData("detail",
2022  appInfo.getDetail()); // get detail
2023  xmlOut.addTextElementToData(
2024  "class",
2025  appInfo.getClass()); // get application class
2026  xmlOut.addTextElementToData(
2027  "url",
2028  appInfo.getURL()); // get application url
2029  xmlOut.addTextElementToData(
2030  "context",
2031  appInfo.getContextName()); // get context
2032  auto subappElement = xmlOut.addTextElementToData("subapps", "");
2033  for(auto& subappInfoPair : appInfo.getSubappInfo())
2034  {
2035  xmlOut.addTextElementToParent(
2036  "subapp_name", subappInfoPair.first, subappElement);
2037  xmlOut.addTextElementToParent("subapp_status",
2038  subappInfoPair.second.status,
2039  subappElement); // get status
2040  xmlOut.addTextElementToParent(
2041  "subapp_time",
2042  subappInfoPair.second.lastStatusTime
2043  ? StringMacros::getTimestampString(
2044  subappInfoPair.second.lastStatusTime)
2045  : "0",
2046  subappElement); // get time stamp
2047  xmlOut.addTextElementToParent(
2048  "subapp_stale",
2049  std::to_string(time(0) -
2050  subappInfoPair.second.lastStatusTime),
2051  subappElement); // time since update
2052  xmlOut.addTextElementToParent(
2053  "subapp_progress",
2054  std::to_string(subappInfoPair.second.progress),
2055  subappElement); // get progress
2056  xmlOut.addTextElementToParent("subapp_detail",
2057  subappInfoPair.second.detail,
2058  subappElement); // get detail
2059  xmlOut.addTextElementToParent("subapp_url",
2060  subappInfoPair.second.url,
2061  subappElement); // get url
2062  xmlOut.addTextElementToParent(
2063  "subapp_class",
2064  subappInfoPair.second.class_name,
2065  subappElement); // get class
2066  }
2067  }
2068 
2069  if(remoteGatewayStatus) //also return System Messages and console count and user-with-lock
2070  {
2071  __COUT_TYPE__(TLVL_DEBUG + 12)
2072  << "Giving extra Gateway info to remote monitor..." << __E__;
2073 
2074  xmlOut.addTextElementToData("systemMessages",
2075  theWebUsers_.getAllSystemMessages());
2076  xmlOut.addTextElementToData("usernameWithLock",
2077  theWebUsers_.getUserWithLock());
2078 
2079  std::lock_guard<std::mutex> lock(
2080  theSupervisor->systemStatusMutex_); //lock for rest of scope
2081  xmlOut.addTextElementToData(
2082  "console_err_count",
2083  std::to_string(theSupervisor->systemConsoleErrCount_));
2084  xmlOut.addTextElementToData(
2085  "console_warn_count",
2086  std::to_string(theSupervisor->systemConsoleWarnCount_));
2087  }
2088 
2089  std::stringstream out;
2090  xmlOut.outputXmlDocument((std::ostringstream*)&out,
2091  false /*dispStdOut*/,
2092  false /*allowWhiteSpace*/);
2093  __COUT_TYPE__(TLVL_DEBUG + 23)
2094  << __COUT_HDR__ << "App status to monitor: " << out.str()
2095  << __E__;
2096  sock.acknowledge(out.str(), false /* verbose */);
2097  continue;
2098  } //end GetRemoteAppStatus
2099  if(buffer == "ResetConsoleCounts")
2100  {
2101  __COUT__ << "Remote request to reset Console Counts..." << __E__;
2102 
2103  //zero out console count and retake first messages
2104 
2105  for(const auto& it :
2106  theSupervisor->allSupervisorInfo_.getAllSupervisorInfo())
2107  {
2108  const auto& appInfo = it.second;
2109  if(appInfo.isTypeConsoleSupervisor())
2110  {
2111  xoap::MessageReference tempMessage =
2112  SOAPUtilities::makeSOAPMessageReference(
2113  "ResetConsoleCounts");
2114  std::string reply =
2115  theSupervisor->send(appInfo.getDescriptor(), tempMessage);
2116 
2117  if(reply != "Done")
2118  {
2119  __SS__ << "Error while resetting console counts of "
2120  "Supervisor instance = '"
2121  << appInfo.getName()
2122  << "' [LID=" << appInfo.getId() << "] in Context '"
2123  << appInfo.getContextName()
2124  << "' [URL=" << appInfo.getURL() << "].\n\n"
2125  << reply << __E__;
2126  __SS_THROW__;
2127  }
2128  __COUT__ << "Reset console counts of Supervisor instance = '"
2129  << appInfo.getName() << "' [LID=" << appInfo.getId()
2130  << "] in Context '" << appInfo.getContextName()
2131  << "' [URL=" << appInfo.getURL() << "]." << __E__;
2132  }
2133  } //end loop for Console Supervisors
2134 
2135  //for user display feedback, clear local cached values also
2136  std::lock_guard<std::mutex> lock(
2137  theSupervisor->systemStatusMutex_); //lock for rest of scope
2138  theSupervisor->lastConsoleErrTime_ = "0";
2139  theSupervisor->lastConsoleErr_ = "";
2140  theSupervisor->lastConsoleWarnTime_ = "0";
2141  theSupervisor->lastConsoleWarn_ = "";
2142  theSupervisor->lastConsoleInfoTime_ = "0";
2143  theSupervisor->lastConsoleInfo_ = "";
2144  theSupervisor->firstConsoleErrTime_ = "0";
2145  theSupervisor->firstConsoleErr_ = "";
2146  theSupervisor->firstConsoleWarnTime_ = "0";
2147  theSupervisor->firstConsoleWarn_ = "";
2148  theSupervisor->firstConsoleInfoTime_ = "0";
2149  theSupervisor->firstConsoleInfo_ = "";
2150 
2151  sock.acknowledge("Done", false /* verbose */);
2152  continue;
2153  } //end ResetConsoleCounts
2154  else if(buffer.find("loginVerify") == 0)
2155  {
2156  __COUTT__
2157  << "Checking login verification request from remote gateway..."
2158  << __E__;
2159 
2160  //Lookup cookie code and return refreshed cookie code and user info
2161  // command = loginVerify
2162  // parameters.addParameter("CookieCode");
2163  // parameters.addParameter("RefreshOption");
2164  // parameters.addParameter("IPAddress");
2165  // -- Use name to lookup access level conversion for user
2166  // -- if Desktop Icon has a special permission type, then modify userGroupPermissionsMap's allUsers to match
2167  // parameters.addParameter("RemoteGatewaySelfName");
2168  std::vector<std::string> rxParams =
2169  StringMacros::getVectorFromString(buffer, {','});
2170  __COUTVS__(23, StringMacros::vectorToString(rxParams));
2171 
2172  if(rxParams.size() != 5)
2173  {
2174  __COUT_ERR__ << "Invalid remote login verify attempt! Expected 5 "
2175  "parameters, got "
2176  << rxParams.size() << __E__;
2177  sock.acknowledge("0", false /* verbose */);
2178  continue;
2179  }
2180 
2181  // If TRUE, cookie code is good, and refreshed code is in cookieCode, also pointers
2182  // optionally for uint8_t userPermissions, uint64_t uid Else, error message is
2183  // returned in cookieCode
2184  std::map<std::string /*groupName*/, WebUsers::permissionLevel_t>
2185  userGroupPermissionsMap;
2186  std::string userWithLock = "";
2187  uint64_t uid, userSessionIndex;
2188  std::string cookieCode = rxParams[1];
2189  if(!theWebUsers_.cookieCodeIsActiveForRequest(
2190  cookieCode /*cookieCode*/,
2191  &userGroupPermissionsMap,
2192  &uid /*uid is not given to remote users*/,
2193  "0" /* check at remote location because ip addresses change from subsystem to subsystem depending on tunnels,... rxParams[3] */
2194  /*ip*/,
2195  rxParams[2] /*refresh*/ == "1",
2196  false /* doNotGoRemote */,
2197  &userWithLock,
2198  &userSessionIndex))
2199  {
2200  __COUT_ERR__ << "Remote login failed!" << __E__;
2201  sock.acknowledge("0", false /* verbose */);
2202  continue;
2203  }
2204 
2205  //Modify Permission Map based on Desktop Icon permission requirement
2206  const std::string& remoteName = rxParams[4];
2207  __COUTVS__(23, remoteName);
2208  std::vector<GatewaySupervisor::RemoteGatewayInfo>
2209  remoteGatewayApps; //local copy
2210  { //lock for remainder of scope
2211  std::lock_guard<std::mutex> lock(
2212  theSupervisor->remoteGatewayAppsMutex_);
2213  remoteGatewayApps = theSupervisor->remoteGatewayApps_;
2214  __COUTVS__(22, remoteGatewayApps.size());
2215  }
2216 
2217  bool found = false;
2218  for(const auto& remoteGatewayApp : remoteGatewayApps)
2219  if(remoteName == remoteGatewayApp.appInfo.name)
2220  {
2221  found = true;
2222  __COUTVS__(21, remoteGatewayApp.permissionThresholdString);
2223 
2224  std::map<std::string /*groupName*/,
2225  WebUsers::permissionLevel_t>
2226  remoteIconPermissionsMap;
2227  StringMacros::getMapFromString(
2228  remoteGatewayApp.permissionThresholdString,
2229  remoteIconPermissionsMap);
2230 
2231  //if permission map is only size 1,
2232  // then modify WebUsers::DEFAULT_USER_GROUP for user with the icon's group level of the user
2233  // e.g. if user is 'HW: 255, allUsers: 1'
2234  // and icon is 'HW: 1'
2235  // then give to remote subsystem the user permission as 'HW: 255, allUsers: 255'
2236  //
2237  // ... this way the user is considered an expert at the remote subsystem
2238 
2239  if(remoteIconPermissionsMap.size() == 1 &&
2240  remoteIconPermissionsMap.begin()->first !=
2241  WebUsers::DEFAULT_USER_GROUP)
2242  {
2243  __COUTVS__(
2244  21,
2245  remoteIconPermissionsMap.begin()->first); //the group
2246 
2247  auto it = userGroupPermissionsMap.find(
2248  remoteIconPermissionsMap.begin()->first);
2249  std::map<std::string /*groupName*/,
2250  WebUsers::permissionLevel_t>::iterator it2 =
2251  userGroupPermissionsMap.find(
2252  WebUsers::DEFAULT_USER_GROUP);
2253  if(it != userGroupPermissionsMap.end() &&
2254  it2 != userGroupPermissionsMap.end())
2255  {
2256  __COUT_TYPE__(TLVL_DEBUG + 21)
2257  << __COUT_HDR__ << "Found user group '"
2258  << it->first
2259  << "' to modify: " << (uint16_t)it2->second
2260  << " --> " << (uint16_t)it->second << __E__;
2261  it2->second = it->second;
2262  __COUTVS__(21, (uint16_t)it2->second);
2263  }
2264  else if(
2265  it ==
2266  userGroupPermissionsMap
2267  .end()) //if special group not found, then no access
2268  userGroupPermissionsMap
2269  [remoteIconPermissionsMap.begin()->first] =
2270  WebUsers::PERMISSION_LEVEL_INACTIVE;
2271  }
2272 
2273  break;
2274  }
2275 
2276  if(!found)
2277  {
2278  __COUT_ERR__ << "Did not find any matching subsystems for remote "
2279  "login verify from '"
2280  << remoteName << "' attempted!" << __E__;
2281  }
2282 
2283  // Returned user info:
2284  // retParameters.addParameter("CookieCode", cookieCode);
2285  // "Permissions", StringMacros::mapToString(userGroupPermissionsMap).c_str());
2286  // "UserWithLock", userWithLock);
2287  // "Username", theWebUsers_.getUsersUsername(uid));
2288  // "DisplayName", theWebUsers_.getUsersDisplayName(uid));
2289  // "UserSessionIndex"
2290 
2291  std::string retStr = "";
2292  std::string username = theWebUsers_.getUsersUsername(uid);
2293  retStr += cookieCode;
2294  retStr +=
2295  "," + StringMacros::encodeURIComponent(
2296  StringMacros::mapToString(userGroupPermissionsMap));
2297  retStr += "," + userWithLock;
2298  retStr += "," + username;
2299  retStr += "," + theWebUsers_.getUsersDisplayName(uid);
2300  retStr += "," + std::to_string(userSessionIndex);
2301 
2302  __COUTVS__(23, retStr);
2303  __COUTT__ << "Remote login successful for " << username
2304  << ", userWithLock = " << userWithLock << __E__;
2305  sock.acknowledge(retStr, false /* verbose */);
2306  continue;
2307  }
2308  else if(buffer == "GetRemoteDesktopIcons")
2309  {
2310  __COUT__ << "Giving desktop icons to remote gateway..." << __E__;
2311 
2312  // get icons and create comma-separated string based on user permissions
2313  // note: each icon has own permission threshold, so each user can have
2314  // a unique desktop icon experience.
2315 
2316  // use latest context always from temporary configuration manager,
2317  // to get updated icons every time...
2318  //(so icon changes do no require an ots restart)
2319 
2320  ConfigurationManager
2321  tmpCfgMgr; // Creating new temporary instance so that constructor will activate latest context, note: not using member CorePropertySupervisorBase::theConfigurationManager_
2322  const DesktopIconTable* iconTable =
2323  tmpCfgMgr.__GET_CONFIG__(DesktopIconTable);
2324  const std::vector<DesktopIconTable::DesktopIcon>& icons =
2325  iconTable->getAllDesktopIcons();
2326 
2327  std::string iconString = "";
2328  // comma-separated icon string, 7 fields:
2329  // 0 - caption = text below icon
2330  // 1 - altText = text icon if no image given
2331  // 2 - uniqueWin = if true, only one window is allowed,
2332  // else multiple instances of window
2333  // 3 - permissions = security level needed to see icon
2334  // 4 - picfn = icon image filename
2335  // 5 - linkurl = url of the window to open
2336  // 6 - folderPath = folder and subfolder location '/' separated
2337  // for example: State Machine,FSM,1,200,icon-Physics.gif,/WebPath/html/StateMachine.html?fsm_name=OtherRuns0,,Chat,CHAT,1,1,icon-Chat.png,/urn:xdaq-application:lid=250,,Visualizer,VIS,0,10,icon-Visualizer.png,/WebPath/html/Visualization.html?urn=270,,Configure,CFG,0,10,icon-Configure.png,/urn:xdaq-application:lid=281,,Front-ends,CFG,0,15,icon-Configure.png,/WebPath/html/ConfigurationGUI_subset.html?urn=281&subsetBasePath=FEInterfaceTable&groupingFieldList=Status%2CFEInterfacePluginName&recordAlias=Front%2Dends&editableFieldList=%21%2ACommentDescription%2C%21SlowControls%2A,Config Subsets
2338 
2339  bool getRemoteIcons = true;
2340  bool firstIcon = true;
2341 
2342  //always force insert UserSettings so that the lock can be managed
2343  {
2344  if(firstIcon)
2345  firstIcon = false;
2346  else
2347  iconString += ",";
2348 
2349  iconString += "User Settings"; //icon.caption_;
2350  iconString += "," + std::string("User"); //icon.alternateText_;
2351  iconString +=
2352  "," +
2353  std::string(
2354  "1"); //std::string(icon.enforceOneWindowInstance_ ? "1" : "0");
2355  iconString +=
2356  "," + std::string("1"); // set permission to 1 so the
2357  // desktop shows every icon that the
2358  // server allows (i.e., trust server
2359  // security, ignore client security)
2360  iconString += "," + std::string(
2361  "/WebPath/images/dashboardImages/"
2362  "icon-Settings.png"); //icon.imageURL_;
2363  iconString +=
2364  "," + iconTable->getRemoteURL(
2365  &tmpCfgMgr, "/WebPath/html/UserSettings.html");
2366  iconString += "," + std::string(""); //icon.folderPath_;
2367  }
2368 
2369  for(const auto& icon : icons)
2370  {
2371  __COUTVS__(40, icon.caption_);
2372  __COUTVS__(40, icon.permissionThresholdString_);
2373 
2374  //ignore permission level, and give all icons
2375 
2376  if(getRemoteIcons)
2377  {
2378  __COUTVS__(10, icon.windowContentURL_);
2379  if(icon.windowContentURL_.size() > 4 &&
2380  icon.windowContentURL_[0] == 'o' &&
2381  icon.windowContentURL_[1] == 't' &&
2382  icon.windowContentURL_[2] == 's' &&
2383  icon.windowContentURL_[3] == ':')
2384  {
2385  __COUT_TYPE__(TLVL_DEBUG + 10)
2386  << __COUT_HDR__ << "Retrieving remote icons at "
2387  << icon.windowContentURL_ << __E__;
2388 
2389  std::vector<std::string> parsedFields =
2390  StringMacros::getVectorFromString(
2391  icon.windowContentURL_, {':'});
2392  __COUTVS__(10,
2393  StringMacros::vectorToString(parsedFields));
2394 
2395  if(parsedFields.size() == 3)
2396  {
2397  Socket iconRemoteSocket(
2398  parsedFields[1], atoi(parsedFields[2].c_str()));
2399 
2400  // ConfigurationTree configLinkNode = theSupervisor->CorePropertySupervisorBase::getSupervisorTableNode();
2401  // std::string ipAddressForStateChangesOverUDP = configLinkNode.getNode("IPAddressForStateChangesOverUDP").getValue<std::string>();
2402  __COUTVS__(10, ipAddressForStateChangesOverUDP);
2403  TransceiverSocket iconSocket(
2404  ipAddressForStateChangesOverUDP);
2405  std::string remoteIconString =
2406  iconSocket.sendAndReceive(iconRemoteSocket,
2407  "GetRemoteDesktopIcons",
2408  10 /*timeoutSeconds*/);
2409  __COUTVS__(10, remoteIconString);
2410  continue;
2411  }
2412  }
2413  } //end remote icon handling
2414 
2415  // have icon access, so add to CSV string
2416  if(firstIcon)
2417  firstIcon = false;
2418  else
2419  iconString += ",";
2420 
2421  __COUTVS__(10, icon.caption_);
2422  iconString += icon.caption_;
2423  iconString += "," + icon.alternateText_;
2424  iconString +=
2425  "," + std::string(icon.enforceOneWindowInstance_ ? "1" : "0");
2426  iconString +=
2427  "," + std::string("1"); // set permission to 1 so the
2428  // desktop shows every icon that the
2429  // server allows (i.e., trust server
2430  // security, ignore client security)
2431  iconString += "," + icon.imageURL_;
2432  iconString += "," + iconTable->getRemoteURL(
2433  &tmpCfgMgr, icon.windowContentURL_);
2434  iconString += "," + icon.folderPath_;
2435  }
2436  __COUTVS__(10, iconString);
2437 
2438  sock.acknowledge(iconString, true /* verbose */);
2439  continue;
2440  } //end GetRemoteDesktopIcons
2441  else if(!enableStateChanges) //else it is an FSM Command!
2442  {
2443  __COUT_WARN__ << "Skipping potential FSM Command because "
2444  "enableStateChanges=false"
2445  << __E__;
2446  continue;
2447  }
2448 
2449  __COUT__ << "Received a remote FSM Command attempt!" << __E__;
2450 
2451  size_t nCommas = std::count(buffer.begin(), buffer.end(), ',');
2452  if(nCommas == 0)
2453  {
2454  __SS__ << "Unrecognized State Machine command :-" << buffer
2455  << "-. Format is FiniteStateMachineName,Command,Parameter(s). "
2456  "Where Parameter(s) is/are optional."
2457  << __E__;
2458  __COUT_ERR__ << ss.str();
2459  if(acknowledgementEnabled)
2460  {
2461  __COUTT__ << "Ack'ing" << __E__;
2462  sock.acknowledge(ss.str(), true /* verbose */);
2463  }
2464  continue;
2465  }
2466  begin = 0;
2467  commaCounter = 0;
2468  parameters.clear();
2469  while((commaPosition = buffer.find(',', begin)) != std::string::npos ||
2470  commaCounter == nCommas)
2471  {
2472  if(commaCounter == nCommas)
2473  commaPosition = buffer.size();
2474  if(commaCounter == 0)
2475  fsmName = buffer.substr(begin, commaPosition - begin);
2476  else if(commaCounter == 1)
2477  command = buffer.substr(begin, commaPosition - begin);
2478  else
2479  parameters.push_back(buffer.substr(begin, commaPosition - begin));
2480  __COUT__ << "Word[" << commaCounter
2481  << "]: " << buffer.substr(begin, commaPosition - begin)
2482  << __E__;
2483 
2484  begin = commaPosition + 1;
2485  ++commaCounter;
2486  }
2487  __COUTV__(fsmName);
2488  __COUTV__(command);
2489  __COUTV__(StringMacros::vectorToString(parameters));
2490 
2491  // set scope of mutex
2492  std::string extraDoneContent = "";
2493  {
2494  // should be mutually exclusive with GatewaySupervisor main thread state
2495  // machine accesses lockout the messages array for the remainder of the
2496  // scope this guarantees the reading thread can safely access the
2497  // messages
2498  if(theSupervisor->VERBOSE_MUTEX)
2499  __COUT__ << "Waiting for FSM access" << __E__;
2500  std::lock_guard<std::mutex> lock(
2501  theSupervisor->stateMachineAccessMutex_);
2502  if(theSupervisor->VERBOSE_MUTEX)
2503  __COUT__ << "Have FSM access" << __E__;
2504 
2505  errorStr = theSupervisor->attemptStateMachineTransition(
2506  0,
2507  0,
2508  command,
2509  fsmName,
2510  WebUsers::DEFAULT_STATECHANGER_USERNAME /*fsmWindowName*/,
2511  WebUsers::DEFAULT_STATECHANGER_USERNAME,
2512  parameters);
2513 
2514  if(errorStr == "" &&
2515  command == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
2516  extraDoneContent =
2517  theSupervisor
2518  ->activeStateMachineConfigurationDumpOnConfigure_;
2519 
2520  if(errorStr == "" &&
2521  command == RunControlStateMachine::START_TRANSITION_NAME)
2522  extraDoneContent =
2523  theSupervisor->activeStateMachineConfigurationDumpOnRun_;
2524  }
2525 
2526  if(errorStr != "")
2527  {
2528  __SS__
2529  << "UDP State Changer failed to execute command because of the "
2530  "following error: "
2531  << errorStr;
2532  __COUT_ERR__ << ss.str();
2533  if(acknowledgementEnabled)
2534  sock.acknowledge(errorStr, true /* verbose */);
2535  }
2536  else
2537  {
2538  __SS__ << "Successfully executed state change command '" << command
2539  << ".'" << __E__;
2540  __COUT_INFO__ << ss.str();
2541  if(acknowledgementEnabled)
2542  sock.acknowledge(
2543  "Done" + (extraDoneContent.size()
2544  ? ("," + extraDoneContent)
2545  : "" //append extra done content, if any
2546  ),
2547  true /* verbose */);
2548  }
2549  }
2550  catch(...)
2551  {
2552  __SS__ << "Error was caught handling UDP command." << __E__;
2553  try
2554  {
2555  throw;
2556  }
2557  catch(const std::runtime_error& e)
2558  {
2559  ss << "Here is the error: " << e.what() << __E__;
2560  }
2561  catch(...)
2562  {
2563  ss << "Unrecognized error." << __E__;
2564  }
2565 
2566  __COUT_ERR__ << ss.str();
2567  if(acknowledgementEnabled)
2568  sock.acknowledge(ss.str(), true /* verbose */);
2569  }
2570  }
2571  else
2572  {
2573  __COUT_TYPE__(TLVL_DEBUG + 9)
2574  << __COUT_HDR__ << "UDP State Changer waiting..." << __E__;
2575  sleep(1);
2576  }
2577  }
2578 } // end StateChangerWorkLoop()
2579 
2580 //==============================================================================
2586 void GatewaySupervisor::makeSystemLogEntry(const std::string& entryText,
2587  const std::string& subjectText /* = "" */)
2588 {
2589  __COUT__ << "Making System Logbook Entry: " << entryText << __E__;
2590  if(subjectText.size())
2591  __COUTV__(subjectText);
2592  lastLogbookEntry_ = entryText;
2593  lastLogbookEntryTime_ = time(0);
2594 
2595  SupervisorInfoMap logbookInfoMap =
2596  allSupervisorInfo_.getAllLogbookTypeSupervisorInfo();
2597 
2598  if(logbookInfoMap.size() == 0)
2599  {
2600  __COUT__ << "No logbooks found! Here is entry: " << entryText << __E__;
2601  return;
2602  }
2603  else
2604  {
2605  __COUT__ << "Making logbook entry: " << entryText << __E__;
2606  }
2607 
2608  SOAPParameters parameters("EntryText", StringMacros::encodeURIComponent(entryText));
2609  parameters.addParameter("SubjectText", StringMacros::encodeURIComponent(subjectText));
2610 
2611  for(auto& logbookInfo : logbookInfoMap)
2612  {
2613  try
2614  {
2615  xoap::MessageReference retMsg = SOAPMessenger::sendWithSOAPReply(
2616  logbookInfo.second.getDescriptor(), "MakeSystemLogEntry", parameters);
2617 
2618  SOAPParameters retParameters("Status");
2619  SOAPUtilities::receive(retMsg, retParameters);
2620 
2621  std::string status = retParameters.getValue("Status");
2622  __COUT__ << "Returned Status: " << status
2623  << __E__; // retParameters[0].getValue() << __E__ << __E__;
2624  if(status != "Success")
2625  {
2626  __SS__ << "Invalid return status on MakeSystemLogEntry: " << status
2627  << __E__;
2628  __SS_THROW__;
2629  }
2630  }
2631  catch(const xdaq::exception::Exception& e) // due to xoap send failure
2632  {
2633  __SS__ << "Failed to send system log SOAP entry to "
2634  << logbookInfo.second.getContextName() << "/"
2635  << logbookInfo.second.getName() << " w/app ID=" << logbookInfo.first
2636  << __E__ << e.what();
2637 
2638  __SS_THROW__;
2639  }
2640  catch(std::runtime_error& e)
2641  {
2642  __SS__ << "Error during handling of system log SOAP entry at "
2643  << logbookInfo.second.getContextName() << "/"
2644  << logbookInfo.second.getName() << " w/app ID=" << logbookInfo.first
2645  << __E__ << e.what();
2646  __SS_THROW__;
2647  }
2648  }
2649 } // end makeSystemLogEntry()
2650 
2651 //==============================================================================
2652 void GatewaySupervisor::Default(xgi::Input* /*in*/, xgi::Output* out)
2653 {
2654  if(!supervisorGuiHasBeenLoaded_ &&
2655  (supervisorGuiHasBeenLoaded_ =
2656  true)) // make system logbook entry that ots has been started
2657  {
2658  bool doLog = false;
2659  try
2660  {
2661  doLog = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
2662  }
2663  catch(...)
2664  { /* ignore errors */
2665  ;
2666  }
2667 
2668  if(doLog)
2669  makeSystemLogEntry("ots started.");
2670  }
2671 
2672  *out << "<!DOCTYPE HTML><html lang='en'><head><title>ots</title>"
2673  << GatewaySupervisor::getIconHeaderString() <<
2674  // end show ots icon
2675  "</head>"
2676  << "<frameset col='100%' row='100%'>"
2677  << "<frame src='/WebPath/html/Desktop.html?urn="
2678  << this->getApplicationDescriptor()->getLocalId()
2679  << "&securityType=" << securityType_ << "'></frameset></html>";
2680 } // end Default()
2681 
2682 //==============================================================================
2683 std::string GatewaySupervisor::getIconHeaderString(void)
2684 {
2685  // show ots icon
2686  // from http://www.favicon-generator.org/
2687  return "<link rel='apple-touch-icon' sizes='57x57' href='/WebPath/images/otsdaqIcons/apple-icon-57x57.png'>\
2688  <link rel='apple-touch-icon' sizes='60x60' href='/WebPath/images/otsdaqIcons/apple-icon-60x60.png'>\
2689  <link rel='apple-touch-icon' sizes='72x72' href='/WebPath/images/otsdaqIcons/apple-icon-72x72.png'>\
2690  <link rel='apple-touch-icon' sizes='76x76' href='/WebPath/images/otsdaqIcons/apple-icon-76x76.png'>\
2691  <link rel='apple-touch-icon' sizes='114x114' href='/WebPath/images/otsdaqIcons/apple-icon-114x114.png'>\
2692  <link rel='apple-touch-icon' sizes='120x120' href='/WebPath/images/otsdaqIcons/apple-icon-120x120.png'>\
2693  <link rel='apple-touch-icon' sizes='144x144' href='/WebPath/images/otsdaqIcons/apple-icon-144x144.png'>\
2694  <link rel='apple-touch-icon' sizes='152x152' href='/WebPath/images/otsdaqIcons/apple-icon-152x152.png'>\
2695  <link rel='apple-touch-icon' sizes='180x180' href='/WebPath/images/otsdaqIcons/apple-icon-180x180.png'>\
2696  <link rel='icon' type='image/png' sizes='192x192' href='/WebPath/images/otsdaqIcons/android-icon-192x192.png'>\
2697  <link rel='icon' type='image/png' sizes='144x144' href='/WebPath/images/otsdaqIcons/android-icon-144x144.png'>\
2698  <link rel='icon' type='image/png' sizes='48x48' href='/WebPath/images/otsdaqIcons/android-icon-48x48.png'>\
2699  <link rel='icon' type='image/png' sizes='72x72' href='/WebPath/images/otsdaqIcons/android-icon-72x72.png'>\
2700  <link rel='icon' type='image/png' sizes='32x32' href='/WebPath/images/otsdaqIcons/favicon-32x32.png'>\
2701  <link rel='icon' type='image/png' sizes='96x96' href='/WebPath/images/otsdaqIcons/favicon-96x96.png'>\
2702  <link rel='icon' type='image/png' sizes='16x16' href='/WebPath/images/otsdaqIcons/favicon-16x16.png'>\
2703  <link rel='manifest' href='/WebPath/images/otsdaqIcons/manifest.json'>\
2704  <meta name='msapplication-TileColor' content='#ffffff'>\
2705  <meta name='msapplication-TileImage' content='/WebPath/images/otsdaqIcons/ms-icon-144x144.png'>\
2706  <meta name='theme-color' content='#ffffff'>";
2707 
2708 } // end getIconHeaderString()
2709 
2710 //==============================================================================
2712 void GatewaySupervisor::XGI_Turtle(xgi::Input* /*in*/, xgi::Output* out)
2713 {
2714  //test if ImageMagick is installed to do convert, if not just return existing png
2715  if(!picGen_.imageMagickInstallChecked)
2716  {
2717  //to install on AL9, sudo dnf install -y ImageMagick
2718  std::string ret =
2719  StringMacros::exec("convert --version"); //check if ImageMagick is installed
2720  __COUTVS__(50, ret);
2721  picGen_.imageMagickInstallChecked = true;
2722  picGen_.imageMagickInstalled = ret == "" ? false : true;
2723  }
2724 
2725  std::string filepath =
2726  __ENV__("OTSDAQ_WEB_PATH") + std::string("/images/otsdaqIcons/");
2727 
2728  std::string filename = filepath + "generated/turtle.png";
2729  if(picGen_.imageMagickInstalled)
2730  picGen_.generateTurtle(filepath);
2731  else
2732  filename = filepath + "turtle.png";
2733 
2734  //insertPngRawData(out,"images/generated/turtle.png");
2735  {
2736  //write raw picture data to output stream
2737  std::ifstream is;
2738  is.open(filename.c_str());
2739 
2740  *out << "data:image/png;charset=US-ASCII,";
2741 
2742  char CodeURL[4];
2743  while(is.good())
2744  { //print out safe Ascii equivalent
2745  sprintf(CodeURL, "%%%2.2X", (unsigned char)(is.get()));
2746  *out << CodeURL;
2747  }
2748 
2749  is.close();
2750  }
2751 } //end XGI_Turtle()
2752 
2753 //==============================================================================
2757 void GatewaySupervisor::stateMachineIterationBreakpoint(xgi::Input* in, xgi::Output* out)
2758 try
2759 {
2760  cgicc::Cgicc cgiIn(in);
2761 
2762  std::string requestType = CgiDataUtilities::getData(cgiIn, "Request");
2763 
2764  HttpXmlDocument xmlOut;
2765  WebUsers::RequestUserInfo userInfo(requestType,
2766  CgiDataUtilities::postData(cgiIn, "CookieCode"));
2767 
2768  CorePropertySupervisorBase::getRequestUserInfo(userInfo);
2769 
2770  if(!theWebUsers_.xmlRequestOnGateway(cgiIn, out, &xmlOut, userInfo))
2771  return; // access failed
2772 
2773  __COUTV__(requestType);
2774 
2775  try
2776  {
2777  if(requestType == "get")
2778  {
2779  std::stringstream v;
2780  { // start mutex scope
2781  std::lock_guard<std::mutex> lock(broadcastIterationBreakpointMutex_);
2782  v << broadcastIterationBreakpoint_;
2783  } // end mutex scope
2784 
2785  xmlOut.addTextElementToData("iterationBreakpoint", v.str());
2786  }
2787  else if(requestType == "set")
2788  {
2789  unsigned int breakpointSetValue =
2790  CgiDataUtilities::getDataAsInt(cgiIn, "breakpointSetValue");
2791  __COUTV__(breakpointSetValue);
2792 
2793  { // start mutex scope
2794  std::lock_guard<std::mutex> lock(broadcastIterationBreakpointMutex_);
2795  broadcastIterationBreakpoint_ = breakpointSetValue;
2796  } // end mutex scope
2797 
2798  // return the value that was set
2799  std::stringstream v;
2800  v << breakpointSetValue;
2801  xmlOut.addTextElementToData("iterationBreakpoint", v.str());
2802  }
2803  else
2804  {
2805  __SS__ << "Unknown iteration breakpoint request type = " << requestType
2806  << __E__;
2807  __SS_THROW__;
2808  }
2809  }
2810  catch(const std::runtime_error& e)
2811  {
2812  __SS__ << "Error caught handling iteration breakpoint command: " << e.what()
2813  << __E__;
2814  __COUT_ERR__ << ss.str();
2815  xmlOut.addTextElementToData("Error", ss.str());
2816  }
2817  catch(...)
2818  {
2819  __SS__ << "Unknown error caught handling iteration breakpoint command." << __E__;
2820  try
2821  {
2822  throw;
2823  } //one more try to printout extra info
2824  catch(const std::exception& e)
2825  {
2826  ss << "Exception message: " << e.what();
2827  }
2828  catch(...)
2829  {
2830  }
2831  __COUT_ERR__ << ss.str();
2832  xmlOut.addTextElementToData("Error", ss.str());
2833  } // end stateMachineIterationBreakpoint() catch
2834 
2835  xmlOut.outputXmlDocument((std::ostringstream*)out, false, true);
2836 
2837 } // end stateMachineIterationBreakpoint()
2838 catch(const std::runtime_error& e)
2839 {
2840  __SS__ << "Error caught handling iteration breakpoint command: " << e.what() << __E__;
2841  __COUT_ERR__ << ss.str();
2842 }
2843 catch(...)
2844 {
2845  __SS__ << "Unknown error caught handling iteration breakpoint command." << __E__;
2846  try
2847  {
2848  throw;
2849  } //one more try to printout extra info
2850  catch(const std::exception& e)
2851  {
2852  ss << "Exception message: " << e.what();
2853  }
2854  catch(...)
2855  {
2856  }
2857  __COUT_ERR__ << ss.str();
2858 } // end stateMachineIterationBreakpoint() catch
2859 
2860 //==============================================================================
2861 void GatewaySupervisor::stateMachineXgiHandler(xgi::Input* in, xgi::Output* out)
2862 {
2863  // for simplicity assume all commands should be mutually exclusive with iterator
2864  // thread state machine accesses (really should just be careful with
2865  // RunControlStateMachine access)
2866  if(VERBOSE_MUTEX)
2867  __COUT__ << "Waiting for FSM access" << __E__;
2868  std::lock_guard<std::mutex> lock(stateMachineAccessMutex_);
2869  if(VERBOSE_MUTEX)
2870  __COUT__ << "Have FSM access" << __E__;
2871 
2872  out->getHTTPResponseHeader().addHeader(
2873  "Access-Control-Allow-Origin",
2874  "*"); // to avoid block by blocked by CORS policy of browser
2875  cgicc::Cgicc cgiIn(in);
2876 
2877  std::string command = CgiDataUtilities::getData(cgiIn, "StateMachine");
2878  std::string requestType =
2879  "StateMachine-" + command; // prepend StateMachine to request type
2880  __COUTV__(requestType);
2881 
2882  HttpXmlDocument xmlOut;
2883  WebUsers::RequestUserInfo userInfo(requestType,
2884  CgiDataUtilities::postData(cgiIn, "CookieCode"));
2885 
2886  CorePropertySupervisorBase::getRequestUserInfo(userInfo);
2887 
2888  if(!theWebUsers_.xmlRequestOnGateway(cgiIn, out, &xmlOut, userInfo))
2889  return; // access failed
2890 
2891  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
2892  std::string fsmWindowName = CgiDataUtilities::getData(cgiIn, "fsmWindowName");
2893  fsmWindowName = StringMacros::decodeURIComponent(fsmWindowName);
2894  std::string currentState = theStateMachine_.getCurrentStateName();
2895 
2896  __COUT__ << "Check for Handled by theIterator_" << __E__;
2897 
2898  // check if Iterator should handle
2899  if((activeStateMachineWindowName_ == "" ||
2900  activeStateMachineWindowName_ == "iterator" ||
2901  (activeStateMachineName_ == fsmName &&
2902  command.find("iterate") ==
2903  0) /* for combo iterate/fsm GUIs like SubsystemLaunch.js */) &&
2904  theIterator_.handleCommandRequest(xmlOut, command, fsmWindowName))
2905  {
2906  __COUT__ << "Handled by theIterator_" << __E__;
2907  xmlOut.outputXmlDocument((std::ostringstream*)out, false);
2908  return;
2909  }
2910 
2911  // Do not allow transition while in transition
2912  if(theStateMachine_.isInTransition())
2913  {
2914  __SS__ << "Error - Can not accept request because the State Machine is already "
2915  "in transition!"
2916  << __E__;
2917  __COUT_ERR__ << "\n" << ss.str();
2918 
2919  xmlOut.addTextElementToData("state_tranisition_attempted",
2920  "0"); // indicate to GUI transition NOT attempted
2921  xmlOut.addTextElementToData(
2922  "state_tranisition_attempted_err",
2923  ss.str()); // indicate to GUI transition NOT attempted
2924  xmlOut.outputXmlDocument((std::ostringstream*)out, false, true);
2925  return;
2926  }
2927 
2928  // At this point, attempting transition!
2929 
2930  std::vector<std::string> parameters;
2931 
2932  if(command == "Configure")
2933  parameters.push_back(CgiDataUtilities::postData(cgiIn, "ConfigurationAlias"));
2934 
2935  std::string logEntry =
2936  StringMacros::decodeURIComponent(CgiDataUtilities::postData(cgiIn, "logEntry"));
2937 
2938  attemptStateMachineTransition(&xmlOut,
2939  out,
2940  command,
2941  fsmName,
2942  fsmWindowName,
2943  userInfo.username_,
2944  parameters,
2945  logEntry);
2946 
2947 } // end stateMachineXgiHandler()
2948 
2949 //==============================================================================
2950 std::string GatewaySupervisor::attemptStateMachineTransition(
2951  HttpXmlDocument* xmldoc,
2952  std::ostringstream* out,
2953  const std::string& command,
2954  const std::string& fsmName,
2955  const std::string& fsmWindowName,
2956  const std::string& username,
2957  const std::vector<std::string>& commandParameters,
2958  std::string logEntry /* = "" */)
2959 try
2960 {
2961  std::string errorStr = "";
2962 
2963  std::string currentState = theStateMachine_.getCurrentStateName();
2964  __COUT__ << "State Machine command = " << command << __E__;
2965  __COUT__ << "fsmName = " << fsmName << __E__;
2966  __COUT__ << "fsmWindowName = " << fsmWindowName << __E__;
2967  __COUTV__(username);
2968  __COUT__ << "activeStateMachineName_ = " << activeStateMachineName_ << __E__;
2969  __COUTV__(logEntry);
2970  __COUT__ << "command = " << command << __E__;
2971  __COUT__ << "commandParameters.size = " << commandParameters.size() << __E__;
2972  __COUTV__(StringMacros::vectorToString(commandParameters));
2973 
2974  //check if logEntry is in parameters
2975  if(!logEntry.size() && commandParameters.size() &&
2976  commandParameters.back().find("LogEntry:") == 0 &&
2977  commandParameters.back().size() > strlen("LogEntry:"))
2978  {
2979  logEntry = commandParameters.back().substr(strlen("LogEntry:"));
2980  __COUTV__(logEntry);
2981  }
2982 
2984  // Validate FSM name (do here because remote commands bypass stateMachineXgiHandler)
2985  // if fsm name != active fsm name
2986  // only allow, if current state is halted or init
2987  // take active fsm name when configured
2988  // else, allow
2989  if(activeStateMachineName_ != "" && activeStateMachineName_ != fsmName)
2990  {
2991  __COUT__ << "Validating... currentFSM = " << activeStateMachineName_
2992  << ", currentState = " << currentState << ", newFSM = " << fsmName
2993  << ", command = " << command << __E__;
2994  if(currentState != RunControlStateMachine::HALTED_STATE_NAME &&
2995  currentState != RunControlStateMachine::INITIAL_STATE_NAME)
2996  {
2997  // illegal for this FSM name to attempt transition
2998 
2999  __SS__ << "Error - Can not accept request because the State Machine "
3000  << "with window name '" << activeStateMachineWindowName_
3001  << "' (UID: " << activeStateMachineName_
3002  << ") "
3003  "is currently "
3004  << "in control of State Machine progress. ";
3005  ss << "\n\nIn order for this State Machine with window name '"
3006  << fsmWindowName << "' (UID: " << fsmName
3007  << ") "
3008  "to control progress, please transition to "
3009  << RunControlStateMachine::HALTED_STATE_NAME << " using the active "
3010  << "State Machine '" << activeStateMachineWindowName_ << ".'" << __E__;
3011  __SS_THROW__;
3012  }
3013  else // clear active state machine
3014  {
3015  activeStateMachineName_ = "";
3016  activeStateMachineWindowName_ = "";
3017  }
3018  }
3019  //FSM name validated
3020 
3021  if(logEntry != "")
3022  {
3023  logEntry += " (" + StringMacros::getTimestampString(time(0)) + ")";
3024 
3025  if(command == RunControlStateMachine::START_TRANSITION_NAME &&
3026  getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME).size())
3027  {
3028  //add configure log entry to start log entry
3029 
3030  logEntry +=
3031  "\n\nThe last Configure transition log entry was this:\n" +
3032  getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME);
3033  }
3034 
3035  bool doLog = false;
3036  try
3037  {
3038  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
3039  }
3040  catch(...)
3041  { /* ignore errors */
3042  ;
3043  }
3044 
3045  if(doLog)
3046  makeSystemLogEntry("Attempting FSM command '" + command + "' from state '" +
3047  currentState + "' with user log entry: " + logEntry);
3048  }
3049 
3050  setLastLogEntry(command, logEntry);
3051 
3052  SOAPParameters parameters;
3053  if(command == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
3054  {
3055  activeStateMachineConfigurationDumpOnConfigure_ =
3056  ""; //clear (and set if enabled during configure transition)
3057  activeStateMachineConfigurationDumpOnRun_ =
3058  ""; //clear (and set if enabled during configure transition)
3059  activeStateMachineConfigurationDumpOnRunEnable_ = false,
3060  activeStateMachineConfigurationDumpOnConfigureEnable_ =
3061  false; //clear (and set if enabled during configure transition)
3062  activeStateMachineConfigurationDumpOnConfigureFilename_ =
3063  ""; //clear (and set if enabled during configure transition)
3064  activeStateMachineConfigurationDumpOnRunFilename_ =
3065  ""; //clear (and set if enabled during configure transition)
3066 
3067  activeStateMachineRequireUserLogOnRun_ = false,
3068  activeStateMachineRequireUserLogOnConfigure_ =
3069  false; //clear (and set if enabled during configure transition)
3070  activeStateMachineRunInfoPluginType_ = TableViewColumnInfo::
3071  DATATYPE_STRING_DEFAULT; //clear (and set if enabled during configure transition)
3072 
3073  if(currentState != RunControlStateMachine::HALTED_STATE_NAME &&
3074  currentState != RunControlStateMachine::
3075  INITIAL_STATE_NAME) // check if out of sync command
3076  {
3077  __SS__ << "Error - Can only transition to Configured if the current "
3078  << "state is Initial or Halted. The current state is '" << currentState
3079  << ".' Perhaps your state machine is out of sync, or you need to Halt "
3080  "before Configuring."
3081  << __E__;
3082  __SS_THROW__;
3083  }
3084 
3085  // Note: Original name of the configuration key was RUN_KEY
3086  // parameters.addParameter("RUN_KEY",CgiDataUtilities::postData(cgi,"ConfigurationAlias"));
3087  if(commandParameters.size() == 0)
3088  {
3089  __SS__ << "Error - Can only transition to Configured if a Configuration "
3090  "Alias parameter is provided."
3091  << __E__;
3092  __SS_THROW__;
3093  }
3094 
3095  // check if configuration dump is enabled on configure transition
3096  std::string dumpFormatOnConfigure, dumpFormatOnRun;
3097  {
3098  ConfigurationTree configLinkNode =
3099  CorePropertySupervisorBase::theConfigurationManager_
3100  ->getSupervisorTableNode(supervisorContextUID_,
3101  supervisorApplicationUID_);
3102  if(!configLinkNode.isDisconnected())
3103  {
3104  bool doThrow = false;
3105  try // for backwards compatibility
3106  {
3107  ConfigurationTree fsmLinkNode =
3108  configLinkNode.getNode("LinkToStateMachineTable")
3109  .getNode(fsmName);
3110 
3111  try
3112  {
3113  activeStateMachineRequireUserLogOnConfigure_ =
3114  fsmLinkNode
3115  .getNode("RequireUserLogInputOnConfigureTransition")
3116  .getValue<bool>();
3117  }
3118  catch(...)
3119  {
3120  ;
3121  }
3122  try
3123  {
3124  activeStateMachineRequireUserLogOnRun_ =
3125  fsmLinkNode.getNode("RequireUserLogInputOnRunTransition")
3126  .getValue<bool>();
3127  }
3128  catch(...)
3129  {
3130  ;
3131  }
3132  try
3133  {
3134  activeStateMachineRunAlias_ =
3135  fsmLinkNode.getNode("RunDisplayAlias")
3136  .getValueWithDefault<std::string>(
3137  "Run" /* defaultValue */);
3138  }
3139  catch(...)
3140  {
3141  activeStateMachineRunAlias_ = "Run";
3142  }
3143  try
3144  {
3145  activeStateMachineRollOverLogOnConfigure_ =
3146  fsmLinkNode.getNode("RollOverLogOnConfigure")
3147  .getValueWithDefault<bool>(false /* defaultValue */);
3148  }
3149  catch(...)
3150  {
3151  activeStateMachineRollOverLogOnConfigure_ = false;
3152  }
3153  try
3154  {
3155  activeStateMachineRollOverLogOnStart_ =
3156  fsmLinkNode.getNode("RollOverLogOnStart")
3157  .getValueWithDefault<bool>(false /* defaultValue */);
3158  }
3159  catch(...)
3160  {
3161  activeStateMachineRollOverLogOnStart_ = false;
3162  }
3163 
3164  try
3165  {
3166  activeStateMachineRunInfoPluginType_ =
3167  fsmLinkNode.getNode("RunInfoPluginType")
3168  .getValue<std::string>();
3169  }
3170  catch(...) //ignore missing RunInfoPluginType
3171  {
3172  __COUT__ << "RunInfoPluginType not defined for FSM name '"
3173  << fsmName
3174  << "' - please setup a valid run info plugin type to "
3175  "enable external Run Number coordination and dumping "
3176  "configuration info to an external location."
3177  << __E__;
3178  }
3179 
3180  activeStateMachineConfigurationDumpOnConfigureEnable_ =
3181  fsmLinkNode
3182  .getNode("EnableConfigurationDumpOnConfigureTransition")
3183  .getValue<bool>();
3184  activeStateMachineConfigurationDumpOnRunEnable_ =
3185  fsmLinkNode.getNode("EnableConfigurationDumpOnRunTransition")
3186  .getValue<bool>();
3187 
3188  doThrow = true; // at this point throw the exception!
3189 
3190  dumpFormatOnConfigure =
3191  fsmLinkNode.getNode("ConfigurationDumpOnConfigureFormat")
3192  .getValue<std::string>();
3193  dumpFormatOnRun = fsmLinkNode.getNode("ConfigurationDumpOnRunFormat")
3194  .getValue<std::string>();
3195 
3196  std::string dumpFilePath, dumpFileRadix;
3197  dumpFilePath =
3198  fsmLinkNode.getNode("ConfigurationDumpOnConfigureFilePath")
3199  .getValueWithDefault<std::string>(__ENV__("OTSDAQ_LOG_DIR"));
3200  dumpFileRadix =
3201  fsmLinkNode.getNode("ConfigurationDumpOnConfigureFileRadix")
3202  .getValueWithDefault<std::string>(
3203  "ConfigTransitionConfigurationDump");
3204  activeStateMachineConfigurationDumpOnConfigureFilename_ =
3205  dumpFilePath + "/" + dumpFileRadix;
3206  dumpFilePath =
3207  fsmLinkNode.getNode("ConfigurationDumpOnRunFilePath")
3208  .getValueWithDefault<std::string>(__ENV__("OTSDAQ_LOG_DIR"));
3209  dumpFileRadix = fsmLinkNode.getNode("ConfigurationDumpOnRunFileRadix")
3210  .getValueWithDefault<std::string>(
3211  "ConfigTransitionConfigurationDump");
3212  activeStateMachineConfigurationDumpOnRunFilename_ =
3213  dumpFilePath + "/" + dumpFileRadix;
3214  }
3215  catch(
3216  std::runtime_error&
3217  e) // throw exception on missing fields if dumpConfiguration set
3218  {
3219  if(doThrow &&
3220  (activeStateMachineConfigurationDumpOnConfigureEnable_ ||
3221  activeStateMachineConfigurationDumpOnRunEnable_))
3222  {
3223  __SS__ << "Configuration Dump was enabled, but there are missing "
3224  "fields! "
3225  << e.what() << __E__;
3226  __SS_THROW__;
3227  }
3228  else
3229  __COUT_INFO__ << "FSM configuration dump Link disconnected at '"
3230  << ConfigurationManager::XDAQ_CONTEXT_TABLE_NAME
3231  << "/" << supervisorContextUID_ << "/"
3232  << supervisorApplicationUID_ << "/"
3233  << "LinkToStateMachineTable/" << fsmName << "/"
3234  << "EnableConfigurationDumpOnConfigureTransition "
3235  "and/or EnableConfigurationDumpOnRunTransition"
3236  << __E__;
3237  }
3238  } //end configuration dump check/handling
3239  else
3240  __COUT_INFO__ << "No Gateway Supervisor configuration record found at '"
3241  << ConfigurationManager::XDAQ_CONTEXT_TABLE_NAME << "/"
3242  << supervisorContextUID_ << "/" << supervisorApplicationUID_
3243  << "' - consider adding one to control configuration dumps "
3244  "and state machine properties."
3245  << __E__;
3246  } //end check if configuration dump is enabled on configure transition
3247 
3248  __COUTTV__(activeStateMachineRequireUserLogOnConfigure_);
3249  __COUTTV__(activeStateMachineRequireUserLogOnRun_);
3250  __COUTTV__(activeStateMachineRunAlias_);
3251  __COUTTV__(activeStateMachineRunInfoPluginType_);
3252  __COUTTV__(activeStateMachineConfigurationDumpOnConfigureEnable_);
3253  __COUTTV__(activeStateMachineConfigurationDumpOnRunEnable_);
3254  __COUTTV__(dumpFormatOnConfigure);
3255  __COUTTV__(dumpFormatOnRun);
3256  __COUTTV__(activeStateMachineConfigurationDumpOnConfigureFilename_);
3257  __COUTTV__(activeStateMachineConfigurationDumpOnRunFilename_);
3258  __COUTTV__(activeStateMachineRollOverLogOnConfigure_);
3259  __COUTTV__(activeStateMachineRollOverLogOnStart_);
3260 
3261  if(activeStateMachineRequireUserLogOnConfigure_ &&
3262  getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME).size() < 3)
3263  {
3264  __SS__ << "Error - the state machine property "
3265  "'RequireUserLogInputOnConfigureTransition' has been enabled which "
3266  "requires the user to enter "
3267  "at least 3 characters of log info to proceed with the Configure "
3268  "transition."
3269  << __E__;
3270  __SS_THROW__;
3271  }
3272 
3273  parameters.addParameter("ConfigurationAlias", commandParameters[0]);
3274 
3275  std::string configurationAlias = parameters.getValue("ConfigurationAlias");
3276  __COUT__ << "Configure --> Name: ConfigurationAlias Value: " << configurationAlias
3277  << __E__;
3278  lastConfigurationAlias_ = configurationAlias;
3279  // save last used config alias
3280  std::string fn = ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH + "/" +
3281  FSM_LAST_GROUP_ALIAS_FILE_START + fsmName + "." +
3282  FSM_USERS_PREFERENCES_FILETYPE;
3283 
3284  __COUT__ << "Save FSM preferences: " << fn << __E__;
3285  FILE* fp = fopen(fn.c_str(), "w");
3286  if(!fp)
3287  {
3288  __SS__ << ("Could not open file: " + fn) << __E__;
3289  __SS_THROW__;
3290  }
3291  fprintf(fp, "FSM_last_configuration_alias %s", configurationAlias.c_str());
3292  fclose(fp);
3293 
3294  activeStateMachineName_ = fsmName;
3295  activeStateMachineWindowName_ = fsmWindowName;
3296 
3297  if(activeStateMachineName_ == "")
3298  __COUT_WARN__
3299  << "The active state machine is an empty string, this is allowed for "
3300  "backwards compatibility, but may not be intentional! "
3301  << "Make sure you or your system admins understand why the active FSM "
3302  "name is blank."
3303  << __E__;
3304 
3305  //Note: Must create configuration dump at this point!! In case this is a remote subsystem and must respond with string
3306  //Must define activeStateMachineConfigurationDumpOnRun_, activeStateMachineConfigurationDumpOnConfigure_; //cached at Configure transition
3307 
3308  try
3309  {
3310  CorePropertySupervisorBase::theConfigurationManager_
3311  ->init(); // completely reset to re-align with any changes
3312  }
3313  catch(...)
3314  {
3315  __SS__ << "\nTransition to Configuring interrupted! "
3316  << "The Configuration Manager could not be initialized." << __E__;
3317  __SS_THROW__;
3318  }
3319 
3320  // Translate the system alias to a group name/key
3321  try
3322  {
3323  theConfigurationTableGroup_ =
3324  CorePropertySupervisorBase::theConfigurationManager_
3325  ->getTableGroupFromAlias(configurationAlias);
3326  }
3327  catch(...)
3328  {
3329  __COUT_INFO__
3330  << "Exception occurred translating the Configuration System Alias."
3331  << __E__;
3332  }
3333 
3334  if(theConfigurationTableGroup_.second.isInvalid())
3335  {
3336  __SS__
3337  << "\nTransition to Configuring interrupted! System Configuration Alias '"
3338  << configurationAlias
3339  << "' could not be translated to a group name and key." << __E__;
3340  __SS_THROW__;
3341  }
3342 
3343  __COUT_INFO__ << "Configuration table group name: "
3344  << theConfigurationTableGroup_.first
3345  << " key: " << theConfigurationTableGroup_.second << __E__;
3346 
3347  // load and activate Configuration Alias
3348  try
3349  {
3350  //first get group type - it must be Configuration type!
3351  std::string groupTypeString;
3352  CorePropertySupervisorBase::theConfigurationManager_->loadTableGroup(
3353  theConfigurationTableGroup_.first,
3354  theConfigurationTableGroup_.second,
3355  false /*doActivate*/,
3356  0 /*groupMembers */,
3357  0 /*progressBar */,
3358  0 /*accumulateWarnings*/,
3359  0 /*groupComment */,
3360  0 /*groupAuthor */,
3361  0 /*groupCreateTime */,
3362  true /*doNotLoadMember */,
3363  &groupTypeString);
3364  if(groupTypeString != ConfigurationManager::GROUP_TYPE_NAME_CONFIGURATION)
3365  {
3366  __SS__ << "Illegal attempted configuration group type. The table group '"
3367  << theConfigurationTableGroup_.first << "("
3368  << theConfigurationTableGroup_.second << ")' is of type "
3369  << groupTypeString << ". It must be "
3370  << ConfigurationManager::GROUP_TYPE_NAME_CONFIGURATION << "."
3371  << __E__;
3372  __SS_THROW__;
3373  }
3374 
3375  CorePropertySupervisorBase::theConfigurationManager_->loadTableGroup(
3376  theConfigurationTableGroup_.first,
3377  theConfigurationTableGroup_.second,
3378  true /*doActivate*/);
3379 
3380  __COUT__ << "Done loading Configuration Alias." << __E__;
3381 
3382  // mark the translated group as the last activated group
3383  std::pair<std::string /*group name*/, TableGroupKey> activatedGroup(
3384  std::string(theConfigurationTableGroup_.first),
3385  theConfigurationTableGroup_.second);
3386  ConfigurationManager::saveGroupNameAndKey(
3387  activatedGroup, ConfigurationManager::LAST_ACTIVATED_CONFIG_GROUP_FILE);
3388 
3389  __COUT__ << "Done activating Configuration Alias." << __E__;
3390  }
3391  catch(const std::runtime_error& e)
3392  {
3393  __SS__
3394  << "\nTransition to Configuring interrupted! System Configuration Alias "
3395  << configurationAlias << " was translated to "
3396  << theConfigurationTableGroup_.first << " ("
3397  << theConfigurationTableGroup_.second
3398  << ") but could not be loaded and initialized." << __E__;
3399  ss << "\n\nHere was the error: " << e.what()
3400  << "\n\nTo help debug this problem, try activating this group in the "
3401  "Configuration "
3402  "GUI "
3403  << " and detailed errors will be shown." << __E__;
3404  __SS_THROW__;
3405  }
3406  catch(...)
3407  {
3408  __SS__
3409  << "\nTransition to Configuring interrupted! System Configuration Alias "
3410  << configurationAlias << " was translated to "
3411  << theConfigurationTableGroup_.first << " ("
3412  << theConfigurationTableGroup_.second
3413  << ") but could not be loaded and initialized." << __E__;
3414  try
3415  {
3416  throw;
3417  } //one more try to printout extra info
3418  catch(const std::exception& e)
3419  {
3420  ss << "Exception message: " << e.what();
3421  }
3422  catch(...)
3423  {
3424  }
3425  ss << "\n\nTo help debug this problem, try activating this group in the "
3426  "Configuration "
3427  "GUI "
3428  << " and detailed errors will be shown." << __E__;
3429  __SS_THROW__;
3430  }
3431 
3432  //at this point Configuration Tree is fully loaded
3433 
3434  //handle configuration dump if enabled on configure transition
3435  try // errors in dump are not tolerated
3436  {
3437  //get/cache Run transition dump
3438  if(activeStateMachineConfigurationDumpOnRunEnable_ ||
3439  ((activeStateMachineRunInfoPluginType_ !=
3440  TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3441  activeStateMachineRunInfoPluginType_ != "No Run Info Plugin")))
3442  {
3443  __COUT_INFO__
3444  << "Caching the Configuration Dump for the Run transition..."
3445  << __E__;
3446 
3447  // dump configuration
3448  std::stringstream dumpSs;
3449  CorePropertySupervisorBase::theConfigurationManager_
3450  ->dumpActiveConfiguration(
3451  "", //dumpFilePath + "/" + dumpFileRadix + "_" + std::to_string(time(0)) + ".dump",
3452  dumpFormatOnRun,
3453  lastConfigurationAlias_,
3454  getLastLogEntry(
3455  RunControlStateMachine::CONFIGURE_TRANSITION_NAME),
3456  theWebUsers_.getActiveUsersString(),
3457  dumpSs);
3458 
3459  activeStateMachineConfigurationDumpOnRun_ = dumpSs.str();
3460  }
3461  else
3462  __COUT_INFO__
3463  << "Not caching the Configuration Dump on the Run transition."
3464  << __E__;
3465 
3466  //get/cache Configuration transition dump
3467  if(activeStateMachineConfigurationDumpOnConfigureEnable_ ||
3468  ((activeStateMachineRunInfoPluginType_ !=
3469  TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3470  activeStateMachineRunInfoPluginType_ != "No Run Info Plugin")))
3471  {
3472  __COUT_INFO__
3473  << "Caching the Configuration Dump for the Configure transition..."
3474  << __E__;
3475 
3476  // dump configuration
3477  std::stringstream dumpSs;
3478  CorePropertySupervisorBase::theConfigurationManager_
3479  ->dumpActiveConfiguration(
3480  "", //dumpFilePath + "/" + dumpFileRadix + "_" + std::to_string(time(0)) + ".dump",
3481  dumpFormatOnConfigure,
3482  lastConfigurationAlias_,
3483  getLastLogEntry(
3484  RunControlStateMachine::CONFIGURE_TRANSITION_NAME),
3485  theWebUsers_.getActiveUsersString(),
3486  dumpSs);
3487 
3488  activeStateMachineConfigurationDumpOnConfigure_ = dumpSs.str();
3489  }
3490  else
3491  __COUT_INFO__
3492  << "Not caching the Configuration Dump on the Configure transition."
3493  << __E__;
3494 
3495  } //end handle configuration dump if enabled on configure transition
3496  catch(const std::runtime_error& e)
3497  {
3498  __SS__ << "Error encoutered during configuration dump. Here is the error: "
3499  << e.what();
3500  __SS_THROW__;
3501  }
3502  catch(...)
3503  {
3504  __SS__ << "Unknown error encoutered during configuration dump.";
3505  __SS_THROW__;
3506  }
3507 
3508  } //end Configure transition
3509  else if(command == RunControlStateMachine::START_TRANSITION_NAME)
3510  {
3511  if(currentState !=
3512  RunControlStateMachine::CONFIGURED_STATE_NAME) // check if out of sync command
3513  {
3514  __SS__
3515  << "Error - Can only transition to Configured if the current "
3516  << "state is Halted. Perhaps your state machine is out of sync. "
3517  << "(Likely the server was restarted or another user changed the state)"
3518  << __E__;
3519  __SS_THROW__;
3520  }
3521 
3522  if(activeStateMachineRequireUserLogOnRun_ &&
3523  getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME).size() < 3)
3524  {
3525  __SS__
3526  << "Error - the state machine property "
3527  "'RequireUserLogInputOnRunTransition' has been enabled which requires "
3528  "the user to enter "
3529  "at least 3 characters of log info to proceed with the Run transition."
3530  << __E__;
3531  __SS_THROW__;
3532  }
3533 
3534  unsigned long runNumber;
3535  if(commandParameters.size() == 0)
3536  {
3537  runNumber = getNextRunNumber();
3538  // Check if run number should come from db, if so create run info record into database
3539 
3540  __COUTV__(activeStateMachineRunInfoPluginType_);
3541 
3542  if(activeStateMachineRunInfoPluginType_ !=
3543  TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3544  activeStateMachineRunInfoPluginType_ != "No Run Info Plugin")
3545  {
3546  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
3547  try
3548  {
3549  runInfoInterface.reset(makeRunInfo(
3550  activeStateMachineRunInfoPluginType_, activeStateMachineName_));
3551  }
3552  catch(...)
3553  {
3554  ;
3555  }
3556  if(runInfoInterface == nullptr)
3557  {
3558  __SS__ << "Run Info interface plugin construction failed of type "
3559  << activeStateMachineRunInfoPluginType_
3560  << " for claiming next run number!" << __E__;
3561  __SS_THROW__;
3562  }
3563 
3564  //FIXME -- October 2024, by rrivera (need future simplification from agioiosa) - Should this 2nd param be activeStateMachineConfigurationDumpOnConfigure_?! What is the 2nd param for? Is conditionID_ enough?
3565  runNumber = runInfoInterface->claimNextRunNumber(
3566  conditionID_, activeStateMachineConfigurationDumpOnRun_);
3567  } // end Run Info Plugin handling
3568 
3569  setNextRunNumber(runNumber + 1);
3570  }
3571  else
3572  {
3573  sscanf(commandParameters[0].c_str(), "%lu", &runNumber);
3574  __COUTV__(runNumber);
3575  setNextRunNumber(runNumber + 1);
3576  }
3577 
3578  setLastLogEntry(command, "Run #" + std::to_string(runNumber) + ": " + logEntry);
3579  parameters.addParameter("RunNumber", runNumber);
3580  } //end Start transition
3581  else if(!(command == RunControlStateMachine::HALT_TRANSITION_NAME ||
3582  command == RunControlStateMachine::SHUTDOWN_TRANSITION_NAME ||
3583  command == RunControlStateMachine::ERROR_TRANSITION_NAME ||
3584  command == RunControlStateMachine::FAIL_TRANSITION_NAME ||
3585  command == RunControlStateMachine::STARTUP_TRANSITION_NAME ||
3586  command == RunControlStateMachine::INIT_TRANSITION_NAME ||
3587  command == RunControlStateMachine::ABORT_TRANSITION_NAME ||
3588  command == RunControlStateMachine::PAUSE_TRANSITION_NAME ||
3589  command == RunControlStateMachine::RESUME_TRANSITION_NAME ||
3590  command == RunControlStateMachine::STOP_TRANSITION_NAME))
3591  {
3592  __SS__ << "Error - illegal state machine command received '" << command << ".'"
3593  << __E__;
3594  __SS_THROW__;
3595  }
3596 
3597  theStateMachine_.setErrorMessage(
3598  ""); //clear State Machine error message in prep for transition
3599  xoap::MessageReference message =
3600  SOAPUtilities::makeSOAPMessageReference(command, parameters);
3601  // Maybe we return an acknowledgment that the message has been received and processed
3602  xoap::MessageReference reply = stateMachineXoapHandler(message);
3603  // stateMachineWorkLoopManager_.removeProcessedRequests();
3604  // stateMachineWorkLoopManager_.processRequest(message);
3605 
3606  if(xmldoc)
3607  xmldoc->addTextElementToData("state_tranisition_attempted",
3608  "1"); // indicate to GUI transition attempted
3609  if(out)
3610  xmldoc->outputXmlDocument((std::ostringstream*)out, false);
3611  __COUT__ << "FSM state transition launched!" << __E__;
3612 
3613  stateMachineLastCommandInput_ = command;
3614  return errorStr;
3615 } // end attemptStateMachineTransition()
3616 catch(...)
3617 {
3618  __SS__ << "Error - transition '" << command << "' attempt failed!" << __E__;
3619  try
3620  {
3621  throw;
3622  }
3623  catch(const std::runtime_error& e)
3624  {
3625  ss << "\nHere is the error: " << e.what() << __E__;
3626  }
3627  catch(...)
3628  {
3629  ss << "Uknown error caught." << __E__;
3630  }
3631 
3632  __COUT_ERR__ << "\n" << ss.str();
3633 
3634  if(xmldoc)
3635  xmldoc->addTextElementToData("state_tranisition_attempted",
3636  "0"); // indicate to GUI transition NOT attempted
3637  if(xmldoc)
3638  xmldoc->addTextElementToData(
3639  "state_tranisition_attempted_err",
3640  ss.str()); // indicate to GUI transition NOT attempted
3641  if(out)
3642  xmldoc->outputXmlDocument(
3643  (std::ostringstream*)out, false /*dispStdOut*/, true /*allowWhiteSpace*/);
3644 
3645  return ss.str();
3646 } // end attemptStateMachineTransition() error handling
3647 
3648 //==============================================================================
3649 xoap::MessageReference GatewaySupervisor::stateMachineXoapHandler(
3650  xoap::MessageReference message)
3651 
3652 {
3653  __COUT__ << "FSM Soap Handler!" << __E__;
3654  stateMachineWorkLoopManager_.removeProcessedRequests();
3655  stateMachineWorkLoopManager_.processRequest(message);
3656  __COUT__ << "Done - FSM Soap Handler!" << __E__;
3657  return message;
3658 } // end stateMachineXoapHandler()
3659 
3660 //==============================================================================
3666 bool GatewaySupervisor::stateMachineThread(toolbox::task::WorkLoop* workLoop)
3667 {
3668  stateMachineSemaphore_.take();
3669  std::string command =
3670  SOAPUtilities::translate(stateMachineWorkLoopManager_.getMessage(workLoop))
3671  .getCommand();
3672 
3673  __COUT__ << "Propagating FSM command '" << command << "'..." << __E__;
3674 
3675  std::string reply = send(allSupervisorInfo_.getGatewayDescriptor(),
3676  stateMachineWorkLoopManager_.getMessage(workLoop));
3677  stateMachineWorkLoopManager_.report(workLoop, reply, 100, true);
3678 
3679  __COUT__ << "Done with FSM command '" << command << ".' Reply = " << reply << __E__;
3680  stateMachineSemaphore_.give();
3681 
3682  if(reply == "Fault")
3683  {
3684  __SS__ << "Failure to send Workloop transition command '" << command
3685  << "!' An error response '" << reply << "' was received." << __E__;
3686  __COUT_ERR__ << ss.str();
3687  }
3688  return false; // execute once and automatically remove the workloop so in
3689  // WorkLoopManager the try workLoop->remove(job_) could be commented
3690  // out return true;//go on and then you must do the
3691  // workLoop->remove(job_) in WorkLoopManager
3692 } // end stateMachineThread()
3693 
3694 //==============================================================================
3695 void GatewaySupervisor::stateInitial(toolbox::fsm::FiniteStateMachine& /*fsm*/)
3696 
3697 {
3698  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
3699 
3700 } // end stateInitial()
3701 
3702 //==============================================================================
3703 void GatewaySupervisor::statePaused(toolbox::fsm::FiniteStateMachine& /*fsm*/)
3704 
3705 {
3706  auto pause = std::chrono::system_clock::now();
3707  std::time_t pause_time = std::chrono::system_clock::to_time_t(pause);
3708  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << " at "
3709  << std::ctime(&pause_time) << __E__;
3710 
3711  if(theStateMachine_.getProvenanceStateName() ==
3712  RunControlStateMachine::RUNNING_STATE_NAME)
3713  {
3714  try
3715  {
3716  ConfigurationTree configLinkNode =
3717  CorePropertySupervisorBase::theConfigurationManager_
3718  ->getSupervisorTableNode(supervisorContextUID_,
3719  supervisorApplicationUID_);
3720  if(!configLinkNode.isDisconnected())
3721  {
3722  ConfigurationTree fsmLinkNode =
3723  configLinkNode.getNode("LinkToStateMachineTable")
3724  .getNode(activeStateMachineName_);
3725  std::string runInfoPluginType =
3726  fsmLinkNode.getNode("RunInfoPluginType").getValue<std::string>();
3727  __COUTV__(runInfoPluginType);
3728  if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3729  runInfoPluginType != "No Run Info Plugin")
3730  {
3731  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
3732  try
3733  {
3734  runInfoInterface.reset(
3735  makeRunInfo(runInfoPluginType, activeStateMachineName_));
3736  }
3737  catch(...)
3738  {
3739  }
3740 
3741  if(runInfoInterface == nullptr)
3742  {
3743  __SS__ << "Run Info interface plugin construction failed of type "
3744  << runInfoPluginType << __E__;
3745  __SS_THROW__;
3746  }
3747 
3748  runInfoInterface->updateRunInfo(
3749  getNextRunNumber(activeStateMachineName_) - 1,
3750  RunInfoVInterface::RunStopType::PAUSE);
3751  }
3752  }
3753  }
3754  catch(const std::runtime_error& e)
3755  {
3756  // ERROR
3757  __SS__ << "RUN INFO PAUSE TIME UPDATE INTO DATABASE FAILED!!! " << e.what()
3758  << __E__;
3759  __SS_THROW__;
3760  }
3761  catch(...)
3762  {
3763  // ERROR
3764  __SS__ << "RUN INFO PAUSE TIME UPDATE INTO DATABASE FAILED!!! " << __E__;
3765  try
3766  {
3767  throw;
3768  } //one more try to printout extra info
3769  catch(const std::exception& e)
3770  {
3771  ss << "Exception message: " << e.what();
3772  }
3773  catch(...)
3774  {
3775  }
3776  __SS_THROW__;
3777  } // End update pause time into run info db
3778  } // end update Run Info handling
3779 } // end statePaused()
3780 
3781 //==============================================================================
3782 void GatewaySupervisor::stateRunning(toolbox::fsm::FiniteStateMachine& /*fsm*/)
3783 {
3784  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
3785 
3786  if(theStateMachine_.getProvenanceStateName() ==
3787  RunControlStateMachine::PAUSED_STATE_NAME)
3788  {
3789  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName()
3790  << " coming from resume" << __E__;
3791 
3792  try
3793  {
3794  ConfigurationTree configLinkNode =
3795  CorePropertySupervisorBase::theConfigurationManager_
3796  ->getSupervisorTableNode(supervisorContextUID_,
3797  supervisorApplicationUID_);
3798  if(!configLinkNode.isDisconnected())
3799  {
3800  ConfigurationTree fsmLinkNode =
3801  configLinkNode.getNode("LinkToStateMachineTable")
3802  .getNode(activeStateMachineName_);
3803  std::string runInfoPluginType =
3804  fsmLinkNode.getNode("RunInfoPluginType").getValue<std::string>();
3805  __COUTV__(runInfoPluginType);
3806  if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3807  runInfoPluginType != "No Run Info Plugin")
3808  {
3809  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
3810  try
3811  {
3812  runInfoInterface.reset(
3813  makeRunInfo(runInfoPluginType, activeStateMachineName_));
3814  }
3815  catch(...)
3816  {
3817  }
3818 
3819  if(runInfoInterface == nullptr)
3820  {
3821  __SS__ << "Run Info interface plugin construction failed of type "
3822  << runInfoPluginType << __E__;
3823  __SS_THROW__;
3824  }
3825 
3826  runInfoInterface->updateRunInfo(
3827  getNextRunNumber(activeStateMachineName_) - 1,
3828  RunInfoVInterface::RunStopType::RESUME);
3829  }
3830  }
3831  }
3832  catch(const std::runtime_error& e)
3833  {
3834  // ERROR
3835  __SS__ << "RUN INFO RESUME TIME UPDATE INTO DATABASE FAILED!!! " << e.what()
3836  << __E__;
3837  __SS_THROW__;
3838  }
3839  catch(...)
3840  {
3841  // ERROR
3842  __SS__ << "RUN INFO RESUME TIME UPDATE INTO DATABASE FAILED!!! " << __E__;
3843  try
3844  {
3845  throw;
3846  } //one more try to printout extra info
3847  catch(const std::exception& e)
3848  {
3849  ss << "Exception message: " << e.what();
3850  }
3851  catch(...)
3852  {
3853  }
3854  __SS_THROW__;
3855  } // End update pause time into run info db
3856  } // end update Run Info handling
3857 } // end stateRunning()
3858 
3859 //==============================================================================
3860 void GatewaySupervisor::stateHalted(toolbox::fsm::FiniteStateMachine& /*fsm*/)
3861 {
3862  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName()
3863  << " from " << theStateMachine_.getProvenanceStateName() << __E__;
3864  __COUT__ << "Fsm is in transition? "
3865  << (theStateMachine_.isInTransition() ? "yes" : "no") << __E__;
3866 
3867  __COUTV__(
3868  SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand());
3869 
3870  // if coming from Running or Paused, update Run Info w/HALT
3871  if(theStateMachine_.getProvenanceStateName() ==
3872  RunControlStateMachine::RUNNING_STATE_NAME ||
3873  theStateMachine_.getProvenanceStateName() ==
3874  RunControlStateMachine::PAUSED_STATE_NAME)
3875  {
3876  try
3877  {
3878  ConfigurationTree configLinkNode =
3879  CorePropertySupervisorBase::theConfigurationManager_
3880  ->getSupervisorTableNode(supervisorContextUID_,
3881  supervisorApplicationUID_);
3882  if(!configLinkNode.isDisconnected())
3883  {
3884  ConfigurationTree fsmLinkNode =
3885  configLinkNode.getNode("LinkToStateMachineTable")
3886  .getNode(activeStateMachineName_);
3887  std::string runInfoPluginType =
3888  fsmLinkNode.getNode("RunInfoPluginType").getValue<std::string>();
3889  __COUTV__(runInfoPluginType);
3890  if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3891  runInfoPluginType != "No Run Info Plugin")
3892  {
3893  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
3894  try
3895  {
3896  runInfoInterface.reset(
3897  makeRunInfo(runInfoPluginType, activeStateMachineName_));
3898  // ,
3899  // CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(supervisorContextUID_, supervisorApplicationUID_),
3900  // CorePropertySupervisorBase::getSupervisorConfigurationPath());
3901  }
3902  catch(...)
3903  {
3904  }
3905 
3906  if(runInfoInterface == nullptr)
3907  {
3908  __SS__ << "Run Info interface plugin construction failed of type "
3909  << runInfoPluginType << __E__;
3910  __SS_THROW__;
3911  }
3912 
3913  runInfoInterface->updateRunInfo(
3914  getNextRunNumber(activeStateMachineName_) - 1,
3915  RunInfoVInterface::RunStopType::HALT);
3916  }
3917  }
3918  }
3919  catch(const std::runtime_error& e)
3920  {
3921  // ERROR
3922  __SS__ << "RUN INFO UPDATE INTO DATABASE FAILED!!! " << e.what() << __E__;
3923  __SS_THROW__;
3924  }
3925  catch(...)
3926  {
3927  // ERROR
3928  __SS__ << "RUN INFO UPDATE INTO DATABASE FAILED!!! " << __E__;
3929  try
3930  {
3931  throw;
3932  } //one more try to printout extra info
3933  catch(const std::exception& e)
3934  {
3935  ss << "Exception message: " << e.what();
3936  }
3937  catch(...)
3938  {
3939  }
3940  __SS_THROW__;
3941  } // End write run info into db
3942  } // end update Run Info handling
3943 } // end stateHalted()
3944 
3945 //==============================================================================
3946 void GatewaySupervisor::stateConfigured(toolbox::fsm::FiniteStateMachine& /*fsm*/)
3947 {
3948  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName()
3949  << " from " << theStateMachine_.getProvenanceStateName() << __E__;
3950  __COUT__ << "Fsm is in transition? "
3951  << (theStateMachine_.isInTransition() ? "yes" : "no") << __E__;
3952 
3953  __COUTV__(
3954  SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand());
3955 
3956  // if coming from Running or Paused, update Run Info w/STOP
3957  if(theStateMachine_.getProvenanceStateName() ==
3958  RunControlStateMachine::RUNNING_STATE_NAME ||
3959  theStateMachine_.getProvenanceStateName() ==
3960  RunControlStateMachine::PAUSED_STATE_NAME)
3961  {
3962  try
3963  {
3964  ConfigurationTree configLinkNode =
3965  CorePropertySupervisorBase::theConfigurationManager_
3966  ->getSupervisorTableNode(supervisorContextUID_,
3967  supervisorApplicationUID_);
3968  if(!configLinkNode.isDisconnected())
3969  {
3970  ConfigurationTree fsmLinkNode =
3971  configLinkNode.getNode("LinkToStateMachineTable")
3972  .getNode(activeStateMachineName_);
3973  std::string runInfoPluginType =
3974  fsmLinkNode.getNode("RunInfoPluginType").getValue<std::string>();
3975  __COUTV__(runInfoPluginType);
3976  if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
3977  runInfoPluginType != "No Run Info Plugin")
3978  {
3979  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
3980  try
3981  {
3982  runInfoInterface.reset(
3983  makeRunInfo(runInfoPluginType, activeStateMachineName_));
3984  // ,
3985  // CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(supervisorContextUID_, supervisorApplicationUID_),
3986  // CorePropertySupervisorBase::getSupervisorConfigurationPath());
3987  }
3988  catch(...)
3989  {
3990  }
3991 
3992  if(runInfoInterface == nullptr)
3993  {
3994  __SS__ << "Run Info interface plugin construction failed of type "
3995  << runInfoPluginType << __E__;
3996  __SS_THROW__;
3997  }
3998 
3999  runInfoInterface->updateRunInfo(
4000  getNextRunNumber(activeStateMachineName_) - 1,
4001  RunInfoVInterface::RunStopType::STOP);
4002  }
4003  }
4004  }
4005  catch(const std::runtime_error& e)
4006  {
4007  // ERROR
4008  __SS__ << "RUN INFO INSERT OR UPDATE INTO DATABASE FAILED!!! " << e.what()
4009  << __E__;
4010  __SS_THROW__;
4011  }
4012  catch(...)
4013  {
4014  // ERROR
4015  __SS__ << "RUN INFO INSERT OR UPDATE INTO DATABASE FAILED!!! " << __E__;
4016  try
4017  {
4018  throw;
4019  } //one more try to printout extra info
4020  catch(const std::exception& e)
4021  {
4022  ss << "Exception message: " << e.what();
4023  }
4024  catch(...)
4025  {
4026  }
4027  __SS_THROW__;
4028  } // End write run info into db
4029  } // end update Run Info handling
4030 
4031 } // end stateConfigured()
4032 
4033 //==============================================================================
4034 void GatewaySupervisor::inError(toolbox::fsm::FiniteStateMachine& /*fsm*/)
4035 {
4036  __COUT__ << "Error occured - FSM current state: "
4037  << "Failed? = " << theStateMachine_.getCurrentStateName()
4038  << // There may be a race condition here
4039  // when async errors occur (e.g. immediately in running)
4040  " from " << theStateMachine_.getProvenanceStateName() << __E__;
4041 
4042  __COUT__
4043  << "Error occured on command: "
4044  << (SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand())
4045  << __E__;
4046 
4047  // if coming from Running or Paused, update Run Info w/ERROR
4048  if(theStateMachine_.getProvenanceStateName() ==
4049  RunControlStateMachine::RUNNING_STATE_NAME ||
4050  theStateMachine_.getProvenanceStateName() ==
4051  RunControlStateMachine::PAUSED_STATE_NAME)
4052  {
4053  try
4054  {
4055  ConfigurationTree configLinkNode =
4056  CorePropertySupervisorBase::theConfigurationManager_
4057  ->getSupervisorTableNode(supervisorContextUID_,
4058  supervisorApplicationUID_);
4059  if(!configLinkNode.isDisconnected())
4060  {
4061  ConfigurationTree fsmLinkNode =
4062  configLinkNode.getNode("LinkToStateMachineTable")
4063  .getNode(activeStateMachineName_);
4064  std::string runInfoPluginType =
4065  fsmLinkNode.getNode("RunInfoPluginType").getValue<std::string>();
4066  __COUTV__(runInfoPluginType);
4067  if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
4068  runInfoPluginType != "No Run Info Plugin")
4069  {
4070  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
4071  try
4072  {
4073  runInfoInterface.reset(
4074  makeRunInfo(runInfoPluginType, activeStateMachineName_));
4075  // ,
4076  // CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(supervisorContextUID_, supervisorApplicationUID_),
4077  // CorePropertySupervisorBase::getSupervisorConfigurationPath());
4078  }
4079  catch(...)
4080  {
4081  }
4082 
4083  if(runInfoInterface == nullptr)
4084  {
4085  __SS__ << "Run Info interface plugin construction failed of type "
4086  << runInfoPluginType << __E__;
4087  __SS_THROW__;
4088  }
4089 
4090  runInfoInterface->updateRunInfo(
4091  getNextRunNumber(activeStateMachineName_) - 1,
4092  RunInfoVInterface::RunStopType::ERROR);
4093  }
4094  }
4095  }
4096  catch(const std::runtime_error& e)
4097  {
4098  // ERROR
4099  __SS__ << "RUN INFO INSERT OR UPDATE INTO DATABASE FAILED!!! " << e.what()
4100  << __E__;
4101  __SS_THROW__;
4102  }
4103  catch(...)
4104  {
4105  // ERROR
4106  __SS__ << "RUN INFO INSERT OR UPDATE INTO DATABASE FAILED!!! " << __E__;
4107  try
4108  {
4109  throw;
4110  } //one more try to printout extra info
4111  catch(const std::exception& e)
4112  {
4113  ss << "Exception message: " << e.what();
4114  }
4115  catch(...)
4116  {
4117  }
4118  __SS_THROW__;
4119  } // End write run info into db
4120  } // end update Run Info handling
4121 
4122 } // end inError()
4123 
4124 //==============================================================================
4125 void GatewaySupervisor::enteringError(toolbox::Event::Reference e)
4126 {
4127  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName()
4128  << ", Error event type: " << e->type() << __E__;
4129 
4130  // xdaq 15_14_0_3 broke what() by return c_str() on a temporary string
4131  // https://gitlab.cern.ch/cmsos/core/-/blob/release_15_14_0_3/xcept/src/common/Exception.cc
4132 
4133  // extract error message and save for user interface access
4134  toolbox::fsm::FailedEvent& failedEvent = dynamic_cast<toolbox::fsm::FailedEvent&>(*e);
4135  xcept::Exception& failedException = failedEvent.getException();
4136  //__COUT__ << "History of errors: " << failedException.size() << __E__;
4137  //__COUT__ << "Failed Message: " << failedException.rbegin()->at("message") << __E__;
4138  //__COUT__ << "Failed Message: " << failedException.message() << __E__;
4139  //__COUT__ << "Failed Message: " << failedException.what() << __E__;
4140 
4141  bool asyncFailureIdentified = false;
4142  __SS__;
4143  // handle async error message differently
4144  if(RunControlStateMachine::asyncFailureReceived_)
4145  {
4146  ss << "\nAn asynchronous failure was encountered."
4147  << ".\n\nException:\n"
4148  << failedException.message() << __E__; // rbegin()->at("message") << __E__;
4149  //<< failedEvent.getException().what() << __E__;
4150  RunControlStateMachine::asyncFailureReceived_ = false; // clear async error
4151  asyncFailureIdentified = true;
4152  }
4153  else
4154  {
4155  ss << "\nFailure performing transition from " << failedEvent.getFromState() << "-"
4156  << theStateMachine_.getStateName(failedEvent.getFromState()) << " to "
4157  << failedEvent.getToState() << "-"
4158  << theStateMachine_.getStateName(failedEvent.getToState())
4159  << ".\n\nException:\n"
4160  << failedException.message() << __E__; // rbegin()->at("message") << __E__;
4161  //<< failedEvent.getException().what() << __E__;
4162  }
4163 
4164  __COUT_ERR__ << "\n" << ss.str();
4165 
4166  theStateMachine_.setErrorMessage(ss.str());
4167 
4168  if(!asyncFailureIdentified && theStateMachine_.getCurrentStateName() ==
4169  RunControlStateMachine::FAILED_STATE_NAME)
4170  __COUT__ << "Already in failed state, so not broadcasting Error transition again."
4171  << __E__;
4172  else // move everything else to Error!
4173  broadcastMessage(SOAPUtilities::makeSOAPMessageReference(
4174  RunControlStateMachine::ERROR_TRANSITION_NAME));
4175 } // end enteringError()
4176 
4177 //==============================================================================
4178 void GatewaySupervisor::checkForAsyncError()
4179 {
4180  if(RunControlStateMachine::asyncFailureReceived_)
4181  {
4182  __COUTV__(RunControlStateMachine::asyncFailureReceived_);
4183 
4184  XCEPT_RAISE(toolbox::fsm::exception::Exception,
4185  RunControlStateMachine::getErrorMessage());
4186  return;
4187  }
4188 } // end checkForAsyncError()
4189 
4193 
4194 //==============================================================================
4195 void GatewaySupervisor::transitionConfiguring(toolbox::Event::Reference /* e*/)
4196 try
4197 {
4198  checkForAsyncError();
4199 
4200  RunControlStateMachine::theProgressBar_.step();
4201 
4202  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
4203 
4204  std::string configurationAlias =
4205  SOAPUtilities::translate(theStateMachine_.getCurrentMessage())
4206  .getParameters()
4207  .getValue("ConfigurationAlias");
4208 
4209  __COUT__ << "Transition parameter ConfigurationAlias: " << configurationAlias
4210  << __E__;
4211 
4212  RunControlStateMachine::theProgressBar_.step();
4213 
4214  __COUT__ << "Configuration table group name: " << theConfigurationTableGroup_.first
4215  << " key: " << theConfigurationTableGroup_.second << __E__;
4216 
4217  //Roll over log file if enabled
4218  if(activeStateMachineRollOverLogOnConfigure_)
4219  {
4220  __COUT_INFO__ << "Rolling over log file on Configure transition..." << __E__;
4221  std::stringstream runSs;
4222  runSs << "LOG_ROLLOVER";
4223  runSs << ";"
4224  << "Configure"
4225  << "_" << theConfigurationTableGroup_.first << "_v"
4226  << theConfigurationTableGroup_.second;
4227 
4228  GatewaySupervisor::launchStartOTSCommand(
4229  runSs.str(), CorePropertySupervisorBase::theConfigurationManager_);
4230  }
4231 
4232  RunControlStateMachine::theProgressBar_.step();
4233 
4234  // make logbook entry
4235  bool doLog = false;
4236  try
4237  {
4238  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
4239  }
4240  catch(...)
4241  { /* ignore errors */
4242  ;
4243  }
4244  if(doLog)
4245  {
4246  std::stringstream ss;
4247  ss << "Configuring with System Configuration Alias '" << configurationAlias
4248  << "' which translates to " << theConfigurationTableGroup_.first << "("
4249  << theConfigurationTableGroup_.second << "). Active Context Group "
4250  << CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupName(
4251  ConfigurationManager::GroupType::CONTEXT_TYPE)
4252  << "("
4253  << CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
4254  ConfigurationManager::GroupType::CONTEXT_TYPE)
4255  << ").";
4256 
4257  if(getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME) != "")
4258  ss << "\n\n-----------------\nUser log entry:\n"
4259  << getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
4260  << "\n-----------------\n";
4261  else
4262  ss << " No user log entry.";
4263  makeSystemLogEntry(ss.str());
4264  } // end make logbook entry
4265 
4266  RunControlStateMachine::theProgressBar_.step();
4267 
4268  try
4269  {
4270  CorePropertySupervisorBase::theConfigurationManager_->dumpMacroMakerModeFhicl();
4271  }
4272  catch(...) // ignore error for now
4273  {
4274  __COUT_ERR__ << "Failed to dump MacroMaker mode fhicl." << __E__;
4275  }
4276 
4277  RunControlStateMachine::theProgressBar_.step();
4278  SOAPParameters parameters;
4279  parameters.addParameter("ConfigurationTableGroupName",
4280  theConfigurationTableGroup_.first);
4281  parameters.addParameter("ConfigurationTableGroupKey",
4282  theConfigurationTableGroup_.second.toString());
4283 
4284  // update Macro Maker front end list
4285  if(CorePropertySupervisorBase::allSupervisorInfo_.getAllMacroMakerTypeSupervisorInfo()
4286  .size())
4287  {
4288  __COUT__ << "Initializing Macro Maker." << __E__;
4289  xoap::MessageReference message =
4290  SOAPUtilities::makeSOAPMessageReference("FECommunication");
4291 
4292  SOAPParameters parameters;
4293  parameters.addParameter("type", "initFElist");
4294  parameters.addParameter("groupName", theConfigurationTableGroup_.first);
4295  parameters.addParameter("groupKey",
4296  theConfigurationTableGroup_.second.toString());
4297  SOAPUtilities::addParameters(message, parameters);
4298 
4299  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
4300  << __E__;
4301 
4302  std::string reply =
4303  SOAPMessenger::send(CorePropertySupervisorBase::allSupervisorInfo_
4304  .getAllMacroMakerTypeSupervisorInfo()
4305  .begin()
4306  ->second.getDescriptor(),
4307  message);
4308 
4309  __COUT__ << "Macro Maker init reply: " << reply << __E__;
4310  if(reply == "Error")
4311  {
4312  __SS__ << "\nTransition to Configuring interrupted! There was an error "
4313  "identified initializing Macro Maker.\n\n "
4314  << __E__;
4315  __COUT_ERR__ << "\n" << ss.str();
4316  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4317  return;
4318  }
4319  } // end update Macro Maker front end list
4320  RunControlStateMachine::theProgressBar_.step();
4321 
4322  xoap::MessageReference message = theStateMachine_.getCurrentMessage();
4323  SOAPUtilities::addParameters(message, parameters);
4324  //Note: Must save configuration dump after this point!! In case there are remote subsystems responding with string
4325  broadcastMessage(message); // ---------------------------------- broadcast!
4326  RunControlStateMachine::theProgressBar_.step();
4327 
4328  //check for remote subsystem dumps (after broadcast!)
4329  std::string remoteSubsystemDump = "";
4330  {
4331  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGatewayApps; //local copy
4332  { //lock for remainder of scope
4333  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
4334  __SUP_COUTVS__(22, remoteGatewayApps_.size());
4335  remoteGatewayApps = remoteGatewayApps_;
4336  if(remoteGatewayApps_.size())
4337  __SUP_COUT_TYPE__(TLVL_DEBUG + 22)
4338  << __COUT_HDR__ << remoteGatewayApps_[0].command << " "
4339  << (remoteGatewayApps_[0].appInfo.status) << __E__;
4340  }
4341  for(auto& remoteGatewayApp : remoteGatewayApps)
4342  {
4343  if(!remoteGatewayApp.fsm_included)
4344  continue; //skip if not included
4345  remoteSubsystemDump += remoteGatewayApp.config_dump;
4346  }
4347 
4348  if(remoteSubsystemDump.size())
4349  __COUTV__(remoteSubsystemDump);
4350  } //end check for remote subsystem dumps
4351  RunControlStateMachine::theProgressBar_.step();
4352 
4353  if(activeStateMachineConfigurationDumpOnConfigureEnable_)
4354  {
4355  //write local configuration dump file
4356  std::string fullfilename =
4357  activeStateMachineConfigurationDumpOnConfigureFilename_ + "_" +
4358  std::to_string(time(0)) + ".dump";
4359  FILE* fp = fopen(fullfilename.c_str(), "w");
4360  if(!fp)
4361  {
4362  __SS__ << "Configuration dump failed to file: " << fullfilename << __E__;
4363  __SS_THROW__;
4364  }
4365 
4366  //(a la ConfigurationManager::dumpActiveConfiguration)
4367  fullfilename = __ENV__("HOSTNAME") + std::string(":") + fullfilename;
4368  fprintf(
4369  fp, "Original location of dump: %s\n", fullfilename.c_str());
4370 
4371  if(activeStateMachineConfigurationDumpOnConfigure_.size())
4372  fwrite(&activeStateMachineConfigurationDumpOnConfigure_[0],
4373  1,
4374  activeStateMachineConfigurationDumpOnConfigure_.size(),
4375  fp);
4376  __COUT__ << "Wrote configuration dump of char count "
4377  << activeStateMachineConfigurationDumpOnConfigure_.size()
4378  << " to file: " << fullfilename << __E__;
4379 
4380  if(remoteSubsystemDump.size())
4381  {
4382  fwrite(&remoteSubsystemDump[0], 1, remoteSubsystemDump.size(), fp);
4383 
4384  __COUT__ << "Wrote remote subsystem configuration dump of char count "
4385  << remoteSubsystemDump.size() << " to file: " << fullfilename
4386  << __E__;
4387  }
4388  fclose(fp);
4389 
4390  __COUT_INFO__ << "Configure transition Configuration Dump saved to file: "
4391  << fullfilename << __E__;
4392  } //done with local config dump
4393  RunControlStateMachine::theProgressBar_.step();
4394 
4395  // Check if Run Plugin is defined and, if so, create a new condition record into database
4396  // leave as repeated code in case dumpFormat is different for Run Plugin (in the future)
4397  try
4398  {
4399  if(activeStateMachineRunInfoPluginType_ !=
4400  TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
4401  activeStateMachineRunInfoPluginType_ != "No Run Info Plugin")
4402  {
4403  __COUT_INFO__ << "Instantiating Run Info plugin '"
4404  << activeStateMachineRunInfoPluginType_
4405  << "' to insert Configure run condition entry." << __E__;
4406  std::unique_ptr<RunInfoVInterface> runInfoInterface = nullptr;
4407  try
4408  {
4409  runInfoInterface.reset(makeRunInfo(activeStateMachineRunInfoPluginType_,
4410  activeStateMachineName_));
4411  }
4412  catch(...)
4413  {
4414  ;
4415  }
4416  if(runInfoInterface == nullptr)
4417  {
4418  __SS__ << "Run Info interface plugin construction failed of type "
4419  << activeStateMachineRunInfoPluginType_
4420  << " for inserting Run Condition record of char size "
4421  << activeStateMachineConfigurationDumpOnConfigure_.size() << __E__;
4422  __SS_THROW__;
4423  }
4424 
4425  conditionID_ = runInfoInterface->insertRunCondition(
4426  activeStateMachineConfigurationDumpOnConfigure_ + remoteSubsystemDump);
4427  } // end Run Info Plugin handling
4428  }
4429  catch(const std::runtime_error& e)
4430  {
4431  __SS__ << "RUN CONDITION INSERT INTO DATABASE FAILED!!! " << e.what() << __E__;
4432  __SS_THROW__;
4433  }
4434  catch(...)
4435  {
4436  __SS__ << "RUN CONDITION INSERT INTO DATABASE FAILED!!! " << __E__;
4437  try
4438  {
4439  throw;
4440  } //one more try to printout extra info
4441  catch(const std::exception& e)
4442  {
4443  ss << "Exception message: " << e.what();
4444  }
4445  catch(...)
4446  {
4447  }
4448  __SS_THROW__;
4449  } // End write run condition into db
4450  RunControlStateMachine::theProgressBar_.step();
4451 
4452  // save last configured group name/key
4453  ConfigurationManager::saveGroupNameAndKey(theConfigurationTableGroup_,
4454  FSM_LAST_CONFIGURED_GROUP_ALIAS_FILE);
4455 
4456  activeStateMachineConfigurationAlias_ = configurationAlias;
4457  doLog = true; //default to true
4458  try
4459  {
4460  doLog = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
4461  }
4462  catch(...)
4463  { /* ignore errors */
4464  ;
4465  }
4466  if(doLog)
4467  {
4468  std::stringstream ss;
4469  ss << "Configured with System Configuration Alias '"
4470  << activeStateMachineConfigurationAlias_ << "' which translates to "
4471  << theConfigurationTableGroup_.first << "("
4472  << theConfigurationTableGroup_.second << "). Active Context Group "
4473  << CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupName(
4474  ConfigurationManager::GroupType::CONTEXT_TYPE)
4475  << "("
4476  << CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
4477  ConfigurationManager::GroupType::CONTEXT_TYPE)
4478  << ").";
4479 
4480  if(getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME) != "")
4481  ss << "\n\n-----------------\nUser log entry:\n"
4482  << getLastLogEntry(RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
4483  << "\n-----------------\n";
4484  else
4485  ss << " No user log entry.";
4486 
4487  //insert system and remote subsystem status/detail
4488  {
4489  ss << "\n\n~~~ System Status and Detail ~~~\n";
4490  for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
4491  {
4492  const auto& appInfo = it.second;
4493  if(appInfo.getClass() != XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
4494  continue; //only give Gateway status
4495  ss << "\tStatus: " << appInfo.getStatus() << __E__
4496  << "\tDetail: " << appInfo.getDetail() << __E__;
4497  }
4498 
4499  //also return remote gateways as apps
4500  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteApps; //local copy
4501  { //lock for remainder of scope
4502  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
4503  remoteApps = remoteGatewayApps_;
4504  }
4505 
4506  if(remoteApps.size())
4507  {
4508  ss << "\n\n~~~ Subsystem Status and Detail ~~~\n";
4509 
4510  for(const auto& remoteApp : remoteApps)
4511  {
4512  const auto& appInfo = remoteApp.appInfo;
4513  ss << "Subsystem Name: " << appInfo.name << __E__
4514  << "\tStatus: " << appInfo.status << __E__
4515  << "\tDetail: " << appInfo.detail << __E__;
4516  }
4517  }
4518  }
4519 
4520  makeSystemLogEntry(ss.str());
4521  }
4522  __COUT__ << "Done configuring." << __E__;
4523  RunControlStateMachine::theProgressBar_.complete();
4524 } // end transitionConfiguring()
4525 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4526 {
4527  __SS__ << "\nTransition to Configuring interrupted! There was a system communication "
4528  "error "
4529  "identified. "
4530  << __E__ << e.what();
4531  __COUT_ERR__ << "\n" << ss.str();
4532  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4533 }
4534 catch(std::runtime_error& e)
4535 {
4536  __SS__ << "\nTransition to Configuring interrupted! There was an error "
4537  "identified. "
4538  << __E__ << e.what();
4539  __COUT_ERR__ << "\n" << ss.str();
4540  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4541 }
4542 catch(toolbox::fsm::exception::Exception& e)
4543 {
4544  throw; // just rethrow exceptions of already the correct type
4545 }
4546 catch(...)
4547 {
4548  __SS__ << "\nTransition to Configuring interrupted! There was an unknown error "
4549  "identified. "
4550  << __E__;
4551  try
4552  {
4553  throw;
4554  } //one more try to printout extra info
4555  catch(const std::exception& e)
4556  {
4557  ss << "Exception message: " << e.what();
4558  }
4559  catch(...)
4560  {
4561  }
4562  __COUT_ERR__ << "\n" << ss.str();
4563  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4564 } // end transitionConfiguring() catch
4565 
4566 //==============================================================================
4567 void GatewaySupervisor::transitionHalting(toolbox::Event::Reference /*e*/)
4568 try
4569 {
4570  checkForAsyncError();
4571 
4572  RunControlStateMachine::theProgressBar_.step();
4573 
4574  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
4575 
4576  bool doLog = false;
4577  try
4578  {
4579  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
4580  }
4581  catch(...)
4582  { /* ignore errors */
4583  ;
4584  }
4585  bool doLogIntermediate = false;
4586  try
4587  {
4588  doLogIntermediate = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
4589  }
4590  catch(...)
4591  { /* ignore errors */
4592  ;
4593  }
4594 
4595  if(doLog && doLogIntermediate)
4596  makeSystemLogEntry("System halting.");
4597 
4598  RunControlStateMachine::theProgressBar_.step();
4599 
4600  broadcastMessage(theStateMachine_.getCurrentMessage());
4601 
4602  if(doLogIntermediate)
4603  makeSystemLogEntry("System halted.");
4604  __COUT__ << "Done halting." << __E__;
4605  RunControlStateMachine::theProgressBar_.complete();
4606 } // end transitionHalting()
4607 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4608 {
4609  __SS__
4610  << "\nTransition to Halting interrupted! There was a system communication error "
4611  "identified. "
4612  << __E__ << e.what();
4613  __COUT_ERR__ << "\n" << ss.str();
4614  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4615 }
4616 catch(std::runtime_error& e)
4617 {
4618  __SS__ << "\nTransition to Halting interrupted! There was an error "
4619  "identified. "
4620  << __E__ << e.what();
4621  __COUT_ERR__ << "\n" << ss.str();
4622  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4623 }
4624 catch(toolbox::fsm::exception::Exception& e)
4625 {
4626  throw; // just rethrow exceptions of already the correct type
4627 }
4628 catch(...)
4629 {
4630  __SS__ << "\nTransition to Halting interrupted! There was an unknown error "
4631  "identified. "
4632  << __E__;
4633  try
4634  {
4635  throw;
4636  } //one more try to printout extra info
4637  catch(const std::exception& e)
4638  {
4639  ss << "Exception message: " << e.what();
4640  }
4641  catch(...)
4642  {
4643  }
4644  __COUT_ERR__ << "\n" << ss.str();
4645  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4646 } // end transitionHalting() catch
4647 
4648 //==============================================================================
4649 void GatewaySupervisor::transitionShuttingDown(toolbox::Event::Reference /*e*/)
4650 try
4651 {
4652  checkForAsyncError();
4653 
4654  __COUT__ << "transitionShuttingDown -- Fsm current state: "
4655  << theStateMachine_.getCurrentStateName()
4656  << " message: " << theStateMachine_.getCurrentStateName() << __E__;
4657 
4658  RunControlStateMachine::theProgressBar_.step();
4659  bool doLog = false;
4660  try
4661  {
4662  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
4663  }
4664  catch(...)
4665  { /* ignore errors */
4666  ;
4667  }
4668  bool doLogIntermediate = false;
4669  try
4670  {
4671  doLogIntermediate = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
4672  }
4673  catch(...)
4674  { /* ignore errors */
4675  ;
4676  }
4677 
4678  if(doLog && doLogIntermediate)
4679  makeSystemLogEntry("System shutting down.");
4680  RunControlStateMachine::theProgressBar_.step();
4681 
4682  // kill all non-gateway contexts
4683  GatewaySupervisor::launchStartOTSCommand(
4684  "OTS_APP_SHUTDOWN", CorePropertySupervisorBase::theConfigurationManager_);
4685  RunControlStateMachine::theProgressBar_.step();
4686 
4687  // important to give time for StartOTS script to recognize command (before user does
4688  // Startup again)
4689  for(int i = 0; i < 5; ++i)
4690  {
4691  sleep(1);
4692  RunControlStateMachine::theProgressBar_.step();
4693  }
4694 
4695  broadcastMessage(theStateMachine_.getCurrentMessage());
4696 
4697  if(doLogIntermediate)
4698  makeSystemLogEntry("System shutdown complete.");
4699  __COUT__ << "Done shutting down." << __E__;
4700  RunControlStateMachine::theProgressBar_.complete();
4701 } // end transitionShuttingDown()
4702 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4703 {
4704  __SS__ << "\nTransition to Shutting Down interrupted! There was a system "
4705  "communication error "
4706  "identified. "
4707  << __E__ << e.what();
4708  __COUT_ERR__ << "\n" << ss.str();
4709  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4710 }
4711 catch(std::runtime_error& e)
4712 {
4713  __SS__ << "\nTransition to Shutting Down interrupted! There was an error "
4714  "identified. "
4715  << __E__ << e.what();
4716  __COUT_ERR__ << "\n" << ss.str();
4717  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4718 }
4719 catch(toolbox::fsm::exception::Exception& e)
4720 {
4721  throw; // just rethrow exceptions of already the correct type
4722 }
4723 catch(...)
4724 {
4725  __SS__ << "\nTransition to Shutting Down interrupted! There was an unknown error "
4726  "identified. "
4727  << __E__;
4728  try
4729  {
4730  throw;
4731  } //one more try to printout extra info
4732  catch(const std::exception& e)
4733  {
4734  ss << "Exception message: " << e.what();
4735  }
4736  catch(...)
4737  {
4738  }
4739  __COUT_ERR__ << "\n" << ss.str();
4740  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4741 } // end transitionShuttingDown() catch
4742 
4743 //==============================================================================
4744 void GatewaySupervisor::transitionStartingUp(toolbox::Event::Reference /*e*/)
4745 try
4746 {
4747  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
4748 
4749  RunControlStateMachine::theProgressBar_.step();
4750  bool doLog = false;
4751  try
4752  {
4753  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
4754  }
4755  catch(...)
4756  { /* ignore errors */
4757  ;
4758  }
4759  bool doLogIntermediate = false;
4760  try
4761  {
4762  doLogIntermediate = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
4763  }
4764  catch(...)
4765  { /* ignore errors */
4766  ;
4767  }
4768 
4769  if(doLog && doLogIntermediate)
4770  makeSystemLogEntry("System starting up.");
4771  RunControlStateMachine::theProgressBar_.step();
4772 
4773  // start all non-gateway contexts
4774  GatewaySupervisor::launchStartOTSCommand(
4775  "OTS_APP_STARTUP", CorePropertySupervisorBase::theConfigurationManager_);
4776  RunControlStateMachine::theProgressBar_.step();
4777 
4778  // important to give time for StartOTS script to recognize command and for apps to
4779  // instantiate things (before user does Initialize)
4780  for(int i = 0; i < 10; ++i)
4781  {
4782  sleep(1);
4783  RunControlStateMachine::theProgressBar_.step();
4784  }
4785 
4786  broadcastMessage(theStateMachine_.getCurrentMessage());
4787 
4788  if(doLogIntermediate)
4789  makeSystemLogEntry("System startup complete.");
4790  __COUT__ << "Done starting up." << __E__;
4791  RunControlStateMachine::theProgressBar_.complete();
4792 
4793 } // end transitionStartingUp()
4794 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4795 {
4796  __SS__ << "\nTransition to Starting Up interrupted! There was a system communication "
4797  "error "
4798  "identified. "
4799  << __E__ << e.what();
4800  __COUT_ERR__ << "\n" << ss.str();
4801  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4802 }
4803 catch(std::runtime_error& e)
4804 {
4805  __SS__ << "\nTransition to Starting Up interrupted! There was an error "
4806  "identified. "
4807  << __E__ << e.what();
4808  __COUT_ERR__ << "\n" << ss.str();
4809  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4810 }
4811 catch(toolbox::fsm::exception::Exception& e)
4812 {
4813  throw; // just rethrow exceptions of already the correct type
4814 }
4815 catch(...)
4816 {
4817  __SS__ << "\nTransition to Starting Up interrupted! There was an unknown error "
4818  "identified. "
4819  << __E__;
4820  try
4821  {
4822  throw;
4823  } //one more try to printout extra info
4824  catch(const std::exception& e)
4825  {
4826  ss << "Exception message: " << e.what();
4827  }
4828  catch(...)
4829  {
4830  }
4831  __COUT_ERR__ << "\n" << ss.str();
4832  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4833 } // end transitionStartingUp() catch
4834 
4835 //==============================================================================
4836 void GatewaySupervisor::transitionInitializing(toolbox::Event::Reference event)
4837 try
4838 {
4839  __COUT__ << theStateMachine_.getCurrentStateName() << __E__;
4840 
4841  broadcastMessage(theStateMachine_.getCurrentMessage());
4842 
4843  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
4844  __COUT__ << "Fsm current transition: "
4845  << theStateMachine_.getCurrentTransitionName(event->type()) << __E__;
4846  __COUT__ << "Fsm final state: "
4847  << theStateMachine_.getTransitionFinalStateName(event->type()) << __E__;
4848 
4849  bool doLog = false;
4850  try
4851  {
4852  doLog = __ENV__("OTS_LOG_INTERMEDIATE_STATES") == std::string("1");
4853  }
4854  catch(...)
4855  { /* ignore errors */
4856  ;
4857  }
4858  if(doLog)
4859  makeSystemLogEntry("System initialized.");
4860 
4861  __COUT__ << "Done initializing." << __E__;
4862  RunControlStateMachine::theProgressBar_.complete();
4863 
4864 } // end transitionInitializing()
4865 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4866 {
4867  __SS__ << "\nTransition to Initializing interrupted! There was a system "
4868  "communication error "
4869  "identified. "
4870  << __E__ << e.what();
4871  __COUT_ERR__ << "\n" << ss.str();
4872  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4873 }
4874 catch(std::runtime_error& e)
4875 {
4876  __SS__ << "\nTransition to Initializing interrupted! There was an error "
4877  "identified. "
4878  << __E__ << e.what();
4879  __COUT_ERR__ << "\n" << ss.str();
4880  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4881 }
4882 catch(toolbox::fsm::exception::Exception& e)
4883 {
4884  throw; // just rethrow exceptions of already the correct type
4885 }
4886 catch(...)
4887 {
4888  __SS__ << "\nTransition to Initializing interrupted! There was an unknown error "
4889  "identified. "
4890  << __E__;
4891  try
4892  {
4893  throw;
4894  } //one more try to printout extra info
4895  catch(const std::exception& e)
4896  {
4897  ss << "Exception message: " << e.what();
4898  }
4899  catch(...)
4900  {
4901  }
4902  __COUT_ERR__ << "\n" << ss.str();
4903  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4904 } // end transitionInitializing() catch
4905 
4906 //==============================================================================
4907 void GatewaySupervisor::transitionPausing(toolbox::Event::Reference /*e*/)
4908 try
4909 {
4910  checkForAsyncError();
4911 
4912  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
4913 
4914  RunControlStateMachine::theProgressBar_.step();
4915 
4916  // calculate run duration and post system log entry
4917  bool doLog = false;
4918  try
4919  {
4920  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
4921  }
4922  catch(...)
4923  { /* ignore errors */
4924  ;
4925  }
4926 
4927  std::ostringstream dur_ss;
4928  {
4929  int dur = std::chrono::duration_cast<std::chrono::milliseconds>(
4930  std::chrono::steady_clock::now() - activeStateMachineRunStartTime)
4931  .count() +
4932  activeStateMachineRunDuration_ms;
4933  int dur_s = dur / 1000;
4934  dur = dur % 1000;
4935  int dur_m = dur_s / 60;
4936  dur_s = dur_s % 60;
4937  int dur_h = dur_m / 60;
4938  dur_m = dur_m % 60;
4939  dur_ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
4940  << "' duration so far of " << std::setw(2) << std::setfill('0') << dur_h
4941  << ":" << std::setw(2) << std::setfill('0') << dur_m << ":" << std::setw(2)
4942  << std::setfill('0')
4943  << dur_s; //too much detail "." << dur << " seconds.";
4944  if(dur_h == 0 && dur_m == 0 && dur_s < 5) //if very short, add the detail
4945  dur_ss << "." << dur << " seconds.";
4946  else
4947  dur_ss << ".";
4948 
4949  if(doLog)
4950  makeSystemLogEntry("Run pausing. " + dur_ss.str());
4951  }
4952 
4953  activeStateMachineRunDuration_ms +=
4954  std::chrono::duration_cast<std::chrono::milliseconds>(
4955  std::chrono::steady_clock::now() - activeStateMachineRunStartTime)
4956  .count();
4957 
4958  // the current message is not for Pause if its due to async exception, so rename
4959  if(RunControlStateMachine::asyncPauseExceptionReceived_)
4960  {
4961  __COUT_ERR__ << "Broadcasting pause for async PAUSE exception!" << __E__;
4962  broadcastMessage(SOAPUtilities::makeSOAPMessageReference("Pause"));
4963  }
4964  else
4965  broadcastMessage(theStateMachine_.getCurrentMessage());
4966 
4967  makeSystemLogEntry(
4968  "Run paused. " + dur_ss.str(),
4969  activeStateMachineRunAlias_ + " '" + activeStateMachineRunNumber_ + "' paused");
4970  __COUT__ << "Done pausing." << __E__;
4971  RunControlStateMachine::theProgressBar_.complete();
4972 
4973 } // end transitionPausing()
4974 catch(const xdaq::exception::Exception& e) // due to xoap send failure
4975 {
4976  __SS__
4977  << "\nTransition to Pausing interrupted! There was a system communication error "
4978  "identified. "
4979  << __E__ << e.what();
4980  __COUT_ERR__ << "\n" << ss.str();
4981  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4982 }
4983 catch(std::runtime_error& e)
4984 {
4985  __SS__ << "\nTransition to Pausing interrupted! There was an error "
4986  "identified. "
4987  << __E__ << e.what();
4988  __COUT_ERR__ << "\n" << ss.str();
4989  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
4990 }
4991 catch(toolbox::fsm::exception::Exception& e)
4992 {
4993  throw; // just rethrow exceptions of already the correct type
4994 }
4995 catch(...)
4996 {
4997  __SS__ << "\nTransition to Pausing interrupted! There was an unknown error "
4998  "identified. "
4999  << __E__;
5000  try
5001  {
5002  throw;
5003  } //one more try to printout extra info
5004  catch(const std::exception& e)
5005  {
5006  ss << "Exception message: " << e.what();
5007  }
5008  catch(...)
5009  {
5010  }
5011  __COUT_ERR__ << "\n" << ss.str();
5012  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5013 } // end transitionPausing() catch
5014 
5015 //==============================================================================
5016 void GatewaySupervisor::transitionResuming(toolbox::Event::Reference /*e*/)
5017 try
5018 {
5019  if(RunControlStateMachine::asyncPauseExceptionReceived_)
5020  {
5021  // clear async pause error
5022  __COUT_INFO__ << "Clearing async PAUSE exception!" << __E__;
5023  RunControlStateMachine::asyncPauseExceptionReceived_ = false;
5024  }
5025  else if(RunControlStateMachine::asyncStopExceptionReceived_)
5026  {
5027  // clear async stop error
5028  __COUT_INFO__ << "Clearing async STOP exception!" << __E__;
5029  RunControlStateMachine::asyncStopExceptionReceived_ = false;
5030  }
5031 
5032  checkForAsyncError();
5033 
5034  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
5035 
5036  bool doLog = false;
5037  try
5038  {
5039  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
5040  }
5041  catch(...)
5042  { /* ignore errors */
5043  ;
5044  }
5045  if(doLog)
5046  {
5047  std::stringstream ss;
5048  ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
5049  << "' resuming.";
5050 
5051  if(getLastLogEntry(RunControlStateMachine::RESUME_TRANSITION_NAME) != "")
5052  ss << "\n\n-----------------\nUser log entry:\n"
5053  << getLastLogEntry(RunControlStateMachine::RESUME_TRANSITION_NAME)
5054  << "\n-----------------\n";
5055  else
5056  ss << " No user log entry.";
5057 
5058  makeSystemLogEntry(ss.str());
5059  } //end make logbook entry
5060 
5061  activeStateMachineRunStartTime = std::chrono::steady_clock::now();
5062 
5063  broadcastMessage(theStateMachine_.getCurrentMessage());
5064 
5065  // make logbook entry
5066  {
5067  std::stringstream ss;
5068  ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
5069  << "' resumed.";
5070 
5071  if(getLastLogEntry(RunControlStateMachine::RESUME_TRANSITION_NAME) != "")
5072  ss << "\n\n-----------------\nUser log entry:\n"
5073  << getLastLogEntry(RunControlStateMachine::RESUME_TRANSITION_NAME)
5074  << "\n-----------------\n";
5075  else
5076  ss << " No user log entry.";
5077 
5078  makeSystemLogEntry(ss.str(),
5079  activeStateMachineRunAlias_ + " '" +
5080  activeStateMachineRunNumber_ + "' resumed");
5081  } // end make logbook entry
5082 
5083  __COUT__ << "Done resuming." << __E__;
5084  RunControlStateMachine::theProgressBar_.complete();
5085 } // end transitionResuming()
5086 catch(const xdaq::exception::Exception& e) // due to xoap send failure
5087 {
5088  __SS__
5089  << "\nTransition to Resuming interrupted! There was a system communication error "
5090  "identified. "
5091  << __E__ << e.what();
5092  __COUT_ERR__ << "\n" << ss.str();
5093  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5094 }
5095 catch(std::runtime_error& e)
5096 {
5097  __SS__ << "\nTransition to Resuming interrupted! There was an error "
5098  "identified. "
5099  << __E__ << e.what();
5100  __COUT_ERR__ << "\n" << ss.str();
5101  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5102 }
5103 catch(toolbox::fsm::exception::Exception& e)
5104 {
5105  throw; // just rethrow exceptions of already the correct type
5106 }
5107 catch(...)
5108 {
5109  __SS__ << "\nTransition to Resuming interrupted! There was an unknown error "
5110  "identified. "
5111  << __E__;
5112  try
5113  {
5114  throw;
5115  } //one more try to printout extra info
5116  catch(const std::exception& e)
5117  {
5118  ss << "Exception message: " << e.what();
5119  }
5120  catch(...)
5121  {
5122  }
5123  __COUT_ERR__ << "\n" << ss.str();
5124  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5125 } // end transitionResuming() catch
5126 
5127 //==============================================================================
5128 void GatewaySupervisor::transitionStarting(toolbox::Event::Reference /*e*/)
5129 try
5130 {
5131  if(RunControlStateMachine::asyncPauseExceptionReceived_)
5132  {
5133  // clear async soft error
5134  __COUT_INFO__ << "Clearing async PAUSE exception!" << __E__;
5135  RunControlStateMachine::asyncPauseExceptionReceived_ = false;
5136  }
5137  else if(RunControlStateMachine::asyncStopExceptionReceived_)
5138  {
5139  // clear async stop error
5140  __COUT_INFO__ << "Clearing async STOP exception!" << __E__;
5141  RunControlStateMachine::asyncStopExceptionReceived_ = false;
5142  }
5143 
5144  checkForAsyncError();
5145 
5146  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
5147 
5148  RunControlStateMachine::theProgressBar_.step();
5149 
5150  SOAPParameters parameters("RunNumber");
5151  SOAPUtilities::receive(theStateMachine_.getCurrentMessage(), parameters);
5152 
5153  activeStateMachineRunNumber_ = parameters.getValue("RunNumber");
5154  __COUTV__(activeStateMachineRunNumber_);
5155 
5156  RunControlStateMachine::theProgressBar_.step();
5157 
5158  //Roll over log file if enabled
5159  if(activeStateMachineRollOverLogOnStart_)
5160  {
5161  __COUT_INFO__ << "Rolling over log file on Start transition..." << __E__;
5162  std::stringstream runSs;
5163  runSs << "LOG_ROLLOVER";
5164  runSs << ";" << activeStateMachineRunAlias_ << "_"
5165  << activeStateMachineRunNumber_;
5166 
5167  GatewaySupervisor::launchStartOTSCommand(
5168  runSs.str(), CorePropertySupervisorBase::theConfigurationManager_);
5169  }
5170 
5171  RunControlStateMachine::theProgressBar_.step();
5172 
5173  // make logbook entry
5174  bool doLog = false;
5175  try
5176  {
5177  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
5178  }
5179  catch(...)
5180  { /* ignore errors */
5181  ;
5182  }
5183  if(doLog)
5184  {
5185  std::stringstream ss;
5186  ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
5187  << "' starting.";
5188 
5189  if(getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME) != "")
5190  ss << "\n\n-----------------\nUser log entry:\n"
5191  << getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME)
5192  << "\n-----------------\n";
5193  else
5194  ss << " No user log entry.";
5195 
5196  makeSystemLogEntry(ss.str());
5197  } // end make logbook entry
5198  RunControlStateMachine::theProgressBar_.step();
5199 
5200  activeStateMachineRunStartTime = std::chrono::steady_clock::now();
5201  activeStateMachineRunDuration_ms = 0;
5202  broadcastMessage(
5203  theStateMachine_
5204  .getCurrentMessage()); // ---------------------------------- broadcast!
5205  RunControlStateMachine::theProgressBar_.step();
5206 
5207  //check for remote subsystem dumps (after broadcast!)
5208  std::string remoteSubsystemDump = "";
5209  {
5210  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGatewayApps; //local copy
5211  { //lock for remainder of scope
5212  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
5213  __SUP_COUTVS__(22, remoteGatewayApps_.size());
5214  remoteGatewayApps = remoteGatewayApps_;
5215  if(remoteGatewayApps_.size())
5216  __SUP_COUT_TYPE__(TLVL_DEBUG + 22)
5217  << __COUT_HDR__ << remoteGatewayApps_[0].command << " "
5218  << (remoteGatewayApps_[0].appInfo.status) << __E__;
5219  }
5220 
5221  remoteSubsystemDump +=
5222  "--------------- Remote Subsystem Status ---------------\n";
5223  remoteSubsystemDump +=
5224  "Remote Subsystem Count: " + std::to_string(remoteGatewayApps.size()) + "\n";
5225  size_t ssi = 1;
5226  for(auto& remoteGatewayApp : remoteGatewayApps)
5227  {
5228  remoteSubsystemDump += std::to_string(ssi) + ". ~~ subsystem_name: " +
5229  remoteGatewayApp.appInfo.name + "\n. ";
5230  remoteSubsystemDump +=
5231  "subsystem_url: " + remoteGatewayApp.appInfo.url + "\n. ";
5232  // remoteSubsystemDump += "subsystem_landingPage: " + remoteGatewayApp.landingPage + "\n. ";
5233  remoteSubsystemDump +=
5234  "subsystem_status: " + remoteGatewayApp.appInfo.status + "\n. ";
5235  remoteSubsystemDump += "subsystem_progress: " +
5236  std::to_string(remoteGatewayApp.appInfo.progress) +
5237  "\n. ";
5238  remoteSubsystemDump +=
5239  "subsystem_detail: " + remoteGatewayApp.appInfo.detail + "\n. ";
5240  // remoteSubsystemDump += "subsystem_lastStatusTime: " + StringMacros::getTimestampString(remoteGatewayApp.appInfo.lastStatusTime) + "\n. ";
5241  // remoteSubsystemDump += "subsystem_consoleErrCount: " + std::to_string(remoteGatewayApp.consoleErrCount) + "\n. ";
5242  // remoteSubsystemDump += "subsystem_consoleWarnCount: " + std::to_string(remoteGatewayApp.consoleWarnCount) + "\n. ";
5243  remoteSubsystemDump +=
5244  "subsystem_configAlias: " + remoteGatewayApp.selected_config_alias +
5245  "\n. ";
5246  remoteSubsystemDump +=
5247  "subsystem_fsmMode: " + remoteGatewayApp.getFsmMode() + "\n. ";
5248  remoteSubsystemDump +=
5249  "subsystem_fsmIncluded: " +
5250  std::string(remoteGatewayApp.fsm_included ? "1" : "0") + "\n. ";
5251  }
5252  remoteSubsystemDump +=
5253  "--------------- end Remote Subsystem Status ---------------\n";
5254 
5255  remoteSubsystemDump += "\n\n-----------------\nRemote Configuration dump:\n";
5256  for(auto& remoteGatewayApp : remoteGatewayApps)
5257  {
5258  if(!remoteGatewayApp.fsm_included)
5259  continue; //skip if not included
5260  remoteSubsystemDump += remoteGatewayApp.config_dump;
5261  }
5262  remoteSubsystemDump += "\nEND Remote Configuration dump:\n-----------------\n";
5263 
5264  if(remoteSubsystemDump.size())
5265  __COUTV__(remoteSubsystemDump);
5266  } //end check for remote subsystem dumps
5267  RunControlStateMachine::theProgressBar_.step();
5268 
5269  if(activeStateMachineConfigurationDumpOnRunEnable_)
5270  {
5271  //write local configuration dump file
5272  std::string fullfilename = activeStateMachineConfigurationDumpOnRunFilename_ +
5273  "_" + std::to_string(time(0)) + ".dump";
5274  FILE* fp = fopen(fullfilename.c_str(), "w");
5275  if(!fp)
5276  {
5277  __SS__ << "Configuration dump failed to file: " << fullfilename << __E__;
5278  __SS_THROW__;
5279  }
5280 
5281  //(a la ConfigurationManager::dumpActiveConfiguration)
5282  fullfilename = __ENV__("HOSTNAME") + std::string(":") + fullfilename;
5283  fprintf(
5284  fp, "Original location of dump: %s\n", fullfilename.c_str());
5285 
5286  if(activeStateMachineConfigurationDumpOnRun_.size())
5287  fwrite(&activeStateMachineConfigurationDumpOnRun_[0],
5288  1,
5289  activeStateMachineConfigurationDumpOnRun_.size(),
5290  fp);
5291  __COUT__ << "Wrote configuration dump of char count "
5292  << activeStateMachineConfigurationDumpOnRun_.size()
5293  << " to file: " << fullfilename << __E__;
5294 
5295  if(remoteSubsystemDump.size())
5296  {
5297  fwrite(&remoteSubsystemDump[0], 1, remoteSubsystemDump.size(), fp);
5298 
5299  __COUT__ << "Wrote remote subsystem configuration dump of char count "
5300  << remoteSubsystemDump.size() << " to file: " << fullfilename
5301  << __E__;
5302  }
5303  fclose(fp);
5304 
5305  __COUT_INFO__ << "Run transition Configuration Dump saved to file: "
5306  << fullfilename << __E__;
5307  } //done with local config dump
5308  RunControlStateMachine::theProgressBar_.step();
5309 
5310  // save last started group name/key
5311  ConfigurationManager::saveGroupNameAndKey(theConfigurationTableGroup_,
5312  FSM_LAST_STARTED_GROUP_ALIAS_FILE);
5313  RunControlStateMachine::theProgressBar_.complete();
5314 
5315  // make logbook entry
5316  {
5317  std::stringstream ss;
5318  ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
5319  << "' started.";
5320 
5321  if(getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME) != "")
5322  ss << "\n\n-----------------\nUser log entry:\n"
5323  << getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME)
5324  << "\n-----------------\n";
5325  else
5326  ss << " No user log entry.";
5327 
5328  //insert system and remote subsystem status/detail
5329  {
5330  ss << "\n\n~~~ System Status and Detail ~~~\n";
5331  for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
5332  {
5333  const auto& appInfo = it.second;
5334  if(appInfo.getClass() != XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
5335  continue; //only give Gateway status
5336  ss << "\tStatus: " << appInfo.getStatus() << __E__
5337  << "\tDetail: " << appInfo.getDetail() << __E__;
5338  }
5339 
5340  //also return remote gateways as apps
5341  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteApps; //local copy
5342  { //lock for remainder of scope
5343  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
5344  remoteApps = remoteGatewayApps_;
5345  }
5346 
5347  if(remoteApps.size())
5348  {
5349  ss << "\n\n~~~ Subsystem Status and Detail ~~~\n";
5350 
5351  for(const auto& remoteApp : remoteApps)
5352  {
5353  const auto& appInfo = remoteApp.appInfo;
5354  ss << "Subsystem Name: " << appInfo.name << __E__
5355  << "\tStatus: " << appInfo.status << __E__
5356  << "\tDetail: " << appInfo.detail << __E__;
5357  }
5358  }
5359  }
5360 
5361  if(0) //full configuration dump too verbose for ECL (?)
5362  {
5363  ss << "\n\nConfigured with System Configuration Alias '"
5364  << activeStateMachineConfigurationAlias_ << "' which translates to "
5365  << theConfigurationTableGroup_.first << "("
5366  << theConfigurationTableGroup_.second << "). Active Context Group "
5367  << CorePropertySupervisorBase::theConfigurationManager_
5368  ->getActiveGroupName(ConfigurationManager::GroupType::CONTEXT_TYPE)
5369  << "("
5370  << CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
5371  ConfigurationManager::GroupType::CONTEXT_TYPE)
5372  << ").";
5373 
5374  if(activeStateMachineConfigurationDumpOnRunEnable_)
5375  {
5376  ss << "\n\n-----------------\nConfiguration dump:\n"
5377  << activeStateMachineConfigurationDumpOnRun_;
5378  ss << "\nEND Remote Configuration dump:\n-----------------\n";
5379  }
5380 
5381  if(remoteSubsystemDump.size())
5382  ss << remoteSubsystemDump;
5383  }
5384 
5385  makeSystemLogEntry(ss.str(),
5386  activeStateMachineRunAlias_ + " '" +
5387  activeStateMachineRunNumber_ + "' started");
5388  } // end make logbook entry
5389  __COUT__ << "Done starting run." << __E__;
5390  RunControlStateMachine::theProgressBar_.complete();
5391 
5392 } // end transitionStarting()
5393 catch(const xdaq::exception::Exception& e) // due to xoap send failure
5394 {
5395  __SS__ << "\nTransition to Starting Run interrupted! There was a system "
5396  "communication error "
5397  "identified. "
5398  << __E__ << e.what();
5399  __COUT_ERR__ << "\n" << ss.str();
5400  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5401 }
5402 catch(std::runtime_error& e)
5403 {
5404  __SS__ << "\nTransition to Starting Run interrupted! There was an error "
5405  "identified. "
5406  << __E__ << e.what();
5407  __COUT_ERR__ << "\n" << ss.str();
5408  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5409 }
5410 catch(toolbox::fsm::exception::Exception& e)
5411 {
5412  throw; // just rethrow exceptions of already the correct type
5413 }
5414 catch(...)
5415 {
5416  __SS__ << "\nTransition to Starting Run interrupted! There was an unknown error "
5417  "identified. "
5418  << __E__;
5419  try
5420  {
5421  throw;
5422  } //one more try to printout extra info
5423  catch(const std::exception& e)
5424  {
5425  ss << "Exception message: " << e.what();
5426  }
5427  catch(...)
5428  {
5429  }
5430  __COUT_ERR__ << "\n" << ss.str();
5431  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5432 } // end transitionStarting() catch
5433 
5434 //==============================================================================
5435 void GatewaySupervisor::transitionStopping(toolbox::Event::Reference /*e*/)
5436 try
5437 {
5438  checkForAsyncError();
5439 
5440  __COUT__ << "Fsm current state: " << theStateMachine_.getCurrentStateName() << __E__;
5441 
5442  activeStateMachineRunDuration_ms +=
5443  std::chrono::duration_cast<std::chrono::milliseconds>(
5444  std::chrono::steady_clock::now() - activeStateMachineRunStartTime)
5445  .count();
5446 
5447  RunControlStateMachine::theProgressBar_.step();
5448 
5449  bool doLog = false;
5450  try
5451  {
5452  doLog = __ENV__("OTS_LOG_TRANSITION_STARTS") == std::string("1");
5453  }
5454  catch(...)
5455  { /* ignore errors */
5456  ;
5457  }
5458 
5459  // calculate run duration and make system log entry
5460  std::ostringstream dur_ss;
5461  {
5462  int dur = activeStateMachineRunDuration_ms;
5463  int dur_s = dur / 1000;
5464  dur = dur % 1000;
5465  int dur_m = dur_s / 60;
5466  dur_s = dur_s % 60;
5467  int dur_h = dur_m / 60;
5468  dur_m = dur_m % 60;
5469  dur_ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_
5470  << "' duration of " << std::setw(2) << std::setfill('0') << dur_h << ":"
5471  << std::setw(2) << std::setfill('0') << dur_m << ":" << std::setw(2)
5472  << std::setfill('0')
5473  << dur_s; //too much detail "." << dur << " seconds.";
5474  if(dur_h == 0 && dur_m == 0 && dur_s < 5) //if very short, add the detail
5475  dur_ss << "." << dur << " seconds.";
5476  else
5477  dur_ss << ".";
5478 
5479  if(doLog)
5480  {
5481  std::stringstream ss;
5482  ss << dur_ss.str();
5483  if(getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME) != "")
5484  ss << "\n\n-----------------\nUser log entry:\n"
5485  << getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME)
5486  << "\n-----------------\n";
5487  else
5488  ss << " No user log entry.";
5489 
5490  makeSystemLogEntry("Run stopping. " + ss.str());
5491  }
5492  }
5493 
5494  // the current message is not for Stop if its due to async exception, so rename
5495  if(RunControlStateMachine::asyncStopExceptionReceived_)
5496  {
5497  __COUT_ERR__ << "Broadcasting stop for async STOP exception!" << __E__;
5498  broadcastMessage(SOAPUtilities::makeSOAPMessageReference("Stop"));
5499  }
5500  else
5501  broadcastMessage(theStateMachine_.getCurrentMessage());
5502 
5503  // make logbook entry
5504  {
5505  std::stringstream ss;
5506  ss << dur_ss.str();
5507  if(getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME) != "")
5508  ss << "\n\n-----------------\nUser log entry:\n"
5509  << getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME)
5510  << "\n-----------------\n";
5511  else
5512  ss << " No user log entry.";
5513 
5514  makeSystemLogEntry("Run stopped.\n" + ss.str(),
5515  activeStateMachineRunAlias_ + " '" +
5516  activeStateMachineRunNumber_ + "' stopped");
5517  } // end make logbook entry
5518 
5519  __COUT__ << "Done stopping run." << __E__;
5520  RunControlStateMachine::theProgressBar_.complete();
5521 
5522  //Roll over log file if enabled
5523  if(activeStateMachineRollOverLogOnStart_)
5524  {
5525  __COUT_INFO__ << "Rolling over log file on Stop transition..." << __E__;
5526  std::stringstream runSs;
5527  runSs << "LOG_ROLLOVER";
5528  runSs << ";Post" << activeStateMachineRunAlias_ << "_"
5529  << activeStateMachineRunNumber_;
5530 
5531  GatewaySupervisor::launchStartOTSCommand(
5532  runSs.str(), CorePropertySupervisorBase::theConfigurationManager_);
5533  }
5534 
5535  RunControlStateMachine::theProgressBar_.step();
5536 
5537 } // end transitionStopping()
5538 catch(const xdaq::exception::Exception& e) // due to xoap send failure
5539 {
5540  __SS__ << "\nTransition to Stopping Run interrupted! There was a system "
5541  "communication error "
5542  "identified. "
5543  << __E__ << e.what();
5544  __COUT_ERR__ << "\n" << ss.str();
5545  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5546 }
5547 catch(std::runtime_error& e)
5548 {
5549  __SS__ << "\nTransition to Stopping Run interrupted! There was an error "
5550  "identified. "
5551  << __E__ << e.what();
5552  __COUT_ERR__ << "\n" << ss.str();
5553  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5554 }
5555 catch(toolbox::fsm::exception::Exception& e)
5556 {
5557  throw; // just rethrow exceptions of already the correct type
5558 }
5559 catch(...)
5560 {
5561  __SS__ << "\nTransition to Stopping Run interrupted! There was an unknown error "
5562  "identified. "
5563  << __E__;
5564  try
5565  {
5566  throw;
5567  } //one more try to printout extra info
5568  catch(const std::exception& e)
5569  {
5570  ss << "Exception message: " << e.what();
5571  }
5572  catch(...)
5573  {
5574  }
5575  __COUT_ERR__ << "\n" << ss.str();
5576  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5577 } // end transitionStopping() catch
5578 
5582 
5583 //==============================================================================
5589 bool GatewaySupervisor::handleBroadcastMessageTarget(const SupervisorInfo& appInfo,
5590  xoap::MessageReference message,
5591  const std::string& command,
5592  const unsigned int& iteration,
5593  std::string& reply,
5594  unsigned int threadIndex)
5595 try
5596 {
5597  unsigned int subIteration = 0; // reset for next subIteration loop
5598  bool subIterationsDone = false;
5599  bool iterationsDone = true;
5600 
5601  while(!subIterationsDone) // start subIteration handling loop
5602  {
5603  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5604  << "Supervisor instance = '" << appInfo.getName()
5605  << "' [LID=" << appInfo.getId() << "] in Context '"
5606  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5607  << "] Command = " << command << __E__;
5608 
5609  checkForAsyncError();
5610 
5611  subIterationsDone = true;
5612  RunControlStateMachine::theProgressBar_.step();
5613 
5614  // add subIteration index to message
5615  if(subIteration)
5616  {
5617  SOAPParameters parameters;
5618  parameters.addParameter("subIterationIndex", subIteration);
5619  SOAPUtilities::addParameters(message, parameters);
5620  }
5621 
5622  if(iteration || subIteration)
5623  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5624  << "Adding iteration parameters " << iteration << "." << subIteration
5625  << __E__;
5626 
5627  RunControlStateMachine::theProgressBar_.step();
5628 
5629  std::string givenAppStatus = SupervisorInfo::APP_STATUS_UNKNOWN;
5630  try
5631  {
5632  givenAppStatus = theStateMachine_.getCurrentTransitionName(command);
5633  }
5634  catch(...)
5635  {
5636  //ignoring invalid transition tranistion name error
5637  }
5638 
5639  unsigned int givenAppProgress = appInfo.getProgress();
5640  std::string givenAppDetail = appInfo.getDetail();
5641  if(givenAppProgress >= 100)
5642  {
5643  givenAppProgress = 0; // reset
5644  givenAppDetail = "";
5645  }
5646 
5647  if(iteration == 0 && subIteration == 0) //first time through the supervisors
5648  {
5649  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5650  << "Sending message to Supervisor " << appInfo.getName()
5651  << " [LID=" << appInfo.getId() << "]: " << command << __E__;
5652 
5653  givenAppDetail = "";
5654  }
5655  else // else this not the first time through the supervisors
5656  {
5657  if(givenAppDetail == "")
5658  givenAppDetail =
5659  std::to_string(iteration) + ":" + std::to_string(subIteration);
5660  if(subIteration == 0)
5661  {
5662  for(unsigned int j = 0; j < 4; ++j)
5663  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5664  << "Sending message to Supervisor " << appInfo.getName()
5665  << " [LID=" << appInfo.getId() << "]: " << command
5666  << " (iteration: " << iteration << ")" << __E__;
5667  }
5668  else
5669  {
5670  for(unsigned int j = 0; j < 4; ++j)
5671  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5672  << "Sending message to Supervisor " << appInfo.getName()
5673  << " [LID=" << appInfo.getId() << "]: " << command
5674  << " (iteration: " << iteration
5675  << ", sub-iteration: " << subIteration << ")" << __E__;
5676  }
5677  }
5678 
5679  {
5680  // add the message index
5681  SOAPParameters parameters;
5682  { // mutex scope
5683  std::lock_guard<std::mutex> lock(broadcastCommandMessageIndexMutex_);
5684  parameters.addParameter("commandId", broadcastCommandMessageIndex_++);
5685  } // end mutex scope
5686  SOAPUtilities::addParameters(message, parameters);
5687  }
5688 
5689  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5690  << "Sending... \t" << SOAPUtilities::translate(message) << std::endl;
5691 
5692  try // attempt transmit of transition command
5693  {
5694  __COUT__ << "Broadcast thread " << threadIndex
5695  << "\t givenAppStatus=" << givenAppStatus << __E__;
5696  __COUT__ << "Broadcast thread " << threadIndex
5697  << "\t appInfo.getStatus()=" << appInfo.getStatus() << __E__;
5698 
5699  // wait for app to exist in status before sending commands
5700  int waitAttempts = 0;
5701  while(appInfo.getStatus() == SupervisorInfo::APP_STATUS_UNKNOWN)
5702  {
5703  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5704  << "Waiting for Supervisor " << appInfo.getName()
5705  << " [LID=" << appInfo.getId()
5706  << "] in unknown state. waitAttempts of 10 = " << waitAttempts
5707  << __E__;
5708  ++waitAttempts;
5709  if(waitAttempts == 10)
5710  {
5711  __SS__ << "Error! Gateway Supervisor failed to send message to app "
5712  "in unknown state "
5713  "Supervisor instance = '"
5714  << appInfo.getName() << "' [LID=" << appInfo.getId()
5715  << "] in Context '" << appInfo.getContextName()
5716  << "' [URL=" << appInfo.getURL() << "].\n\n";
5717  __COUT_ERR__ << ss.str();
5718  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5719  }
5720  sleep(2);
5721  }
5722 
5723  // start recursive mutex scope (same thread can lock multiple times, but needs to unlock the same)
5724  std::lock_guard<std::recursive_mutex> lock(
5725  allSupervisorInfo_.getSupervisorInfoMutex(appInfo.getId()));
5726  // set app status, but leave progress and detail alone
5727  allSupervisorInfo_.setSupervisorStatus(
5728  appInfo, givenAppStatus, givenAppProgress, givenAppDetail);
5729 
5730  // for transition attempt, set status for app, in case the request occupies the target app
5731  std::string tmpReply = send(appInfo.getDescriptor(), message);
5732  __COUTV__(tmpReply);
5733  //using the intermediate temporary string seems to possibly help when there are multiple crashes of FSM entities
5734  reply = tmpReply;
5735 
5736  // then release mutex here using scope change, to allow the app to start giving its own updates
5737  }
5738  catch(const xdaq::exception::Exception& e) // due to xoap send failure
5739  {
5740  // do not kill whole system if xdaq xoap failure
5741  __SS__ << "Error! Gateway Supervisor can NOT " << command
5742  << " Supervisor instance = '" << appInfo.getName()
5743  << "' [LID=" << appInfo.getId() << "] in Context '"
5744  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5745  << "].\n\n"
5746  << "Xoap message failure. Did the target Supervisor crash? Try "
5747  "re-initializing or restarting otsdaq."
5748  << __E__;
5749  __COUT_ERR__ << ss.str();
5750 
5751  try
5752  {
5753  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5754  << "Try again.." << __E__;
5755 
5756  {
5757  // add a second try parameter flag
5758  SOAPParameters parameters;
5759  parameters.addParameter("retransmission", "1");
5760  SOAPUtilities::addParameters(message, parameters);
5761  }
5762 
5763  {
5764  // add the message index
5765  SOAPParameters parameters;
5766  { // mutex scope
5767  std::lock_guard<std::mutex> lock(
5768  broadcastCommandMessageIndexMutex_);
5769  parameters.addParameter("commandId",
5770  broadcastCommandMessageIndex_++);
5771  } // end mutex scope
5772  SOAPUtilities::addParameters(message, parameters);
5773  }
5774 
5775  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5776  << "Re-Sending... " << SOAPUtilities::translate(message)
5777  << std::endl;
5778 
5779  reply = send(appInfo.getDescriptor(), message);
5780  }
5781  catch(const xdaq::exception::Exception& e) // due to xoap send failure
5782  {
5783  __COUT_ERR__ << "Broadcast thread " << threadIndex << "\t"
5784  << "Second try failed.." << __E__;
5785  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5786  }
5787  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5788  << "2nd try passed.." << __E__;
5789  } // end send catch
5790 
5791  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5792  << "Reply received from " << appInfo.getName()
5793  << " [LID=" << appInfo.getId() << "]: " << reply << __E__;
5794 
5795  if((reply != command + "Done") && (reply != command + "Response") &&
5796  (reply != command + "Iterate") && (reply != command + "SubIterate"))
5797  {
5798  __SS__ << "Error! Gateway Supervisor can NOT " << command
5799  << " Supervisor instance = '" << appInfo.getName()
5800  << "' [LID=" << appInfo.getId() << "] in Context '"
5801  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5802  << "].\n\n"
5803  << reply;
5804  __COUT_ERR__ << ss.str() << __E__;
5805 
5806  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5807  << "Getting error message..." << __E__;
5808  try
5809  {
5810  xoap::MessageReference errorMessage =
5811  sendWithSOAPReply(appInfo.getDescriptor(),
5812  SOAPUtilities::makeSOAPMessageReference(
5813  "StateMachineErrorMessageRequest"));
5814  SOAPParameters parameters;
5815  parameters.addParameter("ErrorMessage");
5816  SOAPUtilities::receive(errorMessage, parameters);
5817 
5818  std::string error = parameters.getValue("ErrorMessage");
5819  if(error == "")
5820  {
5821  std::stringstream err;
5822  err << "Unknown error from Supervisor instance = '"
5823  << appInfo.getName() << "' [LID=" << appInfo.getId()
5824  << "] in Context '" << appInfo.getContextName()
5825  << "' [URL=" << appInfo.getURL()
5826  << "]. If the problem persists or is repeatable, please notify "
5827  "admins.\n\n";
5828  error = err.str();
5829  }
5830 
5831  __SS__ << "Received error message from Supervisor instance = '"
5832  << appInfo.getName() << "' [LID=" << appInfo.getId()
5833  << "] in Context '" << appInfo.getContextName()
5834  << "' [URL=" << appInfo.getURL()
5835  << "].\n\n Error Message = " << error << __E__;
5836 
5837  __COUT_ERR__ << ss.str() << __E__;
5838 
5839  if(command == RunControlStateMachine::ERROR_TRANSITION_NAME)
5840  return true; // do not throw exception and exit loop if informing all
5841  // apps about error
5842  // else throw exception and go into Error
5843  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5844  }
5845  catch(const xdaq::exception::Exception& e) // due to xoap send failure
5846  {
5847  // do not kill whole system if xdaq xoap failure
5848  __SS__ << "Error! Gateway Supervisor failed to read error message from "
5849  "Supervisor instance = '"
5850  << appInfo.getName() << "' [LID=" << appInfo.getId()
5851  << "] in Context '" << appInfo.getContextName()
5852  << "' [URL=" << appInfo.getURL() << "].\n\n"
5853  << "Xoap message failure. Did the target Supervisor crash? Try "
5854  "re-initializing or restarting otsdaq."
5855  << __E__;
5856  __COUT_ERR__ << ss.str();
5857  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5858  }
5859  } // end error response handling
5860  else if(reply == command + "Iterate")
5861  {
5862  // when 'Working' this front-end is expecting
5863  // to get the same command again with an incremented iteration index
5864  // after all other front-ends see the same iteration index, and all
5865  // front-ends with higher priority see the incremented iteration index.
5866 
5867  iterationsDone = false;
5868  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5869  << "Supervisor instance = '" << appInfo.getName()
5870  << "' [LID=" << appInfo.getId() << "] in Context '"
5871  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5872  << "] flagged for another iteration to " << command
5873  << "... (iteration: " << iteration << ")" << __E__;
5874 
5875  } // end still working response handling
5876  else if(reply == command + "SubIterate")
5877  {
5878  // when 'Working' this front-end is expecting
5879  // to get the same command again with an incremented sub-iteration index
5880  // without any other front-ends taking actions or seeing the sub-iteration
5881  // index.
5882 
5883  subIterationsDone = false;
5884  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5885  << "Supervisor instance = '" << appInfo.getName()
5886  << "' [LID=" << appInfo.getId() << "] in Context '"
5887  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5888  << "] flagged for another sub-iteration to " << command
5889  << "... (iteration: " << iteration
5890  << ", sub-iteration: " << subIteration << ")" << __E__;
5891  }
5892  else // else success response
5893  {
5894  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5895  << "Supervisor instance = '" << appInfo.getName()
5896  << "' [LID=" << appInfo.getId() << "] in Context '"
5897  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5898  << "] was " << command << "'d correctly!" << __E__;
5899  }
5900 
5901  if(subIteration)
5902  __COUT__ << "Broadcast thread " << threadIndex << "\t"
5903  << "Completed sub-iteration: " << subIteration << __E__;
5904  ++subIteration;
5905 
5906  } // end subIteration handling loop
5907 
5908  return iterationsDone;
5909 
5910 } // end handleBroadcastMessageTarget()
5911 catch(const toolbox::fsm::exception::Exception& e)
5912 {
5913  throw;
5914 } //keep existing FSM execptions intact
5915 catch(...)
5916 {
5917  // do not kill whole system if unexpected exception
5918  __SS__ << "Error! Gateway Supervisor failed to broadcast message '" << command
5919  << "' to "
5920  "Supervisor instance = '"
5921  << appInfo.getName() << "' [LID=" << appInfo.getId() << "] in Context '"
5922  << appInfo.getContextName() << "' [URL=" << appInfo.getURL()
5923  << "]. Try re-initializing or restarting otsdaq." << __E__;
5924  __COUT_ERR__ << ss.str();
5925  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
5926 }
5927 
5928 //==============================================================================
5932 void GatewaySupervisor::broadcastMessageThread(
5933  GatewaySupervisor* supervisorPtr,
5934  std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct> threadStruct)
5935 {
5936  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5937  << "established..." << __E__;
5938 
5939  while(!threadStruct->exitThread_)
5940  {
5941  // sleep to give time to main thread to dole out work
5942  usleep(1000 /* 1ms */);
5943 
5944  // take lock for remainder of scope
5945  std::lock_guard<std::mutex> lock(threadStruct->threadMutex_);
5946  if(threadStruct->workToDo_)
5947  {
5948  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5949  << "starting work... command = " << threadStruct->getCommand()
5950  << __E__;
5951 
5952  try
5953  {
5954  if(supervisorPtr->handleBroadcastMessageTarget(
5955  threadStruct->getAppInfo(),
5956  threadStruct->getMessage(),
5957  threadStruct->getCommand(),
5958  threadStruct->getIteration(),
5959  threadStruct->getReply(),
5960  threadStruct->threadIndex_))
5961  threadStruct->getIterationsDone() = true;
5962  }
5963  catch(const toolbox::fsm::exception::Exception& e)
5964  {
5965  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5966  << "going into error: " << e.what() << __E__;
5967 
5968  threadStruct->getReply() = e.what();
5969  threadStruct->error_ = true;
5970  threadStruct->workToDo_ = false;
5971  threadStruct->working_ = false; // indicate exiting
5972  return;
5973  }
5974 
5975  if(!threadStruct->getIterationsDone())
5976  {
5977  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5978  << "flagged for another iteration." << __E__;
5979 
5980  // set global iterationsDone
5981  std::lock_guard<std::mutex> lock(
5982  supervisorPtr->broadcastIterationsDoneMutex_);
5983  supervisorPtr->broadcastIterationsDone_ = false;
5984  }
5985 
5986  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5987  << "done with work." << __E__;
5988 
5989  threadStruct->workToDo_ = false;
5990  } // end work
5991 
5992  } // end primary while loop
5993 
5994  __COUT__ << "Broadcast thread " << threadStruct->threadIndex_ << "\t"
5995  << "exited." << __E__;
5996  threadStruct->working_ = false; // indicate exiting
5997 } // end broadcastMessageThread()
5998 
5999 //==============================================================================
6004 void GatewaySupervisor::broadcastMessage(xoap::MessageReference message)
6005 {
6006  { // create lock scope and clear status
6007  std::lock_guard<std::mutex> lock(broadcastCommandStatusUpdateMutex_);
6008  broadcastCommandStatus_ = "";
6009  }
6010 
6011  RunControlStateMachine::theProgressBar_.step();
6012 
6013  // transition of Gateway Supervisor is assumed successful so update status
6014  allSupervisorInfo_.setSupervisorStatus(this, theStateMachine_.getCurrentStateName());
6015 
6016  std::string command = SOAPUtilities::translate(message).getCommand();
6017 
6018  std::string reply;
6019  broadcastIterationsDone_ = false;
6020  bool assignedJob;
6021 
6022  std::vector<std::vector<const SupervisorInfo*>> orderedSupervisors;
6023 
6024  try
6025  {
6026  orderedSupervisors = allSupervisorInfo_.getOrderedSupervisorDescriptors(
6027  command,
6028  // only gateway apps for special shutdown and startup command broadcast
6029  command == RunControlStateMachine::SHUTDOWN_TRANSITION_NAME ||
6030  command == RunControlStateMachine::STARTUP_TRANSITION_NAME);
6031  }
6032  catch(const std::runtime_error& e)
6033  {
6034  __SS__
6035  << "Error getting supervisor priority. Was there a change in the context?"
6036  << " Remember, if the context was changed, it is recommended to relaunch the "
6037  "ots script. "
6038  << e.what() << __E__;
6039  XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str());
6040  }
6041 
6042  RunControlStateMachine::theProgressBar_.step();
6043 
6044  // std::vector<std::vector<uint8_t/*bool*/>> supervisorIterationsDone; //Note: can not
6045  // use bool because std::vector does not allow access by reference of type bool
6046  GatewaySupervisor::BroadcastMessageIterationsDoneStruct supervisorIterationsDone;
6047 
6048  // initialize to false (not done)
6049  for(const auto& vectorAtPriority : orderedSupervisors)
6050  supervisorIterationsDone.push(vectorAtPriority.size()); // push_back(
6051  // std::vector<uint8_t>(vectorAtPriority.size(),
6052  // false /*initial value*/));
6053 
6054  unsigned int iteration = 0;
6055  // unsigned int subIteration;
6056  unsigned int iterationBreakpoint;
6057 
6058  // send command to all supervisors (for multiple iterations) until all are done
6059 
6060  // make a copy of the message to use as starting point for iterations
6061  xoap::MessageReference originalMessage =
6062  SOAPUtilities::makeSOAPMessageReference(SOAPUtilities::translate(message));
6063 
6064  __COUT__ << "=========> Broadcasting state machine command = " << command << __E__;
6065 
6066  unsigned int numberOfThreads = 1;
6067 
6068  try
6069  {
6070  numberOfThreads = CorePropertySupervisorBase::getSupervisorTableNode()
6071  .getNode("NumberOfStateMachineBroadcastThreads")
6072  .getValue<unsigned int>();
6073  }
6074  catch(...)
6075  {
6076  // ignore error for backwards compatibility
6077  __COUT__ << "Number of threads not in configuration, so defaulting to "
6078  << numberOfThreads << __E__;
6079  }
6080 
6081  // Note: if 1 thread, then create no threads
6082  // i.e. only create threads if 2 or more.
6083  if(numberOfThreads == 1)
6084  numberOfThreads = 0;
6085 
6086  __COUTV__(numberOfThreads);
6087 
6088  // std::vector<GatewaySupervisor::BroadcastThreadStruct> broadcastThreadStructs_(numberOfThreads);
6089  broadcastThreadStructs_.clear();
6090 
6091  // only launch threads if more than 1
6092  // if 1, just use main thread
6093  for(unsigned int i = 0; i < numberOfThreads; ++i)
6094  {
6095  broadcastThreadStructs_.push_back(
6096  std::make_shared<GatewaySupervisor::BroadcastThreadStruct>());
6097  broadcastThreadStructs_[i]->threadIndex_ = i;
6098 
6099  std::thread(
6100  [](GatewaySupervisor* supervisorPtr,
6101  std::shared_ptr<GatewaySupervisor::BroadcastThreadStruct> threadStruct) {
6102  GatewaySupervisor::broadcastMessageThread(supervisorPtr, threadStruct);
6103  },
6104  this,
6105  broadcastThreadStructs_[i])
6106  .detach();
6107  } // end broadcast thread creation loop
6108 
6109  RunControlStateMachine::theProgressBar_.step();
6110 
6111  broadcastMessageToRemoteGateways(originalMessage);
6112 
6113  RunControlStateMachine::theProgressBar_.step();
6114 
6115  try
6116  {
6117  //:::::::::::::::::::::::::::::::::::::::::::::::::::::
6118  // Send a SOAP message to every Supervisor in order by priority
6119  do // while !iterationsDone
6120  {
6121  broadcastIterationsDone_ = true;
6122 
6123  { // start mutex scope
6124  std::lock_guard<std::mutex> lock(broadcastIterationBreakpointMutex_);
6125  iterationBreakpoint = broadcastIterationBreakpoint_; // get breakpoint
6126  } // end mutex scope
6127 
6128  if(iterationBreakpoint < (unsigned int)-1)
6129  __COUT__ << "Iteration breakpoint currently is " << iterationBreakpoint
6130  << __E__;
6131  if(iteration >= iterationBreakpoint)
6132  {
6133  broadcastIterationsDone_ = false;
6134  __COUT__ << "Waiting at transition breakpoint - iteration = " << iteration
6135  << __E__;
6136  usleep(5 * 1000 * 1000 /*5 s*/);
6137  continue; // wait until breakpoint moved
6138  }
6139 
6140  if(iteration)
6141  __COUT__ << "Starting iteration: " << iteration << __E__;
6142 
6143  for(unsigned int i = 0; i < supervisorIterationsDone.size(); ++i)
6144  {
6145  for(unsigned int j = 0; j < supervisorIterationsDone.size(i); ++j)
6146  {
6147  checkForAsyncError();
6148 
6149  if(supervisorIterationsDone[i][j])
6150  continue; // skip if supervisor is already done
6151 
6152  const SupervisorInfo& appInfo = *(orderedSupervisors[i][j]);
6153 
6154  // re-acquire original message
6155  message = SOAPUtilities::makeSOAPMessageReference(
6156  SOAPUtilities::translate(originalMessage));
6157 
6158  // add iteration index to message
6159  if(iteration)
6160  {
6161  // add the iteration index as a parameter to message
6162  SOAPParameters parameters;
6163  parameters.addParameter("iterationIndex", iteration);
6164  SOAPUtilities::addParameters(message, parameters);
6165  }
6166 
6167  if(numberOfThreads)
6168  {
6169  // schedule message to first open thread
6170  assignedJob = false;
6171  do
6172  {
6173  for(unsigned int k = 0; k < numberOfThreads; ++k)
6174  {
6175  if(!broadcastThreadStructs_[k]->workToDo_)
6176  {
6177  // found our thread!
6178  assignedJob = true;
6179  __COUT__ << "Giving work to thread " << k
6180  << ", command = " << command << __E__;
6181 
6182  std::lock_guard<std::mutex> lock(
6183  broadcastThreadStructs_[k]->threadMutex_);
6184  broadcastThreadStructs_[k]->setMessage(
6185  appInfo,
6186  message,
6187  command,
6188  iteration,
6189  supervisorIterationsDone[i][j]);
6190 
6191  break;
6192  }
6193  } // end thread assigning search
6194 
6195  if(!assignedJob)
6196  {
6197  __COUT__ << "No free broadcast threads, "
6198  << "waiting for an available thread..." << __E__;
6199  usleep(100 * 1000 /*100 ms*/);
6200  }
6201  } while(!assignedJob);
6202  }
6203  else // no thread
6204  {
6205  if(handleBroadcastMessageTarget(
6206  appInfo, message, command, iteration, reply))
6207  supervisorIterationsDone[i][j] = true;
6208  else
6209  broadcastIterationsDone_ = false;
6210  }
6211 
6212  } // end supervisors at same priority broadcast loop
6213 
6214  // before proceeding to next priority,
6215  // make sure all threads have completed
6216  if(numberOfThreads)
6217  {
6218  __COUT__
6219  << "Done with priority level. Waiting for threads to finish..."
6220  << __E__;
6221  bool done;
6222  do
6223  {
6224  done = true;
6225  unsigned int numOfThreadsWithWork = 0;
6226  unsigned int lastUnfinishedThread = -1;
6227 
6228  for(unsigned int i = 0; i < numberOfThreads; ++i)
6229  if(broadcastThreadStructs_[i]->workToDo_)
6230  {
6231  done = false;
6232  ++numOfThreadsWithWork;
6233  lastUnfinishedThread = i;
6234  }
6235  else if(broadcastThreadStructs_[i]->error_)
6236  {
6237  __COUT__ << "Found thread in error! Throwing state "
6238  "machine error: "
6239  << broadcastThreadStructs_[i]->getReply()
6240  << __E__;
6241  XCEPT_RAISE(toolbox::fsm::exception::Exception,
6242  broadcastThreadStructs_[i]->getReply());
6243  }
6244 
6245  if(!done) // update status and sleep
6246  {
6247  std::stringstream waitSs;
6248  waitSs << "Waiting on " << numOfThreadsWithWork << " of "
6249  << numberOfThreads
6250  << " threads to finish. Command = " << command;
6251  if(command ==
6252  RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
6253  waitSs << " w/" + RunControlStateMachine::
6254  getLastAttemptedConfigureGroup();
6255  if(numOfThreadsWithWork == 1)
6256  {
6257  waitSs << ".. "
6258  << broadcastThreadStructs_[lastUnfinishedThread]
6259  ->getAppInfo()
6260  .getName()
6261  << ":"
6262  << broadcastThreadStructs_[lastUnfinishedThread]
6263  ->getAppInfo()
6264  .getId();
6265  }
6266  waitSs << __E__;
6267  __COUT__ << waitSs.str();
6268 
6269  { // create lock scope that does not include sleep
6270  std::lock_guard<std::mutex> lock(
6271  broadcastCommandStatusUpdateMutex_);
6272  broadcastCommandStatus_ = waitSs.str();
6273  }
6274  usleep(100 * 1000 /*100ms*/);
6275  }
6276 
6277  } while(!done);
6278  __COUT__ << "All threads done with priority level work." << __E__;
6279  } // end thread complete verification
6280 
6281  } // end supervisor broadcast loop for each priority
6282 
6283  // if (!proceed)
6284  // {
6285  // __COUT__ << "Breaking out of primary loop." << __E__;
6286  // break;
6287  // }
6288 
6289  if(iteration || !broadcastIterationsDone_)
6290  __COUT__ << "Completed iteration: " << iteration << __E__;
6291  ++iteration;
6292 
6293  } while(!broadcastIterationsDone_);
6294 
6295  RunControlStateMachine::theProgressBar_.step();
6296  } // end main transition broadcast try
6297  catch(...)
6298  {
6299  __COUT__ << "Exception caught, exiting broadcast threads..." << __E__;
6300 
6301  // attempt to exit threads
6302  // The threads should already be done with all work.
6303  // If broadcastMessage scope ends, then the
6304  // thread struct will be destructed, and the thread will
6305  // crash on next access attempt (though we probably do not care).
6306  for(unsigned int i = 0; i < numberOfThreads; ++i)
6307  broadcastThreadStructs_[i]->exitThread_ = true;
6308  usleep(100 * 1000 /*100ms*/); // sleep for exit time
6309 
6310  throw; // re-throw
6311  }
6312 
6313  if(numberOfThreads)
6314  {
6315  __COUT__ << "All transitions completed. Wrapping up, exiting broadcast threads..."
6316  << __E__;
6317 
6318  // attempt to exit threads
6319  // The threads should already be done with all work.
6320  // If broadcastMessage scope ends, then the
6321  // thread struct will be destructed, and the thread will
6322  // crash on next access attempt (when the thread crashes, the whole context
6323  // crashes).
6324  for(unsigned int i = 0; i < numberOfThreads; ++i)
6325  broadcastThreadStructs_[i]->exitThread_ = true;
6326  usleep(100 * 1000 /*100ms*/); // sleep for exit time
6327  }
6328 
6329  RunControlStateMachine::theProgressBar_.step();
6330 
6331  if(broadcastMessageToRemoteGatewaysComplete(originalMessage))
6332  {
6333  RunControlStateMachine::theProgressBar_.step();
6334  __COUT__ << "Broadcast complete." << __E__;
6335  }
6336 } // end broadcastMessage()
6337 
6338 //==============================================================================
6339 void GatewaySupervisor::broadcastMessageToRemoteGateways(
6340  const xoap::MessageReference message)
6341 {
6342  SOAPCommand commandObj = SOAPUtilities::translate(message);
6343  std::string command = commandObj.getCommand();
6344  __COUTV__(command);
6345 
6346  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
6347  for(auto& remoteGatewayApp : remoteGatewayApps_)
6348  {
6349  //construct command params based on remote gateway settings
6350  std::string commandAndParams = command;
6351  if(commandObj.hasParameters())
6352  {
6353  //parameters over UDP are much simpler than xoap message, so filter
6354  for(const auto& param : commandObj.getParameters())
6355  {
6356  __COUTTV__(param.first);
6357  __COUTTV__(param.second);
6358  if(param.first == "ConfigurationAlias")
6359  {
6360  if(remoteGatewayApp.selected_config_alias != "") //replace
6361  commandAndParams += "," + remoteGatewayApp.selected_config_alias;
6362  else
6363  commandAndParams += "," + param.second;
6364  }
6365  else if(param.first == "RunNumber")
6366  {
6367  commandAndParams += "," + //param.first + ":" +
6368  param.second;
6369  }
6370 
6371  // else
6372  // commandAndParams += "," + param.first + ":" + param.second;
6373  }
6374  __COUTV__(commandAndParams);
6375  }
6376 
6377  if(!remoteGatewayApp.fsm_included)
6378  {
6379  __COUT__ << "Skipping excluded Remote gateway '"
6380  << remoteGatewayApp.appInfo.name
6381  << "' for FSM command = " << commandAndParams << __E__;
6382  continue; //skip if not included
6383  }
6384 
6385  if(remoteGatewayApp.fsm_mode == RemoteGatewayInfo::FSM_ModeTypes::DoNotHalt &&
6386  //do not allow halt/err transitions:
6387  (command == RunControlStateMachine::ERROR_TRANSITION_NAME ||
6388  command == RunControlStateMachine::FAIL_TRANSITION_NAME ||
6389  command == RunControlStateMachine::HALT_TRANSITION_NAME ||
6390  command == RunControlStateMachine::ABORT_TRANSITION_NAME))
6391  {
6392  __COUT__ << "Skipping '" << remoteGatewayApp.getFsmMode()
6393  << "' Remote gateway '" << remoteGatewayApp.appInfo.name
6394  << "' for FSM command = " << commandAndParams << __E__;
6395  continue; //skip if not included
6396  }
6397 
6398  if(remoteGatewayApp.fsm_mode == RemoteGatewayInfo::FSM_ModeTypes::OnlyConfigure &&
6399  ! //invert of allowed situations:
6400  (remoteGatewayApp.appInfo.status ==
6401  RunControlStateMachine::INITIAL_STATE_NAME ||
6402  remoteGatewayApp.appInfo.status ==
6403  RunControlStateMachine::HALTED_STATE_NAME ||
6404  remoteGatewayApp.appInfo.status.find(
6405  RunControlStateMachine::FAILED_STATE_NAME) == 0 ||
6406  remoteGatewayApp.appInfo.status.find("Error") !=
6407  std::string::npos || // case "Error", "Soft-Error"
6408  (remoteGatewayApp.appInfo.status ==
6409  RunControlStateMachine::HALTED_STATE_NAME &&
6410  command == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)))
6411  {
6412  __COUT__ << "Skipping '" << remoteGatewayApp.getFsmMode()
6413  << "' Remote gateway '" << remoteGatewayApp.appInfo.name
6414  << "' w/status = " << remoteGatewayApp.appInfo.status
6415  << "... for FSM command = " << commandAndParams << __E__;
6416  continue; //skip if not included
6417  }
6418 
6419  __COUT__ << "Launching FSM command '" << commandAndParams
6420  << "' on Remote gateway '" << remoteGatewayApp.appInfo.name << "'..."
6421  << __E__;
6422 
6423  if(remoteGatewayApp.command != "")
6424  {
6425  __SUP_SS__ << "Can not target the remote subsystem '"
6426  << remoteGatewayApp.appInfo.name << "' with command '" << command
6427  << "' which already has a pending command '"
6428  << remoteGatewayApp.command
6429  << ".' Please try again after the pending command is sent."
6430  << __E__;
6431  __SUP_SS_THROW__;
6432  }
6433 
6434  remoteGatewayApp.config_dump = ""; //clear, must come from new command completion
6435  remoteGatewayApp.command = commandAndParams;
6436 
6437  std::string logEntry = getLastLogEntry(command);
6438  if(logEntry.size())
6439  remoteGatewayApp.command +=
6440  ",LogEntry:" + StringMacros::encodeURIComponent(logEntry);
6441  remoteGatewayApp.fsmName =
6442  activeStateMachineName_; //fsmName will be prepended during command send
6443  //force status for immediate user feedback
6444  remoteGatewayApp.appInfo.status = "Launching " + commandAndParams;
6445  remoteGatewayApp.appInfo.progress = 0;
6446  }
6447 } // end broadcastMessageToRemoteGateways()
6448 
6449 //==============================================================================
6450 bool GatewaySupervisor::broadcastMessageToRemoteGatewaysComplete(
6451  const xoap::MessageReference message)
6452 {
6453  std::string command = SOAPUtilities::translate(message).getCommand();
6454  __COUTV__(command);
6455  std::string destinationState = theStateMachine_.getTransitionFinalStateName(command);
6456  __COUTV__(destinationState);
6457 
6458  size_t countOfRemoteGateways = 0;
6459 
6460  bool done = command == "Error"; //dont check for done if Error'ing
6461  while(!done)
6462  {
6463  __COUT__ << "Checking " << remoteGatewayApps_.size()
6464  << " remote gateway(s) completion for command = " << command << __E__;
6465 
6466  done = true;
6467 
6468  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGatewayApps; //local copy
6469  { //lock for remainder of scope
6470  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
6471  __SUP_COUTVS__(22, remoteGatewayApps_.size());
6472  countOfRemoteGateways = remoteGatewayApps_.size();
6473  remoteGatewayApps = remoteGatewayApps_;
6474  if(remoteGatewayApps_.size())
6475  __SUP_COUT_TYPE__(TLVL_DEBUG + 22)
6476  << __COUT_HDR__ << remoteGatewayApps_[0].command << " "
6477  << (remoteGatewayApps_[0].appInfo.status) << __E__;
6478  }
6479 
6480  for(auto& remoteGatewayApp : remoteGatewayApps)
6481  {
6482  //skip remote gateways that were not commanded
6483  if(!remoteGatewayApp.fsm_included)
6484  continue;
6485  if(remoteGatewayApp.fsm_mode == RemoteGatewayInfo::FSM_ModeTypes::DoNotHalt &&
6486  //do not allow halt/err transitions:
6487  (command == RunControlStateMachine::ERROR_TRANSITION_NAME ||
6488  command == RunControlStateMachine::FAIL_TRANSITION_NAME ||
6489  command == RunControlStateMachine::HALT_TRANSITION_NAME ||
6490  command == RunControlStateMachine::ABORT_TRANSITION_NAME))
6491  continue;
6492  if(remoteGatewayApp.fsm_mode ==
6493  RemoteGatewayInfo::FSM_ModeTypes::OnlyConfigure &&
6494  ! //invert of allowed situations:
6495  (remoteGatewayApp.appInfo.status ==
6496  RunControlStateMachine::INITIAL_STATE_NAME ||
6497  remoteGatewayApp.appInfo.status ==
6498  RunControlStateMachine::HALTED_STATE_NAME ||
6499  remoteGatewayApp.appInfo.status.find(
6500  RunControlStateMachine::FAILED_STATE_NAME) == 0 ||
6501  remoteGatewayApp.appInfo.status.find("Error") !=
6502  std::string::npos || // case "Error", "Soft-Error"
6503  (remoteGatewayApp.appInfo.status ==
6504  RunControlStateMachine::HALTED_STATE_NAME &&
6505  command == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)))
6506  continue;
6507  //if here, was commanded, so check status
6508 
6509  if(!((remoteGatewayApp.appInfo.status == destinationState &&
6510  remoteGatewayApp.appInfo.progress == 100) ||
6511  remoteGatewayApp.appInfo.status.find("Error") != std::string::npos ||
6512  remoteGatewayApp.appInfo.status.find("Fail") != std::string::npos))
6513  {
6514  //not done
6515  if(remoteGatewayApp.appInfo.status == SupervisorInfo::APP_STATUS_UNKNOWN)
6516  {
6517  __SS__ << "Can not complete FSM command '" << command
6518  << "' with unknown status from Remote gateway '"
6519  << remoteGatewayApp.appInfo.name
6520  << "' - it seems communication was lost. Please check the "
6521  "connection or notify admins."
6522  << __E__;
6523  __SS_THROW__;
6524  }
6525  __COUT__ << "Remote gateway '" << remoteGatewayApp.appInfo.name
6526  << "' not done w/command '" << command
6527  << "' status = " << remoteGatewayApp.appInfo.status
6528  << ",... progress = " << remoteGatewayApp.appInfo.progress
6529  << __E__;
6530 
6531  done = false;
6532  }
6533  else
6534  {
6535  //done
6536  __COUTT__ << "Done Remote gateway '" << remoteGatewayApp.appInfo.name
6537  << "' w/command '" << command
6538  << "' status = " << remoteGatewayApp.appInfo.status
6539  << ",... progress = " << remoteGatewayApp.appInfo.progress
6540  << __E__;
6541  }
6542  }
6543  if(!done)
6544  sleep(2);
6545 
6546  checkForAsyncError();
6547  }
6548 
6549  __COUT__ << "Done with " << countOfRemoteGateways
6550  << " remote gateway(s) command = " << command << __E__;
6551 
6552  return true;
6553 } // end broadcastMessageToRemoteGatewaysComplete()
6554 
6555 //==============================================================================
6560 void GatewaySupervisor::loginRequest(xgi::Input* in, xgi::Output* out)
6561 {
6562  std::chrono::steady_clock::time_point startClock = std::chrono::steady_clock::now();
6563  cgicc::Cgicc cgi(in);
6564  std::string Command = CgiDataUtilities::getData(cgi, "RequestType");
6565  __COUT__ << "*** Login RequestType = " << Command << " time=" << time(0) << __E__;
6566 
6567  // RequestType Commands:
6568  // login
6569  // sessionId
6570  // checkCookie
6571  // logout
6572 
6573  try
6574  {
6575  // always cleanup expired entries and get a vector std::string of logged out users
6576  std::vector<std::string> loggedOutUsernames;
6577  theWebUsers_.cleanupExpiredEntries(&loggedOutUsernames);
6578  bool doLog = false;
6579  if(loggedOutUsernames.size())
6580  {
6581  try
6582  {
6583  doLog = __ENV__("OTS_LOG_LOGIN_LOGOUT") == std::string("1");
6584  }
6585  catch(...)
6586  { /* ignore errors */
6587  ;
6588  }
6589  }
6590  for(unsigned int i = 0; i < loggedOutUsernames.size();
6591  ++i) // Log logout for logged out users
6592  if(doLog)
6593  makeSystemLogEntry(loggedOutUsernames[i] + " login timed out.");
6594 
6595  if(Command == "sessionId")
6596  {
6597  // When client loads page, client submits unique user id and receives random
6598  // sessionId from server Whenever client submits user name and password it is
6599  // jumbled by sessionId when sent to server and sent along with UUID. Server uses
6600  // sessionId to unjumble.
6601  //
6602  // Server maintains list of active sessionId by UUID
6603  // sessionId expires after set time if no login attempt (e.g. 5 minutes)
6604  std::string uuid = CgiDataUtilities::postData(cgi, "uuid");
6605 
6606  std::string sid = theWebUsers_.createNewLoginSession(
6607  uuid, cgi.getEnvironment().getRemoteAddr() /* ip */);
6608 
6609  // __COUT__ << "uuid = " << uuid << __E__;
6610  // __COUT__ << "SessionId = " << sid.substr(0, 10) << __E__;
6611  *out << sid;
6612  }
6613  else if(Command == "checkCookie")
6614  {
6615  uint64_t uid;
6616  std::string uuid;
6617  std::string jumbledUser;
6618  std::string cookieCode;
6619 
6620  // If client has a cookie, client submits cookie and username, jumbled, to see if
6621  // cookie and user are still active if active, valid cookie code is returned
6622  // and name to display, in XML
6623  // if not, return 0
6624  // params:
6625  // uuid - unique user id, to look up sessionId
6626  // ju - jumbled user name
6627  // CookieCode - cookie code to check
6628 
6629  uuid = CgiDataUtilities::postData(cgi, "uuid");
6630  jumbledUser = CgiDataUtilities::postData(cgi, "ju");
6631  cookieCode = CgiDataUtilities::postData(cgi, "cc");
6632 
6633  // __COUT__ << "uuid = " << uuid << __E__;
6634  // __COUT__ << "Cookie Code = " << cookieCode.substr(0, 10) << __E__;
6635  // __COUT__ << "jumbledUser = " << jumbledUser.substr(0, 10) << __E__;
6636 
6637  // If cookie code is good, then refresh and return with display name, else return
6638  // 0 as CookieCode value
6639  uid = theWebUsers_.isCookieCodeActiveForLogin(
6640  uuid,
6641  cookieCode,
6642  jumbledUser); // after call jumbledUser holds displayName on success
6643 
6644  if(uid == theWebUsers_.NOT_FOUND_IN_DATABASE)
6645  {
6646  __COUT__ << "cookieCode invalid" << __E__;
6647  jumbledUser = ""; // clear display name if failure
6648  cookieCode = "0"; // clear cookie code if failure
6649  }
6650  else
6651  __COUT__ << "cookieCode is good." << __E__;
6652 
6653  // return xml holding cookie code and display name
6654  HttpXmlDocument xmldoc(cookieCode, jumbledUser);
6655 
6656  theWebUsers_.insertSettingsForUser(uid, &xmldoc); // insert settings
6657 
6658  xmldoc.outputXmlDocument((std::ostringstream*)out);
6659  }
6660  else if(Command == "login")
6661  {
6662  // If login attempt or create account, jumbled user and pw are submitted
6663  // if successful, valid cookie code and display name returned.
6664  // if not, return 0
6665  // params:
6666  // uuid - unique user id, to look up sessionId
6667  // nac - new account code for first time logins
6668  // ju - jumbled user name
6669  // jp - jumbled password
6670 
6671  std::string uuid = CgiDataUtilities::postData(cgi, "uuid");
6672  std::string newAccountCode = CgiDataUtilities::postData(cgi, "nac");
6673  std::string jumbledUser = CgiDataUtilities::postData(cgi, "ju");
6674  std::string jumbledPw = CgiDataUtilities::postData(cgi, "jp");
6675 
6676  // __COUT__ << "jumbledUser = " << jumbledUser.substr(0, 10) << __E__;
6677  // __COUT__ << "jumbledPw = " << jumbledPw.substr(0, 10) << __E__;
6678  // __COUT__ << "uuid = " << uuid << __E__;
6679  // __COUT__ << "nac =-" << newAccountCode << "-" << __E__;
6680 
6681  uint64_t uid = theWebUsers_.attemptActiveSession(
6682  uuid,
6683  jumbledUser,
6684  jumbledPw,
6685  newAccountCode,
6686  cgi.getEnvironment()
6687  .getRemoteAddr()); // after call jumbledUser holds displayName on success
6688 
6689  if(uid >= theWebUsers_.ACCOUNT_ERROR_THRESHOLD)
6690  {
6691  __COUT__ << "Login invalid." << __E__;
6692  jumbledUser = ""; // clear display name if failure
6693  if(newAccountCode != "1") // indicates uuid not found
6694  newAccountCode = "0"; // clear cookie code if failure
6695  }
6696  else // Log login in logbook for active experiment
6697  {
6698  bool doLog = false;
6699  try
6700  {
6701  doLog = __ENV__("OTS_LOG_LOGIN_LOGOUT") == std::string("1");
6702  }
6703  catch(...)
6704  { /* ignore errors */
6705  ;
6706  }
6707 
6708  if(doLog)
6709  makeSystemLogEntry(theWebUsers_.getUsersUsername(uid) +
6710  " logged in.");
6711  }
6712 
6713  //__COUT__ << "new cookieCode = " << newAccountCode.substr(0, 10) << __E__;
6714 
6715  HttpXmlDocument xmldoc(newAccountCode, jumbledUser);
6716 
6717  // include extra error detail
6718  if(uid == theWebUsers_.ACCOUNT_INACTIVE)
6719  xmldoc.addTextElementToData("Error",
6720  "Account is inactive. Notify admins.");
6721  else if(uid == theWebUsers_.ACCOUNT_BLACKLISTED)
6722  xmldoc.addTextElementToData("Error",
6723  "Account is blacklisted. Notify admins.");
6724 
6725  theWebUsers_.insertSettingsForUser(uid, &xmldoc); // insert settings
6726 
6727  // insert active session count for user
6728 
6729  if(uid != theWebUsers_.NOT_FOUND_IN_DATABASE)
6730  {
6731  uint64_t asCnt =
6732  theWebUsers_.getActiveSessionCountForUser(uid) -
6733  1; // subtract 1 to remove just started session from count
6734  char asStr[20];
6735  sprintf(asStr, "%lu", asCnt);
6736  xmldoc.addTextElementToData("user_active_session_count", asStr);
6737  }
6738 
6739  xmldoc.outputXmlDocument((std::ostringstream*)out);
6740  }
6741  else if(Command == "cert")
6742  {
6743  // If login attempt or create account, jumbled user and pw are submitted
6744  // if successful, valid cookie code and display name returned.
6745  // if not, return 0
6746  // params:
6747  // uuid - unique user id, to look up sessionId
6748  // nac - new account code for first time logins
6749  // ju - jumbled user name
6750  // jp - jumbled password
6751 
6752  std::string uuid = CgiDataUtilities::postData(cgi, "uuid");
6753  std::string jumbledEmail =
6754  cgicc::form_urldecode(CgiDataUtilities::getData(cgi, "httpsUser"));
6755  std::string username = "";
6756  std::string cookieCode = "";
6757 
6758  // __COUT__ << "CERTIFICATE LOGIN REUEST RECEVIED!!!" << __E__;
6759  // __COUT__ << "jumbledEmail = " << jumbledEmail << __E__;
6760  // __COUT__ << "uuid = " << uuid << __E__;
6761 
6762  uint64_t uid = theWebUsers_.attemptActiveSessionWithCert(
6763  uuid,
6764  jumbledEmail,
6765  cookieCode,
6766  username,
6767  cgi.getEnvironment()
6768  .getRemoteAddr()); // after call jumbledUser holds displayName on success
6769 
6770  if(uid == theWebUsers_.NOT_FOUND_IN_DATABASE)
6771  {
6772  __COUT__ << "cookieCode invalid" << __E__;
6773  jumbledEmail = ""; // clear display name if failure
6774  if(cookieCode != "1") // indicates uuid not found
6775  cookieCode = "0"; // clear cookie code if failure
6776  }
6777  else // Log login in logbook for active experiment
6778  {
6779  bool doLog = false;
6780  try
6781  {
6782  doLog = __ENV__("OTS_LOG_LOGIN_LOGOUT") == std::string("1");
6783  }
6784  catch(...)
6785  { /* ignore errors */
6786  ;
6787  }
6788 
6789  if(doLog)
6790  makeSystemLogEntry(theWebUsers_.getUsersUsername(uid) +
6791  " logged in.");
6792  }
6793 
6794  //__COUT__ << "new cookieCode = " << cookieCode.substr(0, 10) << __E__;
6795 
6796  HttpXmlDocument xmldoc(cookieCode, jumbledEmail);
6797 
6798  theWebUsers_.insertSettingsForUser(uid, &xmldoc); // insert settings
6799 
6800  // insert active session count for user
6801 
6802  if(uid != theWebUsers_.NOT_FOUND_IN_DATABASE)
6803  {
6804  uint64_t asCnt =
6805  theWebUsers_.getActiveSessionCountForUser(uid) -
6806  1; // subtract 1 to remove just started session from count
6807  char asStr[20];
6808  sprintf(asStr, "%lu", asCnt);
6809  xmldoc.addTextElementToData("user_active_session_count", asStr);
6810  }
6811 
6812  xmldoc.outputXmlDocument((std::ostringstream*)out);
6813  }
6814  else if(Command == "logout")
6815  {
6816  std::string cookieCode = CgiDataUtilities::postData(cgi, "CookieCode");
6817  std::string logoutOthers = CgiDataUtilities::postData(cgi, "LogoutOthers");
6818 
6819  // __COUT__ << "Cookie Code = " << cookieCode.substr(0, 10) << __E__;
6820  // __COUT__ << "logoutOthers = " << logoutOthers << __E__;
6821 
6822  uint64_t uid; // get uid for possible system logbook message
6823  if(theWebUsers_.cookieCodeLogout(cookieCode,
6824  logoutOthers == "1",
6825  &uid,
6826  cgi.getEnvironment().getRemoteAddr()) !=
6827  theWebUsers_.NOT_FOUND_IN_DATABASE) // user logout
6828  {
6829  // if did some logging out, check if completely logged out
6830  // if so, system logbook message should be made.
6831  if(!theWebUsers_.isUserIdActive(uid))
6832  {
6833  bool doLog = false;
6834  try
6835  {
6836  doLog = __ENV__("OTS_LOG_LOGIN_LOGOUT") == std::string("1");
6837  }
6838  catch(...)
6839  { /* ignore errors */
6840  ;
6841  }
6842 
6843  if(doLog)
6844  makeSystemLogEntry(theWebUsers_.getUsersUsername(uid) +
6845  " logged out.");
6846  }
6847  }
6848  }
6849  else
6850  {
6851  __COUT_WARN__ << "Invalid Command" << __E__;
6852  *out << "0";
6853  }
6854  }
6855  catch(const std::runtime_error& e)
6856  {
6857  __SS__ << "An error was encountered handling Command '" << Command
6858  << "':" << e.what() << __E__;
6859  __COUT__ << "\n" << ss.str();
6860  HttpXmlDocument xmldoc;
6861  xmldoc.addTextElementToData("Error", ss.str());
6862  xmldoc.outputXmlDocument(
6863  (std::ostringstream*)out, false /*dispStdOut*/, true /*allowWhiteSpace*/);
6864  }
6865  catch(...)
6866  {
6867  __SS__ << "An unknown error was encountered handling Command '" << Command
6868  << ".' "
6869  << "Please check the printouts to debug." << __E__;
6870  try
6871  {
6872  throw;
6873  } //one more try to printout extra info
6874  catch(const std::exception& e)
6875  {
6876  ss << "Exception message: " << e.what();
6877  }
6878  catch(...)
6879  {
6880  }
6881  __COUT__ << "\n" << ss.str();
6882  HttpXmlDocument xmldoc;
6883  xmldoc.addTextElementToData("Error", ss.str());
6884  xmldoc.outputXmlDocument(
6885  (std::ostringstream*)out, false /*dispStdOut*/, true /*allowWhiteSpace*/);
6886  }
6887 
6888  __COUTT__ << "Login end clock=" << artdaq::TimeUtils::GetElapsedTime(startClock)
6889  << __E__;
6890 } // end loginRequest()
6891 
6892 //==============================================================================
6893 void GatewaySupervisor::tooltipRequest(xgi::Input* in, xgi::Output* out)
6894 {
6895  cgicc::Cgicc cgi(in);
6896 
6897  std::string Command = CgiDataUtilities::getData(cgi, "RequestType");
6898  __COUTT__ << "Tooltip RequestType = " << Command << __E__;
6899 
6900  try
6901  {
6902  //**** start LOGIN GATEWAY CODE ***//
6903  // If TRUE, cookie code is good, and refreshed code is in cookieCode, also pointers
6904  // optionally for uint8_t userPermissions, uint64_t uid Else, error message is
6905  // returned in cookieCode Notes: cookie code not refreshed if RequestType is in AutomatedRequestTypes
6906  std::string cookieCode = CgiDataUtilities::postData(cgi, "CookieCode");
6907  uint64_t uid;
6908 
6909  if(!theWebUsers_.cookieCodeIsActiveForRequest(cookieCode,
6910  0 /*userPermissions*/,
6911  &uid,
6912  "0" /*dummy ip*/,
6913  false /*refresh*/,
6914  true /*doNotGoRemote*/))
6915  {
6916  *out << cookieCode;
6917  return;
6918  }
6919 
6920  //**** end LOGIN GATEWAY CODE ***//
6921 
6922  HttpXmlDocument xmldoc(cookieCode);
6923 
6924  if(Command == "check")
6925  {
6926  WebUsers::tooltipCheckForUsername(theWebUsers_.getUsersUsername(uid),
6927  &xmldoc,
6928  CgiDataUtilities::getData(cgi, "srcFile"),
6929  CgiDataUtilities::getData(cgi, "srcFunc"),
6930  CgiDataUtilities::getData(cgi, "srcId"));
6931  }
6932  else if(Command == "setNeverShow")
6933  {
6934  WebUsers::tooltipSetNeverShowForUsername(
6935  theWebUsers_.getUsersUsername(uid),
6936  &xmldoc,
6937  CgiDataUtilities::getData(cgi, "srcFile"),
6938  CgiDataUtilities::getData(cgi, "srcFunc"),
6939  CgiDataUtilities::getData(cgi, "srcId"),
6940  CgiDataUtilities::getData(cgi, "doNeverShow") == "1" ? true : false,
6941  CgiDataUtilities::getData(cgi, "temporarySilence") == "1" ? true : false);
6942  }
6943  else
6944  __COUT__ << "Command Request, " << Command << ", not recognized." << __E__;
6945 
6946  xmldoc.outputXmlDocument((std::ostringstream*)out, false, true);
6947  }
6948  catch(const std::runtime_error& e)
6949  {
6950  __SS__ << "An error was encountered handling Tooltip Command '" << Command
6951  << "':" << e.what() << __E__;
6952  __COUT__ << "\n" << ss.str();
6953  HttpXmlDocument xmldoc;
6954  xmldoc.addTextElementToData("Error", ss.str());
6955  xmldoc.outputXmlDocument(
6956  (std::ostringstream*)out, false /*dispStdOut*/, true /*allowWhiteSpace*/);
6957  }
6958  catch(...)
6959  {
6960  __SS__ << "An unknown error was encountered handling Tooltip Command '" << Command
6961  << ".' "
6962  << "Please check the printouts to debug." << __E__;
6963  try
6964  {
6965  throw;
6966  } //one more try to printout extra info
6967  catch(const std::exception& e)
6968  {
6969  ss << "Exception message: " << e.what();
6970  }
6971  catch(...)
6972  {
6973  }
6974  __COUT__ << "\n" << ss.str();
6975  HttpXmlDocument xmldoc;
6976  xmldoc.addTextElementToData("Error", ss.str());
6977  xmldoc.outputXmlDocument(
6978  (std::ostringstream*)out, false /*dispStdOut*/, true /*allowWhiteSpace*/);
6979  }
6980 
6981  //__COUT__ << "Done" << __E__;
6982 } // end tooltipRequest()
6983 
6984 //==============================================================================
6988 void GatewaySupervisor::setSupervisorPropertyDefaults()
6989 {
6990  CorePropertySupervisorBase::setSupervisorProperty(
6991  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.UserPermissionsThreshold,
6992  std::string() +
6993  "*=1 | gatewayLaunchOTS=-1 | gatewayLaunchWiz=-1"
6994  " | gatewayLaunchOTSInstance=-1"
6995  " | StateMachine-*=10" //state machine transitions through stateMachineXgiHandler
6996  " | cancelStateMachineTransition=10"
6997  " | resetConsoleCounts=10"
6998  " | commandRemoteSubsystem=10 | setRemoteSubsystemFsmControl=10" //remote subsystem control
6999  );
7000 
7001  CorePropertySupervisorBase::setSupervisorProperty(
7002  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AllowNoLoginRequestTypes,
7003  "getCurrentState "
7004  " | getAppStatus | getRemoteSubsystems | getRemoteSubsystemStatus");
7005 } // end setSupervisorPropertyDefaults()
7006 
7007 //==============================================================================
7010 void GatewaySupervisor::forceSupervisorPropertyValues()
7011 {
7012  // note used by these handlers:
7013  // request()
7014  // stateMachineXgiHandler() -- prepend StateMachine to request type
7015 
7016  CorePropertySupervisorBase::setSupervisorProperty(
7017  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
7018  "getSystemMessages | getCurrentState | getIterationPlanStatus"
7019  " | getAppStatus | getRemoteSubsystems | getRemoteSubsystemStatus");
7020  CorePropertySupervisorBase::addSupervisorProperty(
7021  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.RequireUserLockRequestTypes,
7022  "gatewayLaunchOTS | gatewayLaunchWiz | gatewayLaunchOTSInstance"
7023  " | commandRemoteSubsystem");
7024  CorePropertySupervisorBase::addSupervisorProperty(
7025  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.CheckUserLockRequestTypes,
7026  "StateMachine-*"); //for all stateMachineXgiHandler requests
7027 
7028  if(readOnly_)
7029  {
7030  CorePropertySupervisorBase::setSupervisorProperty(
7031  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.UserPermissionsThreshold,
7032  "*=0 | getSystemMessages=1 | getDesktopIcons=1"); // block users from writing if no write access
7033  __COUT_INFO__ << "readOnly true in setSupervisorProperty" << __E__;
7034  }
7035 } // end forceSupervisorPropertyValues()
7036 
7037 //==============================================================================
7038 void GatewaySupervisor::request(xgi::Input* in, xgi::Output* out)
7039 try
7040 {
7041  out->getHTTPResponseHeader().addHeader(
7042  "Access-Control-Allow-Origin",
7043  "*"); // to avoid block by blocked by CORS policy of browser
7044 
7045  // for simplicity assume all commands should be mutually exclusive with iterator
7046  // thread state machine accesses (really should just be careful with
7047  // RunControlStateMachine access)
7048  if(VERBOSE_MUTEX)
7049  __COUT__ << "Waiting for FSM access" << __E__;
7050  std::lock_guard<std::mutex> lock(stateMachineAccessMutex_);
7051  if(VERBOSE_MUTEX)
7052  __COUT__ << "Have FSM access" << __E__;
7053 
7054  cgicc::Cgicc cgiIn(in);
7055 
7056  std::string requestType = CgiDataUtilities::getData(cgiIn, "RequestType");
7057  __COUTVS__(40, requestType);
7058 
7059  HttpXmlDocument xmlOut;
7060  WebUsers::RequestUserInfo userInfo(requestType,
7061  CgiDataUtilities::postData(cgiIn, "CookieCode"));
7062 
7063  CorePropertySupervisorBase::getRequestUserInfo(userInfo);
7064 
7065  if(!theWebUsers_.xmlRequestOnGateway(cgiIn, out, &xmlOut, userInfo))
7066  return; // access failed
7067 
7068  // RequestType Commands:
7069  // getSettings
7070  // setSettings
7071  // accountSettings
7072  // getAliasList
7073  // getAppStatus
7074  // getAppId
7075  // getContextMemberNames
7076  // getSystemMessages
7077  // setUserWithLock
7078  // getStateMachine
7079  // getStateMachineLastLogEntry
7080  // stateMatchinePreferences
7081  // getStateMachineNames
7082  // getCurrentState
7083  // cancelStateMachineTransition
7084  // getIterationPlanStatus
7085  // getErrorInStateMatchine
7086 
7087  // getDesktopIcons
7088  // addDesktopIcon
7089 
7090  // resetConsoleCounts
7091 
7092  // getRemoteSubsystems
7093  // getRemoteSubsystemStatus
7094  // commandRemoteSubsystem
7095  // setRemoteSubsystemFsmControl
7096  // getSubsystemConfigAliasSelectInfo
7097 
7098  // resetUserTooltips
7099  // silenceAllUserTooltips
7100 
7101  // gatewayLaunchOTS
7102  // gatewayLaunchWiz
7103  // gatewayLaunchOTSInstance
7104 
7105  if(0) // leave for debugging
7106  {
7107  ConfigurationTree configLinkNode =
7108  CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(
7109  supervisorContextUID_, supervisorApplicationUID_);
7110 
7111  ConfigurationTree fsmLinkNode = configLinkNode.getNode("LinkToStateMachineTable");
7112 
7113  __COUT__ << "requestType " << requestType << " v"
7114  << (fsmLinkNode.getTableVersion()) << __E__;
7115  }
7116 
7117  try
7118  {
7119  if(requestType == "getSettings")
7120  {
7121  std::string accounts = CgiDataUtilities::getData(cgiIn, "accounts");
7122 
7123  __COUT__ << "Get Settings Request" << __E__;
7124  __COUT__ << "accounts = " << accounts << __E__;
7125  theWebUsers_.insertSettingsForUser(userInfo.uid_, &xmlOut, accounts == "1");
7126  }
7127  else if(requestType == "setSettings")
7128  {
7129  std::string bgcolor = CgiDataUtilities::postData(cgiIn, "bgcolor");
7130  std::string dbcolor = CgiDataUtilities::postData(cgiIn, "dbcolor");
7131  std::string wincolor = CgiDataUtilities::postData(cgiIn, "wincolor");
7132  std::string layout = CgiDataUtilities::postData(cgiIn, "layout");
7133  std::string syslayout = CgiDataUtilities::postData(cgiIn, "syslayout");
7134 
7135  __COUT__ << "Set Settings Request" << __E__;
7136  __COUT__ << "bgcolor = " << bgcolor << __E__;
7137  __COUT__ << "dbcolor = " << dbcolor << __E__;
7138  __COUT__ << "wincolor = " << wincolor << __E__;
7139  __COUT__ << "layout = " << layout << __E__;
7140  __COUT__ << "syslayout = " << syslayout << __E__;
7141 
7142  theWebUsers_.changeSettingsForUser(
7143  userInfo.uid_, bgcolor, dbcolor, wincolor, layout, syslayout);
7144  theWebUsers_.insertSettingsForUser(
7145  userInfo.uid_, &xmlOut, true); // include user accounts
7146  }
7147  else if(requestType == "accountSettings")
7148  {
7149  std::string type = CgiDataUtilities::postData(
7150  cgiIn, "type"); // updateAccount, createAccount, deleteAccount
7151  int type_int = -1;
7152 
7153  if(type == "updateAccount")
7154  type_int = theWebUsers_.MOD_TYPE_UPDATE;
7155  else if(type == "createAccount")
7156  type_int = theWebUsers_.MOD_TYPE_ADD;
7157  else if(type == "deleteAccount")
7158  type_int = theWebUsers_.MOD_TYPE_DELETE;
7159 
7160  std::string username = CgiDataUtilities::postData(cgiIn, "username");
7161  std::string displayname = CgiDataUtilities::postData(cgiIn, "displayname");
7162  std::string email = CgiDataUtilities::postData(cgiIn, "useremail");
7163  std::string permissions = CgiDataUtilities::postData(cgiIn, "permissions");
7164  std::string accounts = CgiDataUtilities::getData(cgiIn, "accounts");
7165 
7166  __COUT__ << "accountSettings Request" << __E__;
7167  __COUT__ << "type = " << type << " - " << type_int << __E__;
7168  __COUT__ << "username = " << username << __E__;
7169  __COUT__ << "useremail = " << email << __E__;
7170  __COUT__ << "displayname = " << displayname << __E__;
7171  __COUT__ << "permissions = " << permissions << __E__;
7172 
7173  theWebUsers_.modifyAccountSettings(
7174  userInfo.uid_, type_int, username, displayname, email, permissions);
7175 
7176  __COUT__ << "accounts = " << accounts << __E__;
7177 
7178  theWebUsers_.insertSettingsForUser(userInfo.uid_, &xmlOut, accounts == "1");
7179  }
7180  else if(requestType == "stateMatchinePreferences")
7181  {
7182  std::string set = CgiDataUtilities::getData(cgiIn, "set");
7183  const std::string DEFAULT_FSM_VIEW = "Default_FSM_View";
7184  if(set == "1")
7185  theWebUsers_.setGenericPreference(
7186  userInfo.uid_,
7187  DEFAULT_FSM_VIEW,
7188  CgiDataUtilities::getData(cgiIn, DEFAULT_FSM_VIEW));
7189  else
7190  theWebUsers_.getGenericPreference(
7191  userInfo.uid_, DEFAULT_FSM_VIEW, &xmlOut);
7192  }
7193  else if(requestType == "getAliasList")
7194  {
7195  // std::string username = userInfo.username_;
7196  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
7197  __SUP_COUTV__(fsmName);
7198 
7199  addFilteredConfigAliasesToXML(xmlOut, fsmName);
7200  if(0)
7201  {
7202  std::string stateMachineAliasFilter = "*"; // default to all
7203 
7204  // IMPORTANT -- use temporary ConfigurationManager to get the Active Group Aliases,
7205  // to avoid changing the Context Configuration tree for the Gateway Supervisor
7206  ConfigurationManager temporaryConfigMgr;
7207  std::map<std::string /*alias*/,
7208  std::pair<std::string /*group name*/, TableGroupKey>>
7209  aliasMap;
7210  aliasMap = temporaryConfigMgr.getActiveGroupAliases();
7211 
7212  // AND IMPORTANT -- to use ConfigurationManager to get the Context settings for the Gateway Supervisor
7213  // get stateMachineAliasFilter if possible
7214  ConfigurationTree configLinkNode =
7215  CorePropertySupervisorBase::theConfigurationManager_
7216  ->getSupervisorTableNode(supervisorContextUID_,
7217  supervisorApplicationUID_);
7218 
7219  if(!configLinkNode.isDisconnected())
7220  {
7221  try // for backwards compatibility
7222  {
7223  ConfigurationTree fsmLinkNode =
7224  configLinkNode.getNode("LinkToStateMachineTable");
7225  if(!fsmLinkNode.isDisconnected() &&
7226  !fsmLinkNode.getNode(fsmName + "/SystemAliasFilter")
7227  .isDefaultValue())
7228  stateMachineAliasFilter =
7229  fsmLinkNode.getNode(fsmName + "/SystemAliasFilter")
7230  .getValue<std::string>();
7231  else
7232  __COUT_INFO__ << "FSM Link disconnected." << __E__;
7233  }
7234  catch(std::runtime_error& e)
7235  {
7236  __COUT_INFO__ << e.what() << __E__;
7237  }
7238  catch(...)
7239  {
7240  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
7241  }
7242  }
7243  else
7244  __COUT_INFO__ << "FSM Link disconnected." << __E__;
7245 
7246  __COUT__ << "For FSM '" << fsmName
7247  << ",' stateMachineAliasFilter = " << stateMachineAliasFilter
7248  << __E__;
7249 
7250  // filter list of aliases based on stateMachineAliasFilter
7251  // ! as first character means choose those that do NOT match filter
7252  // * can be used as wild card.
7253  {
7254  bool invertFilter = stateMachineAliasFilter.size() &&
7255  stateMachineAliasFilter[0] == '!';
7256  std::vector<std::string> filterArr;
7257 
7258  size_t i = 0;
7259  if(invertFilter)
7260  ++i;
7261  size_t f;
7262  std::string tmp;
7263  while((f = stateMachineAliasFilter.find('*', i)) != std::string::npos)
7264  {
7265  tmp = stateMachineAliasFilter.substr(i, f - i);
7266  i = f + 1;
7267  filterArr.push_back(tmp);
7268  //__COUT__ << filterArr[filterArr.size()-1] << " " << i <<
7269  // " of " << stateMachineAliasFilter.size() << __E__;
7270  }
7271  if(i <= stateMachineAliasFilter.size())
7272  {
7273  tmp = stateMachineAliasFilter.substr(i);
7274  filterArr.push_back(tmp);
7275  //__COUT__ << filterArr[filterArr.size()-1] << " last." << __E__;
7276  }
7277 
7278  bool filterMatch;
7279 
7280  for(auto& aliasMapPair : aliasMap)
7281  {
7282  //__COUT__ << "aliasMapPair.first: " << aliasMapPair.first << __E__;
7283 
7284  filterMatch = true;
7285 
7286  if(filterArr.size() == 1)
7287  {
7288  if(filterArr[0] != "" && filterArr[0] != "*" &&
7289  aliasMapPair.first != filterArr[0])
7290  filterMatch = false;
7291  }
7292  else
7293  {
7294  i = -1;
7295  for(f = 0; f < filterArr.size(); ++f)
7296  {
7297  if(!filterArr[f].size())
7298  continue; // skip empty filters
7299 
7300  if(f == 0) // must start with this filter
7301  {
7302  if((i = aliasMapPair.first.find(filterArr[f])) != 0)
7303  {
7304  filterMatch = false;
7305  break;
7306  }
7307  }
7308  else if(f == filterArr.size() -
7309  1) // must end with this filter
7310  {
7311  if(aliasMapPair.first.rfind(filterArr[f]) !=
7312  aliasMapPair.first.size() - filterArr[f].size())
7313  {
7314  filterMatch = false;
7315  break;
7316  }
7317  }
7318  else if((i = aliasMapPair.first.find(filterArr[f])) ==
7319  std::string::npos)
7320  {
7321  filterMatch = false;
7322  break;
7323  }
7324  }
7325  }
7326 
7327  if(invertFilter)
7328  filterMatch = !filterMatch;
7329 
7330  //__COUT__ << "filterMatch=" << filterMatch << __E__;
7331 
7332  if(!filterMatch)
7333  continue;
7334 
7335  xmlOut.addTextElementToData("config_alias", aliasMapPair.first);
7336  xmlOut.addTextElementToData(
7337  "config_key",
7338  TableGroupKey::getFullGroupString(aliasMapPair.second.first,
7339  aliasMapPair.second.second,
7340  /*decorate as (<key>)*/ "(",
7341  ")"));
7342 
7343  // __COUT__ << "config_alias_comment" << " " << temporaryConfigMgr.getNode(
7344  // ConfigurationManager::GROUP_ALIASES_TABLE_NAME).getNode(aliasMapPair.first).getNode(
7345  // TableViewColumnInfo::COL_NAME_COMMENT).getValue<std::string>() << __E__;
7346  xmlOut.addTextElementToData(
7347  "config_alias_comment",
7348  temporaryConfigMgr
7349  .getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
7350  .getNode(aliasMapPair.first)
7351  .getNode(TableViewColumnInfo::COL_NAME_COMMENT)
7352  .getValue<std::string>());
7353 
7354  std::string groupComment, groupAuthor, groupCreationTime;
7355  try
7356  {
7357  temporaryConfigMgr.loadTableGroup(aliasMapPair.second.first,
7358  aliasMapPair.second.second,
7359  false,
7360  0,
7361  0,
7362  0,
7363  &groupComment,
7364  &groupAuthor,
7365  &groupCreationTime,
7366  true /*doNotLoadMembers*/);
7367 
7368  xmlOut.addTextElementToData("config_comment", groupComment);
7369  xmlOut.addTextElementToData("config_author", groupAuthor);
7370  xmlOut.addTextElementToData("config_create_time",
7371  groupCreationTime);
7372  }
7373  catch(...)
7374  {
7375  __COUT_WARN__ << "Failed to load group metadata." << __E__;
7376  }
7377  }
7378  }
7379 
7380  // return last group alias by user
7381  std::string fn = ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH + "/" +
7382  FSM_LAST_GROUP_ALIAS_FILE_START + fsmName + "." +
7383  FSM_USERS_PREFERENCES_FILETYPE;
7384  __COUT__ << "Load preferences: " << fn << __E__;
7385  FILE* fp = fopen(fn.c_str(), "r");
7386  if(fp)
7387  {
7388  char tmpLastAlias[500];
7389  fscanf(fp, "%*s %s", tmpLastAlias);
7390  __COUT__ << "tmpLastAlias: " << tmpLastAlias << __E__;
7391 
7392  xmlOut.addTextElementToData("UserLastConfigAlias", tmpLastAlias);
7393  fclose(fp);
7394  }
7395  else if(aliasMap.size()) //if not set, return first
7396  xmlOut.addTextElementToData("UserLastConfigAlias",
7397  aliasMap.begin()->first);
7398  } //end if 0
7399  }
7400  else if(requestType == "getAppStatus")
7401  {
7402  for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
7403  {
7404  const auto& appInfo = it.second;
7405 
7406  xmlOut.addTextElementToData("name",
7407  appInfo.getName()); // get application name
7408  xmlOut.addNumberElementToData("id",
7409  appInfo.getId()); // get application id
7410  xmlOut.addTextElementToData("status", appInfo.getStatus()); // get status
7411  xmlOut.addTextElementToData(
7412  "time",
7413  appInfo.getLastStatusTime()
7414  ? StringMacros::getTimestampString(appInfo.getLastStatusTime())
7415  : "0"); // get time stamp
7416  xmlOut.addNumberElementToData(
7417  "stale", time(0) - appInfo.getLastStatusTime()); // time since update
7418  xmlOut.addNumberElementToData("progress",
7419  appInfo.getProgress()); // get progress
7420  xmlOut.addTextElementToData("detail", appInfo.getDetail()); // get detail
7421  xmlOut.addTextElementToData("class",
7422  appInfo.getClass()); // get application class
7423  xmlOut.addTextElementToData("url",
7424  appInfo.getURL()); // get application url
7425  xmlOut.addTextElementToData("context",
7426  appInfo.getContextName()); // get context
7427  auto subappElement = xmlOut.addTextElementToData("subapps", "");
7428  for(auto& subappInfoPair : appInfo.getSubappInfo())
7429  {
7430  xmlOut.addTextElementToParent(
7431  "subapp_name", subappInfoPair.first, subappElement);
7432  xmlOut.addTextElementToParent("subapp_status",
7433  subappInfoPair.second.status,
7434  subappElement); // get status
7435  xmlOut.addTextElementToParent(
7436  "subapp_time",
7437  subappInfoPair.second.lastStatusTime
7438  ? StringMacros::getTimestampString(
7439  subappInfoPair.second.lastStatusTime)
7440  : "0",
7441  subappElement); // get timestamp
7442  xmlOut.addNumberElementToParent(
7443  "subapp_stale",
7444  time(0) - subappInfoPair.second.lastStatusTime,
7445  subappElement); // time since update
7446  xmlOut.addNumberElementToParent("subapp_progress",
7447  subappInfoPair.second.progress,
7448  subappElement); // get progress
7449  xmlOut.addTextElementToParent("subapp_detail",
7450  subappInfoPair.second.detail,
7451  subappElement); // get detail
7452  xmlOut.addTextElementToParent("subapp_url",
7453  subappInfoPair.second.url,
7454  subappElement); // get detail
7455  xmlOut.addNumberElementToParent(
7456  "subapp_id", subappInfoPair.second.id, subappElement); // get url
7457  xmlOut.addTextElementToParent("subapp_class",
7458  subappInfoPair.second.class_name,
7459  subappElement); // get class
7460  }
7461  } //end app info loop
7462 
7463  //also return remote gateways as apps
7464  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteApps; //local copy
7465  { //lock for remainder of scope
7466  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
7467  remoteApps = remoteGatewayApps_;
7468  }
7469 
7470  for(const auto& remoteApp : remoteApps)
7471  {
7472  const auto& appInfo = remoteApp.appInfo;
7473 
7474  //skip if no status (will show up as subapp of Gateway)
7475  if(appInfo.status == SupervisorInfo::APP_STATUS_UNKNOWN)
7476  continue;
7477 
7478  xmlOut.addTextElementToData("name",
7479  appInfo.name); // get application name
7480  xmlOut.addNumberElementToData("id", appInfo.id); // get application id
7481  xmlOut.addTextElementToData("status", appInfo.status); // get status
7482  xmlOut.addTextElementToData(
7483  "time",
7484  appInfo.lastStatusTime
7485  ? StringMacros::getTimestampString(appInfo.lastStatusTime)
7486  : "0"); // get timestamp
7487  xmlOut.addNumberElementToData(
7488  "stale",
7489  time(0) - appInfo.lastStatusTime); // time since update
7490  xmlOut.addNumberElementToData("progress",
7491  appInfo.progress); // get progress
7492  xmlOut.addTextElementToData("detail", appInfo.detail); // get detail
7493  xmlOut.addTextElementToData("class",
7494  appInfo.class_name); // get application class
7495  xmlOut.addTextElementToData("url",
7496  appInfo.parent_url); // get application url
7497  xmlOut.addTextElementToData(
7498  "context", appInfo.name + " at " + appInfo.url); // get context
7499  auto subappElement = xmlOut.addTextElementToData("subapps", "");
7500  for(auto& subappInfoPair : remoteApp.subapps)
7501  {
7502  xmlOut.addTextElementToParent(
7503  "subapp_name", subappInfoPair.first, subappElement);
7504  xmlOut.addTextElementToParent("subapp_status",
7505  subappInfoPair.second.status,
7506  subappElement); // get status
7507  xmlOut.addTextElementToParent(
7508  "subapp_time",
7509  subappInfoPair.second.lastStatusTime
7510  ? StringMacros::getTimestampString(
7511  subappInfoPair.second.lastStatusTime)
7512  : "0",
7513  subappElement); // get time stamp
7514  xmlOut.addNumberElementToParent(
7515  "subapp_stale",
7516  time(0) - subappInfoPair.second.lastStatusTime,
7517  subappElement); // time since update
7518  xmlOut.addNumberElementToParent("subapp_progress",
7519  subappInfoPair.second.progress,
7520  subappElement); // get progress
7521  xmlOut.addTextElementToParent("subapp_detail",
7522  subappInfoPair.second.detail,
7523  subappElement); // get detail
7524  xmlOut.addTextElementToParent("subapp_url",
7525  subappInfoPair.second.parent_url,
7526  subappElement); // get detail
7527  xmlOut.addNumberElementToParent(
7528  "subapp_id", subappInfoPair.second.id, subappElement); // get url
7529  xmlOut.addTextElementToParent("subapp_class",
7530  subappInfoPair.second.class_name,
7531  subappElement); // get class
7532  }
7533  } //end remote app info loop
7534  }
7535  else if(requestType == "getAppId")
7536  {
7537  GatewaySupervisor::handleGetApplicationIdRequest(
7538  &allSupervisorInfo_, cgiIn, xmlOut);
7539  }
7540  else if(requestType == "getContextMemberNames")
7541  {
7542  const XDAQContextTable* contextTable =
7543  CorePropertySupervisorBase::theConfigurationManager_->__GET_CONFIG__(
7544  XDAQContextTable);
7545 
7546  auto contexts = contextTable->getContexts();
7547  for(const auto& context : contexts)
7548  {
7549  xmlOut.addTextElementToData(
7550  "ContextMember", context.contextUID_); // get context member name
7551  }
7552 
7553  //Also add Remote Subystems and consider them Context Member Names!
7554  std::vector<GatewaySupervisor::RemoteGatewayInfo>
7555  remoteGatewayApps; //local copy
7556  { //lock for remainder of scope
7557  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
7558  remoteGatewayApps = remoteGatewayApps_;
7559  }
7560 
7561  for(const auto& remoteGatewayApp : remoteGatewayApps)
7562  xmlOut.addTextElementToData("RemoteGateway",
7563  remoteGatewayApp.appInfo.name + " at " +
7564  remoteGatewayApp.appInfo.url);
7565  }
7566  else if(requestType == "getSystemMessages")
7567  {
7568  xmlOut.addTextElementToData(
7569  "systemMessages", theWebUsers_.getSystemMessage(userInfo.displayName_));
7570 
7571  xmlOut.addTextElementToData(
7572  "username_with_lock",
7573  theWebUsers_.getUserWithLock()); // always give system lock update
7574 
7575  __COUTVS__(20, theWebUsers_.getUserWithLock());
7576 
7577  //Also add Remote Subystems users-with-lock!
7578  std::vector<GatewaySupervisor::RemoteGatewayInfo>
7579  remoteGatewayApps; //local copy
7580  { //lock for remainder of scope
7581  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
7582  remoteGatewayApps = remoteGatewayApps_;
7583  }
7584 
7585  for(const auto& remoteGatewayApp : remoteGatewayApps)
7586  {
7587  __COUTVS__(21, remoteGatewayApp.appInfo.status);
7588 
7589  //skip disconnected remote gateways
7590  if(remoteGatewayApp.appInfo.status == SupervisorInfo::APP_STATUS_UNKNOWN)
7591  continue;
7592 
7593  xmlOut.addTextElementToData("RemoteGateway_name",
7594  remoteGatewayApp.appInfo.name);
7595  xmlOut.addTextElementToData("RemoteGateway_usernameWithLock",
7596  remoteGatewayApp.usernameWithLock);
7597  } //end remote subsystem loop
7598  }
7599  else if(requestType == "setUserWithLock")
7600  {
7601  std::string username = CgiDataUtilities::postData(cgiIn, "username");
7602  std::string lock = CgiDataUtilities::postData(cgiIn, "lock");
7603  std::string accounts = CgiDataUtilities::getData(cgiIn, "accounts");
7604 
7605  __COUTV__(username);
7606  __COUTV__(lock);
7607  __COUTV__(accounts);
7608  __COUTV__(userInfo.uid_);
7609 
7610  std::string tmpUserWithLock = theWebUsers_.getUserWithLock();
7611  if(!theWebUsers_.setUserWithLock(userInfo.uid_, lock == "1", username))
7612  xmlOut.addTextElementToData(
7613  "server_alert",
7614  std::string("Set user lock action failed. You must have valid "
7615  "permissions and ") +
7616  "locking user must be currently logged in.");
7617 
7618  theWebUsers_.insertSettingsForUser(userInfo.uid_, &xmlOut, accounts == "1");
7619 
7620  if(tmpUserWithLock !=
7621  theWebUsers_
7622  .getUserWithLock()) // if there was a change, broadcast system message
7623  theWebUsers_.addSystemMessage(
7624  "*",
7625  theWebUsers_.getUserWithLock() == ""
7626  ? tmpUserWithLock + " has unlocked ots."
7627  : theWebUsers_.getUserWithLock() + " has locked ots.");
7628 
7629  //Also add Remote Subystems users-with-lock!
7630  std::vector<GatewaySupervisor::RemoteGatewayInfo>
7631  remoteGatewayApps; //local copy
7632  { //lock for remainder of scope
7633  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
7634  remoteGatewayApps = remoteGatewayApps_;
7635  }
7636 
7637  for(const auto& remoteGatewayApp : remoteGatewayApps)
7638  {
7639  //skip disconnected remote gateways
7640  if(remoteGatewayApp.appInfo.status == SupervisorInfo::APP_STATUS_UNKNOWN)
7641  continue;
7642 
7643  xmlOut.addTextElementToData("RemoteGateway_name",
7644  remoteGatewayApp.appInfo.name);
7645  xmlOut.addTextElementToData("RemoteGateway_usernameWithLock",
7646  remoteGatewayApp.usernameWithLock);
7647  } //end remote subsystem loop
7648  }
7649  else if(requestType == "getStateMachineLastLogEntry")
7650  {
7651  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
7652  std::string transition = CgiDataUtilities::getData(cgiIn, "transition");
7653  __SUP_COUTV__(fsmName);
7654  __SUP_COUTV__(transition);
7655 
7656  //remove appended date and, for start, remove prepended run #
7657  std::string lastLog = getLastLogEntry(transition, fsmName);
7658  __SUP_COUTTV__(lastLog);
7659  size_t i = lastLog.rfind('(');
7660  if(i != std::string::npos && i > 1) //remove appended date
7661  {
7662  lastLog = lastLog.substr(0, i - 1);
7663  __SUP_COUTTV__(lastLog);
7664  }
7665 
7666  if(transition == RunControlStateMachine::START_TRANSITION_NAME)
7667  {
7668  i = lastLog.find(':');
7669  if(i != std::string::npos &&
7670  i + 2 < lastLog.size()) //remove prepended run #
7671  {
7672  lastLog = lastLog.substr(i + 2);
7673  __SUP_COUTTV__(lastLog);
7674  }
7675  }
7676 
7677  xmlOut.addTextElementToData("lastLogEntry", lastLog);
7678  }
7679  else if(requestType == "getStateMachine")
7680  {
7681  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
7682  __SUP_COUTVS__(20, fsmName);
7683 
7684  addRequiredFsmLogInputToXML(xmlOut, fsmName);
7685 
7686  std::vector<toolbox::fsm::State> states;
7687  states = theStateMachine_.getStates();
7688  char stateStr[2];
7689  stateStr[1] = '\0';
7690  std::string transName;
7691  std::string transParameter;
7692 
7693  for(unsigned int i = 0; i < states.size(); ++i) // get all states
7694  {
7695  stateStr[0] = states[i];
7696  DOMElement* stateParent = xmlOut.addTextElementToData("state", stateStr);
7697 
7698  xmlOut.addTextElementToParent(
7699  "state_name", theStateMachine_.getStateName(states[i]), stateParent);
7700 
7701  // get all transition final states, transitionNames and actionNames from
7702  // state
7703  std::map<std::string, toolbox::fsm::State, std::less<std::string>> trans =
7704  theStateMachine_.getTransitions(states[i]);
7705  std::set<std::string> actionNames = theStateMachine_.getInputs(states[i]);
7706 
7707  std::map<std::string, toolbox::fsm::State, std::less<std::string>>::
7708  iterator it = trans.begin();
7709  std::set<std::string>::iterator ait = actionNames.begin();
7710 
7711  // handle hacky way to keep "forward" moving states on right of FSM
7712  // display.. must be first!
7713 
7714  for(; it != trans.end() && ait != actionNames.end(); ++it, ++ait)
7715  {
7716  stateStr[0] = it->second;
7717 
7718  if(stateStr[0] == 'R')
7719  {
7720  xmlOut.addTextElementToParent(
7721  "state_transition", stateStr, stateParent);
7722  xmlOut.addTextElementToParent(
7723  "state_transition_action", *ait, stateParent);
7724  transName = theStateMachine_.getTransitionName(states[i], *ait);
7725  xmlOut.addTextElementToParent(
7726  "state_transition_name", transName, stateParent);
7727  transParameter =
7728  theStateMachine_.getTransitionParameter(states[i], *ait);
7729  xmlOut.addTextElementToParent(
7730  "state_transition_parameter", transParameter, stateParent);
7731  break;
7732  }
7733  else if(stateStr[0] == 'C')
7734  {
7735  xmlOut.addTextElementToParent(
7736  "state_transition", stateStr, stateParent);
7737  xmlOut.addTextElementToParent(
7738  "state_transition_action", *ait, stateParent);
7739  transName = theStateMachine_.getTransitionName(states[i], *ait);
7740  xmlOut.addTextElementToParent(
7741  "state_transition_name", transName, stateParent);
7742  transParameter =
7743  theStateMachine_.getTransitionParameter(states[i], *ait);
7744  xmlOut.addTextElementToParent(
7745  "state_transition_parameter", transParameter, stateParent);
7746  break;
7747  }
7748  }
7749 
7750  // reset for 2nd pass (on left of FSM display)
7751  it = trans.begin();
7752  ait = actionNames.begin();
7753 
7754  // other states
7755  for(; it != trans.end() && ait != actionNames.end(); ++it, ++ait)
7756  {
7757  stateStr[0] = it->second;
7758 
7759  if(stateStr[0] == 'R')
7760  continue;
7761  else if(stateStr[0] == 'C')
7762  continue;
7763 
7764  xmlOut.addTextElementToParent(
7765  "state_transition", stateStr, stateParent);
7766  xmlOut.addTextElementToParent(
7767  "state_transition_action", *ait, stateParent);
7768  transName = theStateMachine_.getTransitionName(states[i], *ait);
7769  xmlOut.addTextElementToParent(
7770  "state_transition_name", transName, stateParent);
7771  transParameter =
7772  theStateMachine_.getTransitionParameter(states[i], *ait);
7773  xmlOut.addTextElementToParent(
7774  "state_transition_parameter", transParameter, stateParent);
7775  }
7776  } //end state traversal loop
7777  }
7778  else if(requestType == "getStateMachineNames")
7779  {
7780  // get stateMachineAliasFilter if possible
7781  ConfigurationTree configLinkNode =
7782  CorePropertySupervisorBase::theConfigurationManager_
7783  ->getSupervisorTableNode(supervisorContextUID_,
7784  supervisorApplicationUID_);
7785 
7786  try
7787  {
7788  auto fsmNodes =
7789  configLinkNode.getNode("LinkToStateMachineTable").getChildren();
7790  for(const auto& fsmNode : fsmNodes)
7791  xmlOut.addTextElementToData("stateMachineName", fsmNode.first);
7792  }
7793  catch(...) // else empty set of state machines.. can always choose ""
7794  {
7795  __COUT__ << "Caught exception, assuming no valid FSM names." << __E__;
7796  xmlOut.addTextElementToData("stateMachineName", "");
7797  }
7798  }
7799  else if(requestType == "getIterationPlanStatus")
7800  {
7801  //__COUT__ << "checking it status" << __E__;
7802  theIterator_.handleCommandRequest(xmlOut, requestType, "");
7803  }
7804  else if(requestType == "getCurrentState")
7805  {
7806  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
7807  addStateMachineStatusToXML(xmlOut, fsmName);
7808  }
7809  else if(requestType == "cancelStateMachineTransition")
7810  {
7811  __SS__ << "State transition was cancelled by user!" << __E__;
7812  __COUTV__(ss.str());
7813  RunControlStateMachine::theStateMachine_.setErrorMessage(ss.str());
7814  RunControlStateMachine::asyncFailureReceived_ = true;
7815  }
7816  else if(requestType == "getErrorInStateMatchine")
7817  {
7818  xmlOut.addTextElementToData("FSM_Error", theStateMachine_.getErrorMessage());
7819  }
7820  else if(requestType == "getDesktopIcons")
7821  {
7822  // get icons and create comma-separated string based on user permissions
7823  // note: each icon has own permission threshold, so each user can have
7824  // a unique desktop icon experience.
7825 
7826  // use latest context always from temporary configuration manager,
7827  // to get updated icons every time...
7828  //(so icon changes do no require an ots restart)
7829  ConfigurationManager
7830  tmpCfgMgr; // Creating new temporary instance so that constructor will activate latest context, note: not using member CorePropertySupervisorBase::theConfigurationManager_
7831  const DesktopIconTable* iconTable =
7832  tmpCfgMgr.__GET_CONFIG__(DesktopIconTable);
7833  const std::vector<DesktopIconTable::DesktopIcon>& icons =
7834  iconTable->getAllDesktopIcons();
7835 
7836  std::string iconString = "";
7837  // comma-separated icon string, 7 fields:
7838  // 0 - caption = text below icon
7839  // 1 - altText = text icon if no image given
7840  // 2 - uniqueWin = if true, only one window is allowed,
7841  // else multiple instances of window
7842  // 3 - permissions = security level needed to see icon
7843  // 4 - picfn = icon image filename
7844  // 5 - linkurl = url of the window to open
7845  // 6 - folderPath = folder and subfolder location '/' separated
7846  // for example: State Machine,FSM,1,200,icon-Physics.gif,/WebPath/html/StateMachine.html?fsm_name=OtherRuns0,,Chat,CHAT,1,1,icon-Chat.png,/urn:xdaq-application:lid=250,,Visualizer,VIS,0,10,icon-Visualizer.png,/WebPath/html/Visualization.html?urn=270,,Configure,CFG,0,10,icon-Configure.png,/urn:xdaq-application:lid=281,,Front-ends,CFG,0,15,icon-Configure.png,/WebPath/html/ConfigurationGUI_subset.html?urn=281&subsetBasePath=FEInterfaceTable&groupingFieldList=Status%2CFEInterfacePluginName&recordAlias=Front%2Dends&editableFieldList=%21%2ACommentDescription%2C%21SlowControls%2A,Config Subsets
7847 
7848  std::map<std::string, WebUsers::permissionLevel_t> userPermissionLevelsMap =
7849  theWebUsers_.getPermissionsForUser(userInfo.uid_);
7850  std::map<std::string, WebUsers::permissionLevel_t>
7851  iconPermissionThresholdsMap;
7852 
7853  __COUTVS__(20, StringMacros::mapToString(userPermissionLevelsMap));
7854 
7855  bool getRemoteIcons =
7856  true; //could potentially enable from configuration in future
7857  //also return remote gateway icons from cache
7858  std::vector<GatewaySupervisor::RemoteGatewayInfo>
7859  remoteGatewayApps; //local copy
7860  if(getRemoteIcons)
7861  { //lock for remainder of scope
7862  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
7863  remoteGatewayApps = remoteGatewayApps_;
7864  }
7865 
7866  std::string ipAddressForRemoteIconsOverUDP = "";
7867 
7868  bool firstIcon = true;
7869  for(const auto& icon : icons)
7870  {
7871  __COUTVS__(21, icon.caption_);
7872  __COUTVS__(21, icon.permissionThresholdString_);
7873 
7874  CorePropertySupervisorBase::extractPermissionsMapFromString(
7875  icon.permissionThresholdString_, iconPermissionThresholdsMap);
7876 
7877  if(!CorePropertySupervisorBase::doPermissionsGrantAccess(
7878  userPermissionLevelsMap, iconPermissionThresholdsMap))
7879  {
7880  __COUTT__ << "No user access to icon '" << icon.caption_ << "'"
7881  << __E__;
7882  continue; // skip icon if no access
7883  }
7884 
7885  __COUTVS__(21, icon.caption_);
7886 
7887  if(getRemoteIcons)
7888  {
7889  __COUTV__(icon.windowContentURL_);
7890  if(icon.windowContentURL_.size() > 4 &&
7891  icon.windowContentURL_[0] == 'o' &&
7892  icon.windowContentURL_[1] == 't' &&
7893  icon.windowContentURL_[2] == 's' &&
7894  icon.windowContentURL_[3] == ':')
7895  {
7896  //retrieval from cache!
7897  bool found = false;
7898  for(const auto& remoteGatewayApp : remoteGatewayApps)
7899  {
7900  if(icon.recordUID_ != remoteGatewayApp.appInfo.name)
7901  continue;
7902  __COUTVS__(21, icon.caption_);
7903  found = true;
7904 
7905  if(remoteGatewayApp.iconString ==
7906  "") //then either error or still loading...
7907  {
7908  //add error if it has to do with icons
7909  if(remoteGatewayApp.error.find("desktop icons") !=
7910  std::string::npos)
7911  xmlOut.addTextElementToData("Error",
7912  remoteGatewayApp.error);
7913 
7914  //add placeholder "Loading icon"
7915  if(firstIcon)
7916  firstIcon = false;
7917  else
7918  iconString += ",";
7919 
7920  if(remoteGatewayApp.parentIconFolderPath != "")
7921  iconString += remoteGatewayApp.parentIconFolderPath +
7922  " loading..."; //icon.caption_;
7923  else if(remoteGatewayApp.user_data_path_record != "")
7924  iconString += remoteGatewayApp.user_data_path_record +
7925  " loading..."; //icon.caption_;
7926  else
7927  iconString += remoteGatewayApp.appInfo.name +
7928  " loading..."; //icon.caption_;
7929 
7930  iconString += ",X"; //icon.alternateText_;
7931  iconString +=
7932  ",1"; //std::string(icon.enforceOneWindowInstance_ ? "1" : "0");
7933  iconString +=
7934  ",0"; //std::string("1"); // set permission to 1 so the
7935  // desktop shows every icon that the
7936  // server allows (i.e., trust server
7937  // security, ignore client security)
7938  iconString += ","; //icon.imageURL_;
7939  iconString += ","; //icon.windowContentURL_;
7940  iconString += ","; //icon.folderPath_;
7941 
7942  break; //done adding error/loading icon
7943  }
7944 
7945  if(firstIcon)
7946  firstIcon = false;
7947  else
7948  iconString += ",";
7949 
7950  iconString += remoteGatewayApp.iconString;
7951  break; //done with cache retrieval
7952  } //end loop retrieval
7953 
7954  if(!found)
7955  {
7956  __SUP_SS__ << "Illegal missing remote icon definition for "
7957  "icon record UID '"
7958  << icon.recordUID_ << ".' Please notify admins."
7959  << __E__;
7960  __SUP_SS_THROW__;
7961  }
7962 
7963  continue; //done with remote icon string retrieval
7964  }
7965  } //end remote icon handling
7966 
7967  // have icon access, so add to CSV string
7968  if(firstIcon)
7969  firstIcon = false;
7970  else
7971  iconString += ",";
7972 
7973  iconString += icon.caption_;
7974  iconString += "," + icon.alternateText_;
7975  iconString +=
7976  "," + std::string(icon.enforceOneWindowInstance_ ? "1" : "0");
7977  iconString += "," + std::string("1"); // set permission to 1 so the
7978  // desktop shows every icon that the
7979  // server allows (i.e., trust server
7980  // security, ignore client security)
7981  iconString += "," + icon.imageURL_;
7982  iconString += "," + icon.windowContentURL_;
7983  iconString += "," + icon.folderPath_;
7984  }
7985  __COUTVS__(23, iconString);
7986 
7987  xmlOut.addTextElementToData("iconList", iconString);
7988  }
7989  else if(requestType == "addDesktopIcon")
7990  {
7991  std::vector<DesktopIconTable::DesktopIcon> newIcons;
7992 
7993  bool success = GatewaySupervisor::handleAddDesktopIconRequest(
7994  userInfo.username_, cgiIn, xmlOut, &newIcons);
7995 
7996  if(success)
7997  {
7998  __COUT__ << "Attempting dynamic icon change..." << __E__;
7999 
8000  DesktopIconTable* iconTable =
8001  (DesktopIconTable*)
8002  CorePropertySupervisorBase::theConfigurationManager_
8003  ->getDesktopIconTable();
8004  iconTable->setAllDesktopIcons(newIcons);
8005  }
8006  else
8007  __COUT__ << "Failed dynamic icon add." << __E__;
8008  } //end addDesktopIcon
8009  else if(requestType == "resetConsoleCounts")
8010  {
8011  //zero out console count and retake first messages
8012 
8013  for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
8014  {
8015  const auto& appInfo = it.second;
8016  if(appInfo.isTypeConsoleSupervisor())
8017  {
8018  xoap::MessageReference tempMessage =
8019  SOAPUtilities::makeSOAPMessageReference("ResetConsoleCounts");
8020  std::string reply = send(appInfo.getDescriptor(), tempMessage);
8021 
8022  if(reply != "Done")
8023  {
8024  __SUP_SS__ << "Error while resetting console counts of "
8025  "Supervisor instance = '"
8026  << appInfo.getName() << "' [LID=" << appInfo.getId()
8027  << "] in Context '" << appInfo.getContextName()
8028  << "' [URL=" << appInfo.getURL() << "].\n\n"
8029  << reply << __E__;
8030  __SUP_SS_THROW__;
8031  }
8032  __SUP_COUT__ << "Reset console counts of Supervisor instance = '"
8033  << appInfo.getName() << "' [LID=" << appInfo.getId()
8034  << "] in Context '" << appInfo.getContextName()
8035  << "' [URL=" << appInfo.getURL() << "]." << __E__;
8036  }
8037  } //end loop for Console Supervisors
8038 
8039  //for user display feedback, clear local cached values also
8040  std::lock_guard<std::mutex> lock(
8041  systemStatusMutex_); //lock for rest of scope
8042  lastConsoleErrTime_ = "0";
8043  lastConsoleErr_ = "";
8044  lastConsoleWarnTime_ = "0";
8045  lastConsoleWarn_ = "";
8046  lastConsoleInfoTime_ = "0";
8047  lastConsoleInfo_ = "";
8048  firstConsoleErrTime_ = "0";
8049  firstConsoleErr_ = "";
8050  firstConsoleWarnTime_ = "0";
8051  firstConsoleWarn_ = "";
8052  firstConsoleInfoTime_ = "0";
8053  firstConsoleInfo_ = "";
8054 
8055  } //end resetConsoleCounts
8056  else if(requestType == "getRemoteSubsystems" ||
8057  requestType == "getRemoteSubsystemStatus")
8058  {
8059  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
8060  bool getRunTypeNote = CgiDataUtilities::getDataAsInt(cgiIn, "getRunTypeNote");
8061  bool getFullInfo = (requestType == "getRemoteSubsystems");
8062  bool getRunNumber = CgiDataUtilities::getDataAsInt(cgiIn, "getRunNumber");
8063 
8064  //if full info, then add:
8065  // - system and remote aliases, translations, group notes
8066  // - run type note
8067  // - log file rollover mode
8068 
8069  if(getFullInfo)
8070  {
8071  __SUP_COUTV__(fsmName);
8072  __SUP_COUTV__(getRunTypeNote);
8073 
8074  //get system and remote aliases, translations, group notes (a la getAliasList request)
8075  addFilteredConfigAliasesToXML(xmlOut, fsmName);
8076 
8077  addRequiredFsmLogInputToXML(xmlOut,
8078  fsmName); //(a la getStateMachine request)
8079 
8080  } //end getFullInfo prepend
8081 
8082  { //get system status
8083 
8084  xmlOut.addTextElementToData(
8085  "last_run_log_entry",
8086  getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME,
8087  fsmName));
8088 
8089  //getIterationPlanStatus returns iterator status and does not request next run number (which is expensive)
8090  // .. so only get run number 1:10
8091  //getIterationPlanStatus will repeat a few fields, but js will just take first field, so doesnt matter
8092  addStateMachineStatusToXML(
8093  xmlOut, fsmName, getRunNumber); //(a la getCurrentState request)
8094  theIterator_.handleCommandRequest(xmlOut, "getIterationPlanStatus", "");
8095 
8096  std::lock_guard<std::mutex> lock(
8097  systemStatusMutex_); //lock for rest of scope
8098 
8099  xmlOut.addTextElementToData("last_logbook_entry", lastLogbookEntry_);
8100  xmlOut.addTextElementToData(
8101  "last_logbook_entry_time",
8102  lastLogbookEntryTime_
8103  ? StringMacros::getTimestampString(lastLogbookEntryTime_)
8104  : "0");
8105  auto msgPair = theWebUsers_.getLastSystemMessage();
8106  xmlOut.addTextElementToData("last_system_message", msgPair.first);
8107  xmlOut.addTextElementToData(
8108  "last_system_message_time",
8109  msgPair.second ? StringMacros::getTimestampString(msgPair.second)
8110  : "0");
8111  xmlOut.addNumberElementToData("active_user_count",
8112  theWebUsers_.getActiveUserCount());
8113  xmlOut.addTextElementToData("active_user_list",
8114  theWebUsers_.getActiveUsersString());
8115  xmlOut.addNumberElementToData("console_err_count",
8116  systemConsoleErrCount_);
8117  xmlOut.addNumberElementToData("console_warn_count",
8118  systemConsoleWarnCount_);
8119  xmlOut.addNumberElementToData("console_info_count",
8120  systemConsoleInfoCount_);
8121  xmlOut.addTextElementToData("last_console_err_msg", lastConsoleErr_);
8122  xmlOut.addTextElementToData("last_console_warn_msg", lastConsoleWarn_);
8123  xmlOut.addTextElementToData("last_console_info_msg", lastConsoleInfo_);
8124  xmlOut.addTextElementToData("last_console_err_msg_time",
8125  lastConsoleErrTime_);
8126  xmlOut.addTextElementToData("last_console_warn_msg_time",
8127  lastConsoleWarnTime_);
8128  xmlOut.addTextElementToData("last_console_info_msg_time",
8129  lastConsoleInfoTime_);
8130  xmlOut.addTextElementToData("first_console_err_msg", firstConsoleErr_);
8131  xmlOut.addTextElementToData("first_console_warn_msg", firstConsoleWarn_);
8132  xmlOut.addTextElementToData("first_console_info_msg", firstConsoleInfo_);
8133  xmlOut.addTextElementToData("first_console_err_msg_time",
8134  firstConsoleErrTime_);
8135  xmlOut.addTextElementToData("first_console_warn_msg_time",
8136  firstConsoleWarnTime_);
8137  xmlOut.addTextElementToData("first_console_info_msg_time",
8138  firstConsoleInfoTime_);
8139 
8140  } //end get system status
8141 
8142  std::vector<GatewaySupervisor::RemoteGatewayInfo>
8143  remoteGatewayApps; //local copy
8144  { //lock for remainder of scope
8145  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8146  __SUP_COUTVS__(22, remoteGatewayApps_.size());
8147  remoteGatewayApps = remoteGatewayApps_;
8148  if(remoteGatewayApps_.size())
8149  __SUP_COUT_TYPE__(TLVL_DEBUG + 22)
8150  << __COUT_HDR__ << remoteGatewayApps_[0].command << " "
8151  << (remoteGatewayApps_[0].appInfo.status) << __E__;
8152  }
8153 
8154  std::string accumulateErrors = "";
8155  for(const auto& remoteSubsystem : remoteGatewayApps)
8156  {
8157  xmlOut.addTextElementToData("subsystem_name",
8158  remoteSubsystem.appInfo.name);
8159  xmlOut.addTextElementToData("subsystem_url", remoteSubsystem.appInfo.url);
8160  xmlOut.addTextElementToData("subsystem_landingPage",
8161  remoteSubsystem.landingPage);
8162  xmlOut.addTextElementToData("subsystem_status",
8163  remoteSubsystem.appInfo.status);
8164  xmlOut.addTextElementToData(
8165  "subsystem_progress",
8166  std::to_string(remoteSubsystem.appInfo.progress));
8167  xmlOut.addTextElementToData("subsystem_detail",
8168  remoteSubsystem.appInfo.detail);
8169  xmlOut.addTextElementToData("subsystem_lastStatusTime",
8170  StringMacros::getTimestampString(
8171  remoteSubsystem.appInfo.lastStatusTime));
8172  xmlOut.addTextElementToData(
8173  "subsystem_consoleErrCount",
8174  std::to_string(remoteSubsystem.consoleErrCount));
8175  xmlOut.addTextElementToData(
8176  "subsystem_consoleWarnCount",
8177  std::to_string(remoteSubsystem.consoleWarnCount));
8178 
8179  if(remoteSubsystem.command == "" && remoteSubsystem.error != "")
8180  {
8181  __COUTT__ << "Error from Subsystem '" << remoteSubsystem.appInfo.name
8182  << "' = " << remoteSubsystem.error << __E__;
8183 
8184  if(remoteSubsystem.error.find(
8185  "Failure gathering Remote Gateway desktop icons") ==
8186  std::string::npos) //only add if not Icon error
8187  {
8188  if(accumulateErrors.size())
8189  accumulateErrors += "\n";
8190  accumulateErrors += remoteSubsystem.error;
8191  }
8192  }
8193 
8194  //special values for managing remote subsystems
8195  xmlOut.addTextElementToData("subsystem_configAlias",
8196  remoteSubsystem.selected_config_alias);
8197 
8198  if(getFullInfo)
8199  {
8200  if(remoteSubsystem.user_data_path_record == "")
8201  {
8202  // __SUP_SS__;
8203  __SUP_COUT_WARN__
8204  << "Remote Subsystem '" << remoteSubsystem.appInfo.name
8205  << "' user data path is empty. Perhaps the system is still "
8206  "booting up. If the problem persists, note that Remote "
8207  "Subsystems are specified through their Desktop Icon "
8208  "record. "
8209  "Please specify a valid User Data Path record as the "
8210  "Desktop Icon AlternateText field, targeting a UID in the "
8211  "SubsystemUserDataPathsTable (or contact system admins "
8212  "for assitance)."
8213  << __E__;
8214  // __SUP_SS_THROW__;
8215  }
8216  }
8217  xmlOut.addTextElementToData(
8218  "subsystem_configAliasChoices",
8219  StringMacros::setToString(remoteSubsystem.config_aliases,
8220  {','})); //CSV list of aliases
8221  xmlOut.addTextElementToData("subsystem_fsmMode",
8222  remoteSubsystem.getFsmMode());
8223  xmlOut.addTextElementToData("subsystem_fsmIncluded",
8224  remoteSubsystem.fsm_included ? "1" : "0");
8225  } //end remote app loop
8226 
8227  if(accumulateErrors != "")
8228  xmlOut.addTextElementToData("system_error", accumulateErrors);
8229 
8230  } //end getRemoteSubsystems
8231  else if(requestType == "setRemoteSubsystemFsmControl")
8232  {
8233  std::string targetSubsystem =
8234  CgiDataUtilities::getData(cgiIn, "targetSubsystem"); // * for all
8235  std::string setValue = CgiDataUtilities::getData(cgiIn, "setValue");
8236  std::string controlType =
8237  CgiDataUtilities::getData(cgiIn, "controlType"); // include, mode
8238 
8239  setValue = StringMacros::decodeURIComponent(setValue);
8240  __SUP_COUTV__(targetSubsystem);
8241  __SUP_COUTV__(setValue);
8242  __SUP_COUTV__(controlType);
8243 
8244  bool changedSomething = false;
8245  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8246  for(auto& remoteGatewayApp : remoteGatewayApps_)
8247  if(targetSubsystem == "*" ||
8248  targetSubsystem == remoteGatewayApp.appInfo.name)
8249  {
8250  changedSomething = true;
8251  if(controlType == "include")
8252  remoteGatewayApp.fsm_included = setValue == "1" ? true : false;
8253  if(controlType == "configAlias")
8254  {
8255  if(remoteGatewayApp.config_aliases.find(setValue) ==
8256  remoteGatewayApp.config_aliases.end())
8257  {
8258  __SUP_SS__
8259  << "Configuration Alias value '" << setValue
8260  << "' for target Subsystem '"
8261  << remoteGatewayApp.appInfo.name
8262  << "' is not found in list of Configuration Aliases: "
8263  << StringMacros::setToString(
8264  remoteGatewayApp.config_aliases)
8265  << __E__;
8266  __SUP_SS_THROW__;
8267  }
8268  remoteGatewayApp.selected_config_alias = setValue;
8269  }
8270  else if(controlType == "mode")
8271  remoteGatewayApp.fsm_mode =
8272  setValue == "Do Not Halt"
8273  ? RemoteGatewayInfo::FSM_ModeTypes::DoNotHalt
8274  : (setValue == "Only Configure"
8275  ? RemoteGatewayInfo::FSM_ModeTypes::OnlyConfigure
8276  : RemoteGatewayInfo::FSM_ModeTypes::Follow_FSM);
8277  }
8278 
8279  if(!changedSomething)
8280  {
8281  __SUP_SS__ << "Did not find any matching subsystems for target '"
8282  << targetSubsystem << "' attempted!" << __E__;
8283  __SUP_SS_THROW__;
8284  }
8285 
8286  saveRemoteGatewaySettings();
8287 
8288  } //end setRemoteSubsystemFsmControl
8289  else if(requestType == "getSubsystemConfigAliasSelectInfo")
8290  {
8291  std::string targetSubsystem =
8292  CgiDataUtilities::getData(cgiIn, "targetSubsystem");
8293  __SUP_COUTV__(targetSubsystem);
8294  //return info on selected_config_alias
8295 
8296  bool found = false;
8297  std::vector<GatewaySupervisor::RemoteGatewayInfo>
8298  remoteGatewayApps; //local copy
8299  { //lock for remainder of scope
8300  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8301  __SUP_COUTVS__(22, remoteGatewayApps_.size());
8302  remoteGatewayApps = remoteGatewayApps_;
8303  if(remoteGatewayApps_.size())
8304  __SUP_COUT_TYPE__(TLVL_DEBUG + 22)
8305  << __COUT_HDR__ << remoteGatewayApps_[0].command << " "
8306  << (remoteGatewayApps_[0].appInfo.status) << __E__;
8307  }
8308  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8309  for(auto& remoteGatewayApp : remoteGatewayApps)
8310  if(targetSubsystem == remoteGatewayApp.appInfo.name)
8311  {
8312  found = true;
8313 
8314  if(remoteGatewayApp.selected_config_alias == "")
8315  {
8316  __SUP_SS__ << "No selected Configuration Alias found for target "
8317  "Subsystem '"
8318  << remoteGatewayApp.appInfo.name
8319  << "' - please select one before requesting info."
8320  << __E__;
8321  __SUP_SS_THROW__;
8322  }
8323 
8324  std::pair<std::string, TableGroupKey> groupTranslation;
8325  std::string groupComment, groupAuthor, groupCreationTime;
8326 
8327  ConfigurationManager
8328  tmpCfgMgr; // Creating new temporary instance to not mess up member CorePropertySupervisorBase::theConfigurationManager_
8329  tmpCfgMgr.getOtherSubsystemConfigAliasInfo(
8330  remoteGatewayApp.user_data_path_record,
8331  remoteGatewayApp.selected_config_alias,
8332  groupTranslation,
8333  groupComment,
8334  groupAuthor,
8335  groupCreationTime);
8336 
8337  std::stringstream returnInfo;
8338  returnInfo << "At remote Subsystem <b>'"
8339  << remoteGatewayApp.appInfo.name
8340  << ",'</b> the Configure Alias <b>'"
8341  << remoteGatewayApp.selected_config_alias
8342  << "'</b> translates to <b>" << groupTranslation.first
8343  << "(" << groupTranslation.second
8344  << ")</b> w/comment: <br><br><i>"
8345  << StringMacros::decodeURIComponent(groupComment);
8346  if(groupCreationTime != "" && groupCreationTime != "0")
8347  returnInfo << "<br><br>";
8348  returnInfo << "<b>" << groupTranslation.first << "("
8349  << groupTranslation.second << ")</b> was created by "
8350  << groupAuthor << " ("
8351  << StringMacros::getTimestampString(groupCreationTime)
8352  << ")";
8353  returnInfo << "</i>";
8354 
8355  xmlOut.addTextElementToData("alias_info", returnInfo.str());
8356  break;
8357  }
8358 
8359  if(!found)
8360  {
8361  __SUP_SS__ << "Did not find any matching subsystems for target '"
8362  << targetSubsystem << "' attempted!" << __E__;
8363  __SUP_SS_THROW__;
8364  }
8365 
8366  } //end getSubsystemConfigAliasSelectInfo
8367  else if(requestType == "commandRemoteSubsystem")
8368  {
8369  std::string targetSubsystem =
8370  CgiDataUtilities::getData(cgiIn, "targetSubsystem");
8371  std::string command = CgiDataUtilities::getData(cgiIn, "command");
8372  std::string parameter = CgiDataUtilities::getData(cgiIn, "parameter");
8373  std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
8374 
8375  __SUP_COUTV__(targetSubsystem);
8376  __SUP_COUTV__(command);
8377  __SUP_COUTV__(parameter);
8378  __SUP_COUTV__(fsmName);
8379 
8380  if(command == "")
8381  {
8382  __SUP_SS__
8383  << "Illegal empty command received to target remote subsystem '"
8384  << targetSubsystem << "' attempted!" << __E__;
8385  __SUP_SS_THROW__;
8386  }
8387  if(targetSubsystem == "")
8388  {
8389  __SUP_SS__ << "Illegal empty targetSubsystem received for remote "
8390  "subsystem command '"
8391  << command << "' attempted!" << __E__;
8392  __SUP_SS_THROW__;
8393  }
8394 
8395  bool found = false;
8396  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8397  for(auto& remoteGatewayApp : remoteGatewayApps_)
8398  {
8399  if(targetSubsystem == remoteGatewayApp.appInfo.name)
8400  {
8401  if(remoteGatewayApp.command != "")
8402  {
8403  __SUP_SS__
8404  << "Can not target the remote subsystem '" << targetSubsystem
8405  << "' with command '" << command
8406  << "' which already has a pending command '"
8407  << remoteGatewayApp.command
8408  << ".' Please try again after the pending command is sent."
8409  << __E__;
8410  __SUP_SS_THROW__;
8411  }
8412  remoteGatewayApp.error = ""; //clear to see result of this command
8413  remoteGatewayApp.command =
8414  command + (parameter != "" ? ("," + parameter) : "");
8415 
8416  //for non-FSM commands, do not modify fsmName
8417  if(command != "ResetConsoleCounts")
8418  remoteGatewayApp.fsmName = fsmName;
8419 
8420  //force status for immediate user feedback
8421  remoteGatewayApp.appInfo.status = "Launching " + command;
8422  remoteGatewayApp.appInfo.progress = 0;
8423  found = true;
8424  }
8425  } //end search for targetSubsystem
8426 
8427  if(!found)
8428  {
8429  __SUP_SS__ << "Target remote subsystem '" << targetSubsystem
8430  << "' was not found for attempted command '" << command << "!'"
8431  << __E__;
8432  __SUP_SS_THROW__;
8433  }
8434  }
8435  else if(requestType == "gatewayLaunchOTS" || requestType == "gatewayLaunchWiz")
8436  {
8437  // NOTE: similar to ConfigurationGUI version but DOES keep active login
8438  // sessions
8439 
8440  __COUT_WARN__ << requestType << " requestType received! " << __E__;
8441 
8442  // gateway launch is different, in that it saves user sessions
8443  theWebUsers_.saveActiveSessions();
8444 
8445  // now launch
8446 
8447  if(requestType == "gatewayLaunchOTS")
8448  GatewaySupervisor::launchStartOTSCommand(
8449  "LAUNCH_OTS", CorePropertySupervisorBase::theConfigurationManager_);
8450  else if(requestType == "gatewayLaunchWiz")
8451  GatewaySupervisor::launchStartOneServerCommand(
8452  "LAUNCH_WIZ",
8453  CorePropertySupervisorBase::theConfigurationManager_,
8454  getContextUID());
8455  }
8456  else if(requestType == "gatewayLaunchOTSInstance")
8457  {
8458  __COUT_WARN__ << requestType << " requestType received! " << __E__;
8459 
8460  std::string targetSubsystem =
8461  CgiDataUtilities::getData(cgiIn, "targetSubsystem");
8462  __SUP_COUTV__(targetSubsystem);
8463  //launch Target Subsystem's remote ots instance
8464 
8465  bool found = false;
8466  std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
8467  for(auto& remoteGatewayApp : remoteGatewayApps_)
8468  if(targetSubsystem == remoteGatewayApp.appInfo.name)
8469  {
8470  found = true;
8471 
8472  std::stringstream commandSs;
8473  commandSs << "LAUNCH_INSTANCE";
8474  commandSs << ";" << remoteGatewayApp.instanceUser;
8475  commandSs << ";" << remoteGatewayApp.instanceHost;
8476  //assume ots path is parent of USER_DATA
8477  size_t i = remoteGatewayApp.instancePath.rfind('/');
8478  if(i != std::string::npos)
8479  commandSs << ";" << remoteGatewayApp.instancePath.substr(0, i);
8480  else
8481  commandSs << ";" << remoteGatewayApp.instancePath;
8482  commandSs << ";"
8483  << "Normal";
8484  commandSs << ";" << remoteGatewayApp.setupType;
8485  commandSs << ";"
8486  << remoteGatewayApp.instancePath; //full USER_DATA path
8487  __SUP_COUTV__(commandSs.str());
8488 
8489  GatewaySupervisor::launchStartOneServerCommand(
8490  commandSs.str(),
8491  //"LAUNCH_INSTANCE;mu2ehwdev;mu2e-cfo-01.fnal.gov;/home/mu2ehwdev/ots_spack_fast;Normal;shift1",
8492  CorePropertySupervisorBase::theConfigurationManager_,
8493  getContextUID());
8494 
8495  //force status for immediate user feedback
8496  remoteGatewayApp.command =
8497  "Reboot"; //use command process for getting updated status
8498  remoteGatewayApp.appInfo.status = "Rebooting... ";
8499  remoteGatewayApp.appInfo.progress = 1;
8500  }
8501 
8502  if(!found)
8503  {
8504  __SUP_SS__ << "Did not find any matching subsystems for target '"
8505  << targetSubsystem << "' attempted!" << __E__;
8506  __SUP_SS_THROW__;
8507  }
8508  }
8509  else if(requestType == "resetUserTooltips")
8510  {
8511  WebUsers::resetAllUserTooltips(userInfo.username_);
8512  }
8513  else if(requestType == "silenceUserTooltips")
8514  {
8515  WebUsers::silenceAllUserTooltips(userInfo.username_);
8516  }
8517  else if(requestType == "restartApps") /*NEW: ADDED FOR APPS RESTART*/
8518  {
8519  std::string contextName = CgiDataUtilities::getData(cgiIn, "contextName");
8520  __COUT__ << "launch ots script Command = "
8521  << "OTS_APP_SHUTDOWN" << __E__;
8522  GatewaySupervisor::launchStartOneServerCommand(
8523  "OTS_APP_SHUTDOWN",
8524  CorePropertySupervisorBase::theConfigurationManager_,
8525  contextName);
8526  sleep(5);
8527  GatewaySupervisor::launchStartOneServerCommand(
8528  "OTS_APP_STARTUP",
8529  CorePropertySupervisorBase::theConfigurationManager_,
8530  contextName);
8531 
8532  xmlOut.addTextElementToData("status", "restarted");
8533  }
8534  else
8535  {
8536  __SS__ << "requestType Request, " << requestType << ", not recognized."
8537  << __E__;
8538  __SS_THROW__;
8539  }
8540  }
8541  catch(const std::runtime_error& e)
8542  {
8543  __SS__ << "An error was encountered handling requestType '" << requestType
8544  << "':" << e.what() << __E__;
8545  __COUT__ << "\n" << ss.str();
8546  xmlOut.addTextElementToData("Error", ss.str());
8547  }
8548  catch(...)
8549  {
8550  __SS__ << "An unknown error was encountered handling requestType '" << requestType
8551  << ".' "
8552  << "Please check the printouts to debug." << __E__;
8553  try
8554  {
8555  throw;
8556  } //one more try to printout extra info
8557  catch(const std::exception& e)
8558  {
8559  ss << "Exception message: " << e.what();
8560  }
8561  catch(...)
8562  {
8563  }
8564  __COUT__ << "\n" << ss.str();
8565  xmlOut.addTextElementToData("Error", ss.str());
8566  }
8567 
8568  // return xml doc holding server response
8569  xmlOut.outputXmlDocument(
8570  (std::ostringstream*)out,
8571  false /*dispStdOut*/,
8572  true /*allowWhiteSpace*/); // Note: allow white space need for error response
8573 
8574  //__COUT__ << "Done" << __E__;
8575 } // end request()
8576 catch(const std::runtime_error& e)
8577 {
8578  __COUT_ERR__ << "Caught error at request(): " << e.what() << __E__;
8579  throw;
8580 } // end request() exception handling
8581 
8582 //==============================================================================
8583 void GatewaySupervisor::addStateMachineStatusToXML(HttpXmlDocument& xmlOut,
8584  const std::string& fsmName,
8585  bool getRunNumber /* = true */)
8586 {
8587  xmlOut.addTextElementToData("active_fsmName", activeStateMachineName_);
8588  xmlOut.addTextElementToData("current_state", theStateMachine_.getCurrentStateName());
8589  const std::string& gatewayStatus = allSupervisorInfo_.getGatewayInfo().getStatus();
8590  if(gatewayStatus.size() >
8591  std::string(RunControlStateMachine::FAILED_STATE_NAME).length() &&
8592  (gatewayStatus[0] == 'F' ||
8593  gatewayStatus[0] ==
8594  'E')) //assume it is Failed or Error and send to state machine
8595  xmlOut.addTextElementToData("current_error", gatewayStatus);
8596 
8597  xmlOut.addTextElementToData("in_transition",
8598  theStateMachine_.isInTransition() ? "1" : "0");
8599  if(theStateMachine_.isInTransition())
8600  {
8601  xmlOut.addTextElementToData(
8602  "transition_progress",
8603  RunControlStateMachine::theProgressBar_.readPercentageString());
8604  xmlOut.addTextElementToData("current_transition",
8605  theStateMachine_.getCurrentTransitionName());
8606  }
8607  else
8608  {
8609  xmlOut.addTextElementToData("transition_progress", "100");
8610  xmlOut.addTextElementToData("current_transition", "");
8611  }
8612  xmlOut.addTextElementToData("time_in_state",
8613  std::to_string(theStateMachine_.getTimeInState()));
8614 
8615  // char tmp[20]; old size before adding db run number
8616  char tmp
8617  [50]; // for a 6 digits run number from the DB, this needs to be at least 34 chars
8618 
8619  //__COUT__ << "current state: " << theStateMachine_.getCurrentStateName() <<
8620  //__E__;
8621 
8623 
8624  // std::string fsmName = CgiDataUtilities::getData(cgiIn, "fsmName");
8625  // __COUT__ << "fsmName = " << fsmName << __E__;
8626  // __COUT__ << "activeStateMachineName_ = " << activeStateMachineName_ <<
8627  //__E__;
8628  // __COUT__ << "theStateMachine_.getProvenanceStateName() = " <<
8629  // theStateMachine_.getProvenanceStateName() << __E__;
8630  // __COUT__ << "theStateMachine_.getCurrentStateName() = " <<
8631  // theStateMachine_.getCurrentStateName() << __E__;
8632  bool useRunInfoDb = false;
8633 
8634  if(!theStateMachine_.isInTransition())
8635  {
8636  if(RunControlStateMachine::asyncStopExceptionReceived_)
8637  {
8638  //__COUTV__(RunControlStateMachine::asyncPauseExceptionReceived_);
8639  //__COUTV__(RunControlStateMachine::getErrorMessage());
8640  xmlOut.addTextElementToData("soft_error",
8641  RunControlStateMachine::getErrorMessage());
8642  }
8643 
8644  std::string stateMachineRunAlias = "Run"; // default to "Run"
8645  bool rollOverLogOnConfigure = false, rollOverLogOnStart = false;
8646  std::string rollOverLogOnSize = "";
8647 
8648  // get stateMachineAliasFilter if possible
8649  ConfigurationTree configLinkNode =
8650  CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(
8651  supervisorContextUID_, supervisorApplicationUID_);
8652 
8653  if(!configLinkNode.isDisconnected())
8654  {
8655  try // for backwards compatibility
8656  {
8657  ConfigurationTree fsmLinkNode =
8658  configLinkNode.getNode("LinkToStateMachineTable");
8659  if(!fsmLinkNode.isDisconnected())
8660  {
8661  if(!fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
8662  .isDefaultValue())
8663  stateMachineRunAlias =
8664  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
8665  .getValue<std::string>();
8666  std::string runInfoPluginType =
8667  fsmLinkNode.getNode(fsmName + "/RunInfoPluginType")
8668  .getValue<std::string>();
8669  if(runInfoPluginType !=
8670  TableViewColumnInfo::DATATYPE_STRING_DEFAULT &&
8671  runInfoPluginType != "No Run Info Plugin")
8672  useRunInfoDb = true;
8673 
8674  try
8675  {
8676  rollOverLogOnConfigure =
8677  fsmLinkNode.getNode("RollOverLogOnConfigure")
8678  .getValueWithDefault<bool>(false /* defaultValue */);
8679  }
8680  catch(...)
8681  {
8682  rollOverLogOnConfigure = false;
8683  }
8684  try
8685  {
8686  rollOverLogOnStart =
8687  fsmLinkNode.getNode("RollOverLogOnConfigure")
8688  .getValueWithDefault<bool>(false /* defaultValue */);
8689  }
8690  catch(...)
8691  {
8692  rollOverLogOnStart = false;
8693  }
8694  }
8695  }
8696  catch(std::runtime_error& e)
8697  {
8698  ; //ignoring error
8699  }
8700  catch(...)
8701  {
8702  __COUT_ERR__
8703  << "Unknown error looking for Run alias. Should never happen."
8704  << __E__;
8705  }
8706  }
8707 
8708  xmlOut.addTextElementToData("stateMachineRunAlias", stateMachineRunAlias);
8709 
8710  //generate log rollover string
8711  {
8712  rollOverLogOnSize =
8713  (getenv("OTS_LOG_ROLLOVER") ? getenv("OTS_LOG_ROLLOVER") : "");
8714  std::stringstream ss;
8715  if(rollOverLogOnConfigure || rollOverLogOnStart || rollOverLogOnSize != "")
8716  {
8717  ss << "ots log files will rollover ";
8718  if(!rollOverLogOnConfigure &&
8719  !rollOverLogOnStart) //the rollOverLogOnSize != ""
8720  ss << " on size-in-bytes: " << rollOverLogOnSize
8721  << ". To enable on FSM transitions set RollOverLogOnConfigure "
8722  "and/or RollOverLogOnStart in the FSM Configuration Tree.";
8723  else
8724  {
8725  if(rollOverLogOnConfigure && rollOverLogOnStart)
8726  ss << " on the Configure and Start FSM transitions";
8727  else if(rollOverLogOnConfigure)
8728  ss << " on the Configure FSM transition";
8729  else if(rollOverLogOnStart)
8730  ss << " on the Start FSM transition";
8731 
8732  if(rollOverLogOnSize != "")
8733  ss << ". To enable rollover on 100MB size, for example, export "
8734  "OTS_LOG_ROLLOVER=100000000.";
8735  else
8736  ss << ", and also on size-in-bytes: " << rollOverLogOnSize;
8737  }
8738  }
8739  else
8740  ss << "ots log files will not rollover. "
8741  "To enable rollover on 100MB size, for example, export "
8742  "OTS_LOG_ROLLOVER=100000000; "
8743  "to enable on FSM transitions set RollOverLogOnConfigure and/or "
8744  "RollOverLogOnStart in the FSM Configuration Tree.";
8745 
8746  xmlOut.addTextElementToData("stateMachineLogRollover", ss.str());
8747  }
8748 
8750 
8751  if(theStateMachine_.getCurrentStateName() ==
8752  RunControlStateMachine::RUNNING_STATE_NAME ||
8753  theStateMachine_.getCurrentStateName() ==
8754  RunControlStateMachine::PAUSED_STATE_NAME)
8755  {
8756  if(useRunInfoDb)
8757  sprintf(tmp,
8758  "Current %s Number from DB: %s",
8759  activeStateMachineRunAlias_.c_str(),
8760  activeStateMachineRunNumber_.c_str());
8761  //%u // getNextRunNumber(activeStateMachineName_) - 1);
8762  else
8763  sprintf(
8764  tmp,
8765  "Current %s Number: %s",
8766  activeStateMachineRunAlias_.c_str(),
8767  activeStateMachineRunNumber_
8768  .c_str()); //%u //getNextRunNumber(activeStateMachineName_) - 1);
8769  xmlOut.addTextElementToData("run_number", tmp);
8770 
8771  if(RunControlStateMachine::asyncPauseExceptionReceived_)
8772  {
8773  //__COUTV__(RunControlStateMachine::asyncPauseExceptionReceived_);
8774  //__COUTV__(RunControlStateMachine::getErrorMessage());
8775  xmlOut.addTextElementToData("soft_error",
8776  RunControlStateMachine::getErrorMessage());
8777  }
8778  }
8779  else if(
8780  getRunNumber) //only periodically get next run number (expensive from file, and shouldnt change much)
8781  {
8782  if(useRunInfoDb)
8783  sprintf(tmp, "Next %s Number from DB.", stateMachineRunAlias.c_str());
8784  else
8785  sprintf(tmp,
8786  "Next %s Number: %u",
8787  stateMachineRunAlias.c_str(),
8788  getNextRunNumber(fsmName));
8789 
8790  xmlOut.addTextElementToData("run_number", tmp);
8791  }
8792 
8793  } //end not-in-transition handling
8794 } // end addStateMachineStatusToXML()
8795 
8796 //==============================================================================
8797 void GatewaySupervisor::addRequiredFsmLogInputToXML(HttpXmlDocument& xmlOut,
8798  const std::string& fsmName)
8799 {
8800  bool requireUserLogInputOnConfigure = false, requireUserLogInputOnRun = false;
8801  //if fsmName specified, return log entry requirements from config tree
8802  if(fsmName != "")
8803  {
8804  //------------------
8805  ConfigurationTree configLinkNode =
8806  CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(
8807  supervisorContextUID_, supervisorApplicationUID_);
8808  if(!configLinkNode.isDisconnected())
8809  {
8810  // clang-format off
8811  try //ignore errors
8812  {
8813  ConfigurationTree fsmLinkNode = configLinkNode.getNode("LinkToStateMachineTable").getNode(fsmName);
8814  try { requireUserLogInputOnConfigure = fsmLinkNode.getNode("RequireUserLogInputOnConfigureTransition").getValue<bool>(); } catch(...) { __SUP_COUTT__ << "RequireUserLogInputOnConfigureTransition not set."; }
8815  try { requireUserLogInputOnRun = fsmLinkNode.getNode("RequireUserLogInputOnRunTransition").getValue<bool>(); } catch(...) { __SUP_COUTT__ << "RequireUserLogInputOnRunTransition not set."; }
8816  }
8817  catch(...)
8818  { __SUP_COUTT__ << "Settings not set for fsm name = " << fsmName << __E__; }
8819  // clang-format on
8820  }
8821  } //end log entry requirements gathering
8822 
8823  xmlOut.addTextElementToData("RequireUserLogInputOnConfigureTransition",
8824  requireUserLogInputOnConfigure ? "1" : "0");
8825  xmlOut.addTextElementToData("RequireUserLogInputOnRunTransition",
8826  requireUserLogInputOnRun ? "1" : "0");
8827 } // end request()
8828 
8829 //==============================================================================
8830 void GatewaySupervisor::addFilteredConfigAliasesToXML(HttpXmlDocument& xmlOut,
8831  const std::string& fsmName)
8832 {
8833  __SUP_COUTV__(fsmName);
8834 
8835  // IMPORTANT -- use temporary ConfigurationManager to get the Active Group Aliases,
8836  // to avoid changing the Context Configuration tree for the Gateway Supervisor
8837  ConfigurationManager temporaryConfigMgr;
8838  std::map<std::string /*alias*/, std::pair<std::string /*group name*/, TableGroupKey>>
8839  aliasMap;
8840  aliasMap = temporaryConfigMgr.getActiveGroupAliases();
8841 
8842  // also IMPORTANT -- to use theConfigurationManager_ to get the Context settings for the Gateway Supervisor
8843  // get stateMachineAliasFilter if possible
8844  ConfigurationTree configLinkNode =
8845  CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode(
8846  supervisorContextUID_, supervisorApplicationUID_);
8847 
8848  std::string stateMachineAliasFilter = "*"; // default to all
8849  if(fsmName != "" && !configLinkNode.isDisconnected())
8850  {
8851  try // for backwards compatibility
8852  {
8853  ConfigurationTree fsmLinkNode =
8854  configLinkNode.getNode("LinkToStateMachineTable");
8855  if(!fsmLinkNode.isDisconnected() &&
8856  !fsmLinkNode.getNode(fsmName + "/SystemAliasFilter").isDefaultValue())
8857  stateMachineAliasFilter =
8858  fsmLinkNode.getNode(fsmName + "/SystemAliasFilter")
8859  .getValue<std::string>();
8860  else
8861  __COUT_INFO__ << "FSM Link disconnected." << __E__;
8862  }
8863  catch(std::runtime_error& e)
8864  {
8865  __COUT_INFO__ << e.what() << __E__;
8866  }
8867  catch(...)
8868  {
8869  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
8870  }
8871  }
8872  else
8873  __COUT_INFO__ << "FSM Link disconnected." << __E__;
8874 
8875  __COUT__ << "For FSM '" << fsmName
8876  << ",' stateMachineAliasFilter = " << stateMachineAliasFilter << __E__;
8877 
8878  // filter list of aliases based on stateMachineAliasFilter
8879  // ! as first character means choose those that do NOT match filter
8880  // * can be used as wild card.
8881  {
8882  bool invertFilter =
8883  stateMachineAliasFilter.size() && stateMachineAliasFilter[0] == '!';
8884  std::vector<std::string> filterArr;
8885 
8886  size_t i = 0;
8887  if(invertFilter)
8888  ++i;
8889  size_t f;
8890  std::string tmp;
8891  while((f = stateMachineAliasFilter.find('*', i)) != std::string::npos)
8892  {
8893  tmp = stateMachineAliasFilter.substr(i, f - i);
8894  i = f + 1;
8895  filterArr.push_back(tmp);
8896  //__COUT__ << filterArr[filterArr.size()-1] << " " << i <<
8897  // " of " << stateMachineAliasFilter.size() << __E__;
8898  }
8899  if(i <= stateMachineAliasFilter.size())
8900  {
8901  tmp = stateMachineAliasFilter.substr(i);
8902  filterArr.push_back(tmp);
8903  //__COUT__ << filterArr[filterArr.size()-1] << " last." << __E__;
8904  }
8905 
8906  bool filterMatch;
8907 
8908  for(auto& aliasMapPair : aliasMap)
8909  {
8910  //__COUT__ << "aliasMapPair.first: " << aliasMapPair.first << __E__;
8911 
8912  filterMatch = true;
8913 
8914  if(filterArr.size() == 1)
8915  {
8916  if(filterArr[0] != "" && filterArr[0] != "*" &&
8917  aliasMapPair.first != filterArr[0])
8918  filterMatch = false;
8919  }
8920  else
8921  {
8922  i = -1;
8923  for(f = 0; f < filterArr.size(); ++f)
8924  {
8925  if(!filterArr[f].size())
8926  continue; // skip empty filters
8927 
8928  if(f == 0) // must start with this filter
8929  {
8930  if((i = aliasMapPair.first.find(filterArr[f])) != 0)
8931  {
8932  filterMatch = false;
8933  break;
8934  }
8935  }
8936  else if(f == filterArr.size() - 1) // must end with this filter
8937  {
8938  if(aliasMapPair.first.rfind(filterArr[f]) !=
8939  aliasMapPair.first.size() - filterArr[f].size())
8940  {
8941  filterMatch = false;
8942  break;
8943  }
8944  }
8945  else if((i = aliasMapPair.first.find(filterArr[f])) ==
8946  std::string::npos)
8947  {
8948  filterMatch = false;
8949  break;
8950  }
8951  }
8952  }
8953 
8954  if(invertFilter)
8955  filterMatch = !filterMatch;
8956 
8957  //__COUT__ << "filterMatch=" << filterMatch << __E__;
8958 
8959  if(!filterMatch)
8960  continue;
8961 
8962  xmlOut.addTextElementToData("config_alias", aliasMapPair.first);
8963  xmlOut.addTextElementToData(
8964  "config_key",
8965  TableGroupKey::getFullGroupString(aliasMapPair.second.first,
8966  aliasMapPair.second.second,
8967  /*decorate as (<key>)*/ "(",
8968  ")"));
8969 
8970  // __COUT__ << "config_alias_comment" << " " << temporaryConfigMgr.getNode(
8971  // ConfigurationManager::GROUP_ALIASES_TABLE_NAME).getNode(aliasMapPair.first).getNode(
8972  // TableViewColumnInfo::COL_NAME_COMMENT).getValue<std::string>() << __E__;
8973  xmlOut.addTextElementToData(
8974  "config_alias_comment",
8975  temporaryConfigMgr.getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
8976  .getNode(aliasMapPair.first)
8977  .getNode(TableViewColumnInfo::COL_NAME_COMMENT)
8978  .getValue<std::string>());
8979 
8980  std::string groupComment, groupAuthor, groupCreationTime;
8981  try
8982  {
8983  temporaryConfigMgr.loadTableGroup(aliasMapPair.second.first,
8984  aliasMapPair.second.second,
8985  false /*doActivate*/,
8986  0 /*groupMembers*/,
8987  0 /*progressBar*/,
8988  0 /*accumulateWarnings*/,
8989  &groupComment,
8990  &groupAuthor,
8991  &groupCreationTime,
8992  true /*doNotLoadMembers*/);
8993 
8994  xmlOut.addTextElementToData("config_comment", groupComment);
8995  xmlOut.addTextElementToData("config_author", groupAuthor);
8996  xmlOut.addTextElementToData("config_create_time", groupCreationTime);
8997  }
8998  catch(...)
8999  {
9000  __COUT_WARN__ << "Failed to load group metadata." << __E__;
9001  xmlOut.addTextElementToData("config_comment", "");
9002  xmlOut.addTextElementToData("config_author", "");
9003  xmlOut.addTextElementToData("config_create_time", "");
9004  }
9005  }
9006  }
9007 
9008  // return last group alias by user
9009  std::string fn = ConfigurationManager::LAST_TABLE_GROUP_SAVE_PATH + "/" +
9010  FSM_LAST_GROUP_ALIAS_FILE_START + fsmName + "." +
9011  FSM_USERS_PREFERENCES_FILETYPE;
9012  __COUT__ << "Load preferences: " << fn << __E__;
9013  FILE* fp = fopen(fn.c_str(), "r");
9014  if(fp)
9015  {
9016  char tmpLastAlias[500];
9017  fscanf(fp, "%*s %s", tmpLastAlias);
9018  __COUT__ << "tmpLastAlias: " << tmpLastAlias << __E__;
9019 
9020  xmlOut.addTextElementToData("UserLastConfigAlias", tmpLastAlias);
9021  fclose(fp);
9022  }
9023  else if(aliasMap.size()) //if not set, return first
9024  xmlOut.addTextElementToData("UserLastConfigAlias", aliasMap.begin()->first);
9025 } //end addFilteredConfigAliasesToXML()
9026 
9027 //==============================================================================
9032 void GatewaySupervisor::launchStartOneServerCommand(const std::string& command,
9033  ConfigurationManager* cfgMgr,
9034  const std::string& contextName)
9035 {
9036  __COUT__ << "launch ots script Command = " << command << __E__;
9037  __COUT__ << "Extracting target context hostname... " << __E__;
9038 
9039  std::string hostname;
9040  try
9041  {
9042  cfgMgr->init(); // completely reset to re-align with any changes
9043  const XDAQContextTable* contextTable = cfgMgr->__GET_CONFIG__(XDAQContextTable);
9044  auto contexts = contextTable->getContexts();
9045 
9046  unsigned int i, j;
9047  for(const auto& context : contexts)
9048  {
9049  if(context.contextUID_ != contextName)
9050  continue;
9051 
9052  __COUT__ << "contextUID_ is: " << context.contextUID_ << __E__;
9053 
9054  // find last slash
9055  j = 0; // default to whole string
9056  for(i = 0; i < context.address_.size(); ++i)
9057  if(context.address_[i] == '/')
9058  j = i + 1;
9059  hostname = context.address_.substr(j);
9060  __COUT__ << "ots script command '" << command
9061  << "' launching on hostname = " << hostname << " in context name "
9062  << context.contextUID_ << __E__;
9063  }
9064  }
9065  catch(...)
9066  {
9067  __SS__ << "\nRelaunch of otsdaq interrupted! "
9068  << "The Configuration Manager could not be initialized." << __E__;
9069 
9070  __SS_THROW__;
9071  }
9072 
9073  std::string fn = (std::string(__ENV__("SERVICE_DATA_PATH")) + "/StartOTS_action_" +
9074  hostname + ".cmd");
9075  FILE* fp = fopen(fn.c_str(), "w");
9076  if(fp)
9077  {
9078  fprintf(fp, "%s", command.c_str());
9079  fclose(fp);
9080  }
9081  else
9082  {
9083  __SS__ << "Unable to open command file: " << fn << __E__;
9084  __SS_THROW__;
9085  }
9086 
9087  sleep(2 /*seconds*/); // then verify that the commands were read
9088  // note: StartOTS.sh has a sleep of 1 second
9089 
9090  fn = (std::string(__ENV__("SERVICE_DATA_PATH")) + "/StartOTS_action_" + hostname +
9091  ".cmd");
9092  fp = fopen(fn.c_str(), "r");
9093  if(fp)
9094  {
9095  char line[100];
9096  fgets(line, 100, fp);
9097  fclose(fp);
9098 
9099  if(strcmp(line, command.c_str()) == 0)
9100  {
9101  __SS__ << "The command looks to have been ignored by " << hostname
9102  << ". Is the ots launch script still running on that node?" << __E__;
9103  __SS_THROW__;
9104  }
9105  __COUTV__(line);
9106  }
9107  else
9108  {
9109  __SS__ << "Unable to open command file for verification: " << fn << __E__;
9110  __SS_THROW__;
9111  }
9112 } // end launchStartOneServerCommand
9113 
9114 //==============================================================================
9118 void GatewaySupervisor::launchStartOTSCommand(const std::string& command,
9119  ConfigurationManager* cfgMgr)
9120 {
9121  __COUT__ << "launch ots script Command = " << command << __E__;
9122  __COUT__ << "Extracting target context hostnames... " << __E__;
9123 
9124  std::vector<std::string> hostnames;
9125  try
9126  {
9127  cfgMgr->init(); // completely reset to re-align with any changes
9128 
9129  const XDAQContextTable* contextTable = cfgMgr->__GET_CONFIG__(XDAQContextTable);
9130 
9131  auto contexts = contextTable->getContexts();
9132  unsigned int i, j;
9133  for(const auto& context : contexts)
9134  {
9135  if(!context.status_)
9136  continue;
9137 
9138  // find last slash
9139  j = 0; // default to whole string
9140  for(i = 0; i < context.address_.size(); ++i)
9141  if(context.address_[i] == '/')
9142  j = i + 1;
9143  hostnames.push_back(context.address_.substr(j));
9144  __COUT__ << "ots script command '" << command
9145  << "' launching on hostname = " << hostnames.back() << __E__;
9146  }
9147  }
9148  catch(...)
9149  {
9150  __SS__ << "Launch of command '" << command << "' interrupted! "
9151  << "The Configuration Manager could not be initialized to find targets."
9152  << __E__;
9153 
9154  __SS_THROW__;
9155  }
9156 
9157  for(const auto& hostname : hostnames)
9158  {
9159  std::string fn = (std::string(__ENV__("SERVICE_DATA_PATH")) +
9160  "/StartOTS_action_" + hostname + ".cmd");
9161  FILE* fp = fopen(fn.c_str(), "w");
9162  if(fp)
9163  {
9164  fprintf(fp, "%s", command.c_str());
9165  fclose(fp);
9166  }
9167  else
9168  {
9169  __SS__ << "Unable to open command file: " << fn << __E__;
9170  __SS_THROW__;
9171  }
9172  }
9173 
9174  sleep(2 /*seconds*/); // then verify that the commands were read
9175  // note: StartOTS.sh has a sleep of 1 second
9176 
9177  for(const auto& hostname : hostnames)
9178  {
9179  std::string fn = (std::string(__ENV__("SERVICE_DATA_PATH")) +
9180  "/StartOTS_action_" + hostname + ".cmd");
9181  FILE* fp = fopen(fn.c_str(), "r");
9182  if(fp)
9183  {
9184  char line[100];
9185  fgets(line, 100, fp);
9186  fclose(fp);
9187 
9188  if(strcmp(line, command.c_str()) == 0)
9189  {
9190  __SS__ << "The command '" << command << "' looks to have been ignored by "
9191  << hostname
9192  << ". Is the ots launch script still running on that node?"
9193  << __E__;
9194  __SS_THROW__;
9195  }
9196  __COUTV__(line);
9197  }
9198  else
9199  {
9200  __SS__ << "Unable to open command file for verification: " << fn << __E__;
9201  __SS_THROW__;
9202  }
9203  }
9204 } // end launchStartOTSCommand
9205 
9206 //==============================================================================
9209 xoap::MessageReference GatewaySupervisor::supervisorCookieCheck(
9210  xoap::MessageReference message)
9211 
9212 {
9213  __COUTT__
9214  << "request from remote Supervisor for GatewaySupervisor::supervisorCookieCheck()"
9215  << __E__;
9216 
9217  // SOAPUtilities::receive request parameters
9218  SOAPParameters parameters;
9219  parameters.addParameter("CookieCode");
9220  parameters.addParameter("RefreshOption");
9221  parameters.addParameter("IPAddress");
9222  SOAPUtilities::receive(message, parameters);
9223  std::string cookieCode = parameters.getValue("CookieCode");
9224  std::string refreshOption =
9225  parameters.getValue("RefreshOption"); // give external supervisors option to
9226  // refresh cookie or not, "1" to refresh
9227  std::string ipAddress =
9228  parameters.getValue("IPAddress"); // give external supervisors option to refresh
9229  // cookie or not, "1" to refresh
9230 
9231  // If TRUE, cookie code is good, and refreshed code is in cookieCode, also pointers
9232  // optionally for uint8_t userPermissions, uint64_t uid Else, error message is
9233  // returned in cookieCode
9234  std::map<std::string /*groupName*/, WebUsers::permissionLevel_t>
9235  userGroupPermissionsMap;
9236  std::string userWithLock = "";
9237  uint64_t uid, userSessionIndex;
9238  theWebUsers_.cookieCodeIsActiveForRequest(cookieCode,
9239  &userGroupPermissionsMap,
9240  &uid /*uid is not given to remote users*/,
9241  ipAddress,
9242  refreshOption == "1",
9243  false /* doNotGoRemote */,
9244  &userWithLock,
9245  &userSessionIndex);
9246 
9247  __COUTTV__(userWithLock);
9248 
9249  // fill return parameters
9250  SOAPParameters retParameters;
9251  retParameters.addParameter("CookieCode", cookieCode);
9252  retParameters.addParameter(
9253  "Permissions", StringMacros::mapToString(userGroupPermissionsMap).c_str());
9254  retParameters.addParameter("UserWithLock", userWithLock);
9255  retParameters.addParameter("Username", theWebUsers_.getUsersUsername(uid));
9256  retParameters.addParameter("DisplayName", theWebUsers_.getUsersDisplayName(uid));
9257  retParameters.addParameter("UserSessionIndex", std::to_string(userSessionIndex));
9258 
9259  __COUTT__ << "Login response: " << retParameters.getValue("Username") << __E__;
9260 
9261  return SOAPUtilities::makeSOAPMessageReference("CookieResponse", retParameters);
9262 } // end supervisorCookieCheck()
9263 
9264 //==============================================================================
9267 xoap::MessageReference GatewaySupervisor::supervisorGetActiveUsers(
9268  xoap::MessageReference /*message*/)
9269 {
9270  __COUT__ << __E__;
9271 
9272  SOAPParameters parameters("UserList", theWebUsers_.getActiveUsersString());
9273  return SOAPUtilities::makeSOAPMessageReference("ActiveUserResponse", parameters);
9274 } // end supervisorGetActiveUsers()
9275 
9276 //==============================================================================
9281 xoap::MessageReference GatewaySupervisor::supervisorSystemMessage(
9282  xoap::MessageReference message)
9283 {
9284  SOAPParameters parameters;
9285  parameters.addParameter("ToUser");
9286  parameters.addParameter("Subject");
9287  parameters.addParameter("Message");
9288  parameters.addParameter("DoEmail");
9289  SOAPUtilities::receive(message, parameters);
9290 
9291  std::string toUserCSV = parameters.getValue("ToUser");
9292  std::string subject = parameters.getValue("Subject");
9293  std::string systemMessage = parameters.getValue("Message");
9294  std::string doEmail = parameters.getValue("DoEmail");
9295 
9296  //do not uncomment if using custom counts - they will fire recursively if set to generate System Messages
9297  // __COUT__ << "systemMessage -- toUserCSV: " << toUserCSV << ", doEmail: " << doEmail << ", subject: " << subject << ", msg: " << systemMessage << __E__;
9298 
9299  theWebUsers_.addSystemMessage(toUserCSV, subject, systemMessage, doEmail == "1");
9300 
9301  return SOAPUtilities::makeSOAPMessageReference("SystemMessageResponse");
9302 } // end supervisorSystemMessage()
9303 
9304 //===================================================================================================================
9305 //static add system message (e.g. from remote monitoring)
9306 void GatewaySupervisor::addSystemMessage(std::string toUserCSV, std::string message)
9307 {
9308  __COUTTV__(toUserCSV);
9309  __COUTVS__(45, message);
9310  GatewaySupervisor::theWebUsers_.addSystemMessage(toUserCSV, message);
9311 } //end addSystemMessage
9312 
9313 //===================================================================================================================
9317 xoap::MessageReference GatewaySupervisor::supervisorSystemLogbookEntry(
9318  xoap::MessageReference message)
9319 {
9320  SOAPParameters parameters;
9321  parameters.addParameter("EntryText");
9322  SOAPUtilities::receive(message, parameters);
9323 
9324  __COUT__ << "EntryText: " << parameters.getValue("EntryText").substr(0, 10) << __E__;
9325 
9326  makeSystemLogEntry(parameters.getValue("EntryText"));
9327 
9328  return SOAPUtilities::makeSOAPMessageReference("SystemLogbookResponse");
9329 } //end supervisorSystemLogbookEntry()
9330 
9331 //===================================================================================================================
9336 xoap::MessageReference GatewaySupervisor::supervisorLastTableGroupRequest(
9337  xoap::MessageReference message)
9338 {
9339  SOAPParameters parameters;
9340  parameters.addParameter("ActionOfLastGroup");
9341  SOAPUtilities::receive(message, parameters);
9342 
9343  return GatewaySupervisor::lastTableGroupRequestHandler(parameters);
9344 } //end supervisorLastTableGroupRequest()
9345 
9346 //===================================================================================================================
9352 xoap::MessageReference GatewaySupervisor::lastTableGroupRequestHandler(
9353  const SOAPParameters& parameters)
9354 {
9355  std::string action = parameters.getValue("ActionOfLastGroup");
9356  __COUT__ << "ActionOfLastGroup: " << action.substr(0, 30) << __E__;
9357 
9358  std::vector<std::string> actions;
9359  std::vector<std::string> fileNames;
9360  if(action == "ALL")
9361  {
9362  actions = std::vector<std::string>({"Configured",
9363  "Started",
9364  "ActivatedConfig",
9365  "ActivatedContext",
9366  "ActivatedBackbone",
9367  "ActivatedIterator"});
9368  fileNames = std::vector<std::string>(
9369  {FSM_LAST_CONFIGURED_GROUP_ALIAS_FILE,
9370  FSM_LAST_STARTED_GROUP_ALIAS_FILE,
9371  ConfigurationManager::LAST_ACTIVATED_CONFIG_GROUP_FILE,
9372  ConfigurationManager::LAST_ACTIVATED_CONTEXT_GROUP_FILE,
9373  ConfigurationManager::LAST_ACTIVATED_BACKBONE_GROUP_FILE,
9374  ConfigurationManager::LAST_ACTIVATED_ITERATOR_GROUP_FILE});
9375  }
9376  else
9377  {
9378  actions.push_back(action);
9379 
9380  if(action == "Configured")
9381  fileNames.push_back(FSM_LAST_CONFIGURED_GROUP_ALIAS_FILE);
9382  else if(action == "Started")
9383  fileNames.push_back(FSM_LAST_STARTED_GROUP_ALIAS_FILE);
9384  else if(action == "ActivatedConfig")
9385  fileNames.push_back(ConfigurationManager::LAST_ACTIVATED_CONFIG_GROUP_FILE);
9386  else if(action == "ActivatedContext")
9387  fileNames.push_back(ConfigurationManager::LAST_ACTIVATED_CONTEXT_GROUP_FILE);
9388  else if(action == "ActivatedBackbone")
9389  fileNames.push_back(ConfigurationManager::LAST_ACTIVATED_BACKBONE_GROUP_FILE);
9390  else if(action == "ActivatedIterator")
9391  fileNames.push_back(ConfigurationManager::LAST_ACTIVATED_ITERATOR_GROUP_FILE);
9392  else
9393  {
9394  __COUT_ERR__ << "Invalid last group action requested." << __E__;
9395  return SOAPUtilities::makeSOAPMessageReference(
9396  "LastConfigGroupResponseFailure");
9397  }
9398  }
9399 
9400  std::string groupNames = "";
9401  std::string groupKeys = "";
9402  std::string groupActions = "";
9403  std::string groupTimes = "";
9404  for(size_t i = 0; i < fileNames.size(); ++i)
9405  {
9406  if(i)
9407  {
9408  groupNames += ",";
9409  groupKeys += ",";
9410  groupActions += ",";
9411  groupTimes += ",";
9412  }
9413 
9414  std::string timeString;
9415  std::pair<std::string /*group name*/, TableGroupKey> theGroup =
9416  ConfigurationManager::loadGroupNameAndKey(fileNames[i], timeString);
9417 
9418  groupNames += theGroup.first;
9419  groupKeys += theGroup.second.toString();
9420  groupActions += actions[i];
9421  groupTimes += timeString;
9422  }
9423  // fill return parameters
9424  SOAPParameters retParameters;
9425  retParameters.addParameter("GroupName", groupNames); //theGroup.first);
9426  retParameters.addParameter("GroupKey", groupKeys); //theGroup.second.toString());
9427  retParameters.addParameter("GroupAction", groupActions); //action);
9428  retParameters.addParameter("GroupActionTime", groupTimes); //timeString);
9429 
9430  return SOAPUtilities::makeSOAPMessageReference("LastConfigGroupResponse",
9431  retParameters);
9432 } //end lastTableGroupRequestHandler()
9433 
9434 //==============================================================================
9441 unsigned int GatewaySupervisor::getNextRunNumber(const std::string& fsmNameIn)
9442 {
9443  std::string runNumberFileName = RUN_NUMBER_PATH + "/";
9444  std::string fsmName = fsmNameIn == "" ? activeStateMachineName_ : fsmNameIn;
9445  // prepend sanitized FSM name
9446  for(unsigned int i = 0; i < fsmName.size(); ++i)
9447  if((fsmName[i] >= 'a' && fsmName[i] <= 'z') ||
9448  (fsmName[i] >= 'A' && fsmName[i] <= 'Z') ||
9449  (fsmName[i] >= '0' && fsmName[i] <= '9'))
9450  runNumberFileName += fsmName[i];
9451  runNumberFileName += RUN_NUMBER_FILE_NAME;
9452  //__COUT__ << "runNumberFileName: " << runNumberFileName << __E__;
9453 
9454  std::ifstream runNumberFile(runNumberFileName.c_str());
9455  if(!runNumberFile.is_open())
9456  {
9457  __COUT__ << "Cannot open file: " << runNumberFileName << __E__;
9458 
9459  __COUT__ << "Creating file and setting Run Number to 1: " << runNumberFileName
9460  << __E__;
9461  FILE* fp = fopen(runNumberFileName.c_str(), "w");
9462  fprintf(fp, "1");
9463  fclose(fp);
9464 
9465  runNumberFile.open(runNumberFileName.c_str());
9466  if(!runNumberFile.is_open())
9467  {
9468  __SS__ << "Error. Cannot create file: " << runNumberFileName << __E__;
9469  __SS_THROW__;
9470  }
9471  }
9472  std::string runNumberString;
9473  runNumberFile >> runNumberString;
9474  runNumberFile.close();
9475  return atoi(runNumberString.c_str());
9476 } // end getNextRunNumber()
9477 
9478 //==============================================================================
9479 void GatewaySupervisor::setNextRunNumber(unsigned int runNumber,
9480  const std::string& fsmNameIn)
9481 {
9482  std::string runNumberFileName = RUN_NUMBER_PATH + "/";
9483  std::string fsmName = fsmNameIn == "" ? activeStateMachineName_ : fsmNameIn;
9484  // prepend sanitized FSM name
9485  for(unsigned int i = 0; i < fsmName.size(); ++i)
9486  if((fsmName[i] >= 'a' && fsmName[i] <= 'z') ||
9487  (fsmName[i] >= 'A' && fsmName[i] <= 'Z') ||
9488  (fsmName[i] >= '0' && fsmName[i] <= '9'))
9489  runNumberFileName += fsmName[i];
9490  runNumberFileName += RUN_NUMBER_FILE_NAME;
9491  __COUTTV__(runNumberFileName);
9492 
9493  std::ofstream runNumberFile(runNumberFileName.c_str());
9494  if(!runNumberFile.is_open())
9495  {
9496  __SS__ << "Cannot open file: " << runNumberFileName << __E__;
9497  __SS_THROW__;
9498  }
9499  std::stringstream runNumberStream;
9500  runNumberStream << runNumber;
9501  runNumberFile << runNumberStream.str().c_str();
9502  runNumberFile.close();
9503 } // end setNextRunNumber()
9504 
9505 //==============================================================================
9512 std::string GatewaySupervisor::getLastLogEntry(const std::string& logType,
9513  const std::string& fsmNameIn /* = "" */)
9514 {
9515  std::string logEntryFileName = LOG_ENTRY_PATH + "/";
9516  std::string fsmName = fsmNameIn == "" ? activeStateMachineName_ : fsmNameIn;
9517 
9518  if(logType == RunControlStateMachine::START_TRANSITION_NAME &&
9519  stateMachineStartLogEntry_.find(fsmName) != stateMachineStartLogEntry_.end())
9520  return stateMachineStartLogEntry_.at(fsmName);
9521  else if(logType == RunControlStateMachine::CONFIGURE_TRANSITION_NAME &&
9522  stateMachineConfigureLogEntry_.find(fsmName) !=
9523  stateMachineConfigureLogEntry_.end())
9524  return stateMachineConfigureLogEntry_.at(fsmName);
9525  else if(logType == RunControlStateMachine::STOP_TRANSITION_NAME &&
9526  stateMachineStopLogEntry_.find(fsmName) != stateMachineStopLogEntry_.end())
9527  return stateMachineStopLogEntry_.at(fsmName);
9528 
9529  // prepend sanitized FSM name
9530  for(unsigned int i = 0; i < fsmName.size(); ++i)
9531  if((fsmName[i] >= 'a' && fsmName[i] <= 'z') ||
9532  (fsmName[i] >= 'A' && fsmName[i] <= 'Z') ||
9533  (fsmName[i] >= '0' && fsmName[i] <= '9'))
9534  logEntryFileName += fsmName[i];
9535  logEntryFileName += "_" + logType + "_" + LOG_ENTRY_FILE_NAME;
9536  __SUP_COUTTV__(logEntryFileName);
9537 
9538  std::string contents;
9539  std::FILE* fp = std::fopen(logEntryFileName.c_str(), "rb");
9540  if(!fp)
9541  {
9542  __SUP_COUTT__ << "Could not open file at " << logEntryFileName
9543  << ". Error: " << errno << " - " << strerror(errno) << __E__;
9544  contents = "";
9545  }
9546  else
9547  {
9548  std::fseek(fp, 0, SEEK_END);
9549  contents.resize(std::ftell(fp));
9550  std::rewind(fp);
9551  std::fread(&contents[0], 1, contents.size(), fp);
9552  std::fclose(fp);
9553  }
9554 
9555  __SUP_COUTTV__(contents);
9556 
9557  if(logType == RunControlStateMachine::START_TRANSITION_NAME)
9558  stateMachineStartLogEntry_[fsmName] = contents;
9559  else if(logType == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
9560  stateMachineConfigureLogEntry_[fsmName] = contents;
9561  else if(logType == RunControlStateMachine::STOP_TRANSITION_NAME)
9562  stateMachineStopLogEntry_[fsmName] = contents;
9563 
9564  return contents;
9565 } // end getLastLogEntry()
9566 
9567 //==============================================================================
9574 void GatewaySupervisor::setLastLogEntry(const std::string& logType,
9575  const std::string& logEntry,
9576  const std::string& fsmNameIn /* = "" */)
9577 {
9578  std::string logEntryFileName = LOG_ENTRY_PATH + "/";
9579  std::string fsmName = fsmNameIn == "" ? activeStateMachineName_ : fsmNameIn;
9580 
9581  if(logType == RunControlStateMachine::START_TRANSITION_NAME)
9582  stateMachineStartLogEntry_[fsmName] = logEntry;
9583  else if(logType == RunControlStateMachine::CONFIGURE_TRANSITION_NAME)
9584  stateMachineConfigureLogEntry_[fsmName] = logEntry;
9585  else if(logType == RunControlStateMachine::STOP_TRANSITION_NAME)
9586  stateMachineStopLogEntry_[fsmName] = logEntry;
9587  else
9588  {
9589  if(logEntry != "")
9590  __COUT_WARN__ << "Log entry for log type '" << logType
9591  << "' not implemented for saving." << __E__;
9592  return; //for now, do not save other types of transitions
9593  }
9594 
9595  // prepend sanitized FSM name
9596  for(unsigned int i = 0; i < fsmName.size(); ++i)
9597  if((fsmName[i] >= 'a' && fsmName[i] <= 'z') ||
9598  (fsmName[i] >= 'A' && fsmName[i] <= 'Z') ||
9599  (fsmName[i] >= '0' && fsmName[i] <= '9'))
9600  logEntryFileName += fsmName[i];
9601  logEntryFileName += "_" + logType + "_" + LOG_ENTRY_FILE_NAME;
9602  __COUTTV__(logEntryFileName);
9603  __COUTTV__(logType);
9604  __COUTTV__(logEntry);
9605 
9606  std::FILE* fp = std::fopen(logEntryFileName.c_str(), "w");
9607  if(!fp)
9608  {
9609  __SUP_SS__ << "Could not open file at " << logEntryFileName
9610  << ". Error: " << errno << " - " << strerror(errno) << __E__;
9611  __SUP_SS_THROW__;
9612  }
9613  if(logEntry.size())
9614  std::fwrite(&logEntry[0], 1, logEntry.size(), fp);
9615  fclose(fp);
9616 } // end setLastLogEntry()
9617 
9618 //==============================================================================
9625 void GatewaySupervisor::loadRemoteGatewaySettings(
9626  std::vector<GatewaySupervisor::RemoteGatewayInfo>& remoteGateways,
9627  bool onlyNotFound /* = false */) const
9628 {
9629  std::string filepath = std::string(__ENV__("SERVICE_DATA_PATH")) + "/" +
9630  REMOTE_SUBSYSTEM_SETTINGS_FILE_NAME;
9631  __SUP_COUTV__(filepath);
9632 
9633  std::ifstream settingsFile(filepath.c_str());
9634  if(!settingsFile.is_open())
9635  {
9636  __SUP_COUT__
9637  << "Cannot open Remote Gateway settings file (assuming no settings yet!): "
9638  << filepath << __E__;
9639 
9640  __SUP_COUT__ << "Creating empty Remote Gateway settings file: " << filepath
9641  << __E__;
9642  FILE* fp = fopen(filepath.c_str(), "w");
9643  fprintf(fp, "\n");
9644  fclose(fp);
9645 
9646  settingsFile.open(filepath.c_str());
9647  if(!settingsFile.is_open())
9648  {
9649  __SUP_SS__ << "Error. Cannot create or load Remote Gateway settings file: "
9650  << filepath << __E__;
9651  __SUP_SS_THROW__;
9652  }
9653  }
9654 
9655  size_t NUM_FIELDS = 4; //name, fsmMode, included, selected alias
9656  std::vector<std::string> values;
9657  float formatVersion = 0.0;
9658  bool done = false;
9659  do // Read each line from the file
9660  {
9661  size_t i = 0;
9662  for(i = 0; i < NUM_FIELDS; ++i)
9663  {
9664  if(i >= values.size())
9665  values.push_back(""); //init values vector
9666 
9667  if(!std::getline(settingsFile, values[i]))
9668  {
9669  //no more lines left
9670  if(i) //at illegal moment mid-record?
9671  {
9672  settingsFile.close();
9673  __SUP_SS__
9674  << "Error. Illegal file format in Remote Gateway settings file: "
9675  << filepath << __E__;
9676  __SUP_SS_THROW__;
9677  }
9678  //else end is correctly at record boundary
9679  done = true;
9680  break;
9681  }
9682  __SUP_COUTVS__(20, values[i]);
9683 
9684  if(i < 3 &&
9685  values[i] == "") //do not allow blank lines, except for selected alias
9686  {
9687  //rewind
9688  --i;
9689  continue;
9690  }
9691  else if(
9692  values[i].find(
9693  "Remote Gateway Settings, file format v") != //grab format version if present
9694  std::string::npos)
9695  {
9696  sscanf(values[i].c_str(),
9697  "Remote Gateway Settings, file format v%f",
9698  &formatVersion);
9699  __SUP_COUTV__(formatVersion);
9700 
9701  if(formatVersion > 0.5)
9702  NUM_FIELDS = 4; //name, fsmMode, included, selected_config_alias
9703 
9704  __SUP_COUTV__(NUM_FIELDS);
9705  //rewind
9706  --i;
9707  continue;
9708  }
9709 
9710  } //end record value load
9711  if(done)
9712  break;
9713 
9714  //at this point values vector complete for Remote Gateway
9715 
9716  bool found = false;
9717  for(i = 0; i < remoteGateways.size(); ++i)
9718  if(values[0] == remoteGateways[i].appInfo.name)
9719  {
9720  found = true;
9721  break;
9722  }
9723 
9724  if(!found) //create Remote Gateway (and i will be correctly pointing to back())
9725  {
9726  remoteGateways.push_back(GatewaySupervisor::RemoteGatewayInfo());
9727  remoteGateways[i].appInfo.name = values[0];
9728  }
9729  else if(onlyNotFound)
9730  continue; //skip modifying current settings
9731 
9732  remoteGateways[i].fsm_mode =
9733  values[1] == "Do Not Halt"
9734  ? RemoteGatewayInfo::FSM_ModeTypes::DoNotHalt
9735  : (values[1] == "Only Configure"
9736  ? RemoteGatewayInfo::FSM_ModeTypes::OnlyConfigure
9737  : RemoteGatewayInfo::FSM_ModeTypes::Follow_FSM);
9738  remoteGateways[i].fsm_included = values[2] == "1" ? true : false;
9739  if(values.size() > 3)
9740  remoteGateways[i].selected_config_alias = values[3];
9741 
9742  __SUP_COUT__ << "Loaded Remote Gateway '" << remoteGateways[i].appInfo.name
9743  << "' ==> " << remoteGateways[i].getFsmMode() << " :"
9744  << remoteGateways[i].fsm_included
9745  << "configAlias=" << remoteGateways[i].selected_config_alias
9746  << __E__;
9747 
9748  } while(1); //end file read loop
9749 
9750  settingsFile.close();
9751 } //end loadRemoteGatewaySettings()
9752 
9753 //==============================================================================
9754 void GatewaySupervisor::saveRemoteGatewaySettings() const
9755 {
9756  std::string filepath = std::string(__ENV__("SERVICE_DATA_PATH")) + "/" +
9757  REMOTE_SUBSYSTEM_SETTINGS_FILE_NAME;
9758  __SUP_COUTV__(filepath);
9759 
9760  std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteGateways = remoteGatewayApps_;
9761 
9762  //load existing settings for remote gateways not present
9763  loadRemoteGatewaySettings(remoteGateways, true /* onlyNotFound*/);
9764 
9765  std::ofstream settingsFile(filepath.c_str());
9766  if(!settingsFile.is_open())
9767  {
9768  __SUP_SS__ << "Cannot open Remote Gateway settings file: " << filepath << __E__;
9769  __SUP_SS_THROW__;
9770  }
9771  settingsFile << "Remote Gateway Settings, file format v1.0"
9772  << __E__; //save file format version first
9773  for(size_t i = 0; i < remoteGateways.size(); ++i)
9774  {
9775  settingsFile << remoteGateways[i].appInfo.name << __E__;
9776  settingsFile << remoteGateways[i].getFsmMode() << __E__;
9777  settingsFile << std::string(remoteGateways[i].fsm_included ? "1" : "0") << __E__;
9778  settingsFile << remoteGateways[i].selected_config_alias << __E__;
9779  }
9780 
9781  settingsFile.close();
9782 } // end saveRemoteGatewaySettings()
9783 
9784 //==============================================================================
9785 void GatewaySupervisor::handleGetApplicationIdRequest(
9786  AllSupervisorInfo* allSupervisorInfo, cgicc::Cgicc& cgiIn, HttpXmlDocument& xmlOut)
9787 {
9788  std::string classNeedle =
9789  StringMacros::decodeURIComponent(CgiDataUtilities::getData(cgiIn, "classNeedle"));
9790  __COUTV__(classNeedle);
9791 
9792  for(auto it : allSupervisorInfo->getAllSupervisorInfo())
9793  {
9794  // bool pass = true;
9795 
9796  auto appInfo = it.second;
9797 
9798  if(classNeedle != appInfo.getClass())
9799  continue; // skip non-matches
9800 
9801  xmlOut.addTextElementToData("name",
9802  appInfo.getName()); // get application name
9803  xmlOut.addTextElementToData(
9804  "id", std::to_string(appInfo.getId())); // get application id
9805  xmlOut.addTextElementToData("class",
9806  appInfo.getClass()); // get application class
9807  xmlOut.addTextElementToData("url",
9808  appInfo.getURL()); // get application url
9809  xmlOut.addTextElementToData("context",
9810  appInfo.getContextName()); // get context
9811  }
9812 
9813 } // end handleGetApplicationIdRequest()
9814 
9815 //==============================================================================
9816 bool GatewaySupervisor::handleAddDesktopIconRequest(
9817  const std::string& author,
9818  cgicc::Cgicc& cgiIn,
9819  HttpXmlDocument& xmlOut,
9820  std::vector<DesktopIconTable::DesktopIcon>* newIcons /* = nullptr*/)
9821 {
9822  std::string iconCaption =
9823  CgiDataUtilities::getData(cgiIn, "iconCaption"); // from GET
9824  std::string iconAltText =
9825  CgiDataUtilities::getData(cgiIn, "iconAltText"); // from GET
9826  std::string iconFolderPath =
9827  CgiDataUtilities::getData(cgiIn, "iconFolderPath"); // from GET
9828  std::string iconImageURL =
9829  CgiDataUtilities::getData(cgiIn, "iconImageURL"); // from GET
9830  std::string iconWindowURL =
9831  CgiDataUtilities::getData(cgiIn, "iconWindowURL"); // from GET
9832  std::string iconPermissions =
9833  CgiDataUtilities::getData(cgiIn, "iconPermissions"); // from GET
9834  // windowLinkedApp is one of the only fields that needs to be decoded before write into table cells, because the app class name might be here
9835  std::string windowLinkedApp =
9836  CgiDataUtilities::getData(cgiIn, "iconLinkedApp"); // from GET
9837  unsigned int windowLinkedAppLID =
9838  CgiDataUtilities::getDataAsInt(cgiIn, "iconLinkedAppLID"); // from GET
9839  bool enforceOneWindowInstance =
9840  CgiDataUtilities::getData(cgiIn, "iconEnforceOneWindowInstance") == "1"
9841  ? true
9842  : false; // from GET
9843 
9844  std::string windowParameters = StringMacros::decodeURIComponent(
9845  CgiDataUtilities::postData(cgiIn, "iconParameters")); // from POST
9846 
9847  __COUTV__(author);
9848  __COUTV__(iconCaption);
9849  __COUTV__(iconAltText);
9850  __COUTV__(iconFolderPath);
9851  __COUTV__(iconImageURL);
9852  __COUTV__(iconWindowURL);
9853  __COUTV__(iconPermissions);
9854  __COUTV__(windowLinkedApp);
9855  __COUTV__(windowLinkedAppLID);
9856  __COUTV__(enforceOneWindowInstance);
9857 
9858  __COUTV__(windowParameters); // map: CSV list
9859 
9860  ConfigurationManagerRW tmpCfgMgr(author);
9861 
9862  bool success = ConfigurationSupervisorBase::handleAddDesktopIconXML(
9863  xmlOut,
9864  &tmpCfgMgr,
9865  iconCaption,
9866  iconAltText,
9867  iconFolderPath,
9868  iconImageURL,
9869  iconWindowURL,
9870  iconPermissions,
9871  windowLinkedApp /*= ""*/,
9872  windowLinkedAppLID /*= 0*/,
9873  enforceOneWindowInstance /*= false*/,
9874  windowParameters /*= ""*/);
9875 
9876  if(newIcons && success)
9877  {
9878  __COUT__ << "Passing new icons back to caller..." << __E__;
9879 
9880  const std::vector<DesktopIconTable::DesktopIcon>& tmpNewIcons =
9881  tmpCfgMgr.__GET_CONFIG__(DesktopIconTable)->getAllDesktopIcons();
9882 
9883  newIcons->clear();
9884  for(const auto& tmpNewIcon : tmpNewIcons)
9885  newIcons->push_back(tmpNewIcon);
9886  }
9887 
9888  return success;
9889 } // end handleAddDesktopIconRequest()
9890 
9891 //==============================================================================
9892 xoap::MessageReference GatewaySupervisor::TRACESupervisorRequest(
9893  xoap::MessageReference message)
9894 {
9895  return CorePropertySupervisorBase::TRACESupervisorRequest(message);
9896 } // end TRACESupervisorRequest()
9897 
ots::AllSupervisorInfo::setSupervisorStatus
void setSupervisorStatus(xdaq::Application *app, const std::string &status, const unsigned int progress=100, const std::string &detail="", std::vector< SupervisorInfo::SubappInfo > subapps={})
SETTERs.
Definition: AllSupervisorInfo.cc:352
ots::AllSupervisorInfo::getAllSupervisorInfo
const std::map< unsigned int, SupervisorInfo > & getAllSupervisorInfo(void) const
GETTERs (so searching and iterating is easier)
Definition: AllSupervisorInfo.h:51
ots::CgiDataUtilities::postData
static std::string postData(cgicc::Cgicc &cgi, const std::string &needle)
Definition: CgiDataUtilities.cc:25
ots::CgiDataUtilities::getData
static std::string getData(cgicc::Cgicc &cgi, const std::string &needle)
Definition: CgiDataUtilities.cc:53
ots::ConfigurationManager::loadTableGroup
void loadTableGroup(const std::string &tableGroupName, const TableGroupKey &tableGroupKey, bool doActivate=false, std::map< std::string, TableVersion > *groupMembers=0, ProgressBar *progressBar=0, std::string *accumulateWarnings=0, std::string *groupComment=0, std::string *groupAuthor=0, std::string *groupCreateTime=0, bool doNotLoadMember=false, std::string *groupTypeString=0, std::map< std::string, std::string > *groupAliases=0, ConfigurationManager::LoadGroupType onlyLoadIfBackboneOrContext=ConfigurationManager::LoadGroupType::ALL_TYPES, bool ignoreVersionTracking=false)
Definition: ConfigurationManager.cc:1512
ots::ConfigurationManager::getOtherSubsystemConfigAliases
std::set< std::string > getOtherSubsystemConfigAliases(const std::string &otherSubsystemUID)
Ignore any System Aliases with "Context" or "Iterat" in the name.
Definition: ConfigurationManager.cc:4168
ots::ConfigurationManager::getNode
ConfigurationTree getNode(const std::string &nodeString, bool doNotThrowOnBrokenUIDLinks=false) const
"root/parent/parent/"
Definition: ConfigurationManager.cc:2845
ots::ConfigurationManager::init
void init(std::string *accumulatedErrors=0, bool initForWriteAccess=false, std::string *accumulatedWarnings=0)
Definition: ConfigurationManager.cc:230
ots::ConfigurationManager::getOtherSubsystemConfigAliasInfo
void getOtherSubsystemConfigAliasInfo(const std::string &otherSubsystemUID, const std::string &configAlias, std::pair< std::string, TableGroupKey > &groupTranslation, std::string &groupComment, std::string &groupAuthor, std::string &groupCreationTime)
returns configAlias translation group info by reference
Definition: ConfigurationManager.cc:4404
ots::ConfigurationManager::loadGroupNameAndKey
static std::pair< std::string, TableGroupKey > loadGroupNameAndKey(const std::string &fileName, std::string &returnedTimeString)
Definition: ConfigurationManager.cc:4537
ots::ConfigurationManager::getActiveGroupAliases
std::map< std::string, std::pair< std::string, TableGroupKey > > getActiveGroupAliases(void)
Definition: ConfigurationManager.cc:3263
ots::ConfigurationTree::getTableVersion
const TableVersion & getTableVersion(void) const
getTableVersion
Definition: ConfigurationTree.cc:569
ots::ConfigurationTree::isDisconnected
bool isDisconnected(void) const
Definition: ConfigurationTree.cc:1477
ots::ConfigurationTree::getNode
ConfigurationTree getNode(const std::string &nodeName, bool doNotThrowOnBrokenUIDLinks=false) const
navigating between nodes
Definition: ConfigurationTree.cc:994
ots::ConfigurationTree::getValueWithDefault
T getValueWithDefault(const T &defaultValue) const
ots::ConfigurationTree::getValue
void getValue(T &value) const
ots::ConfigurationTree::getChildren
std::vector< std::pair< std::string, ConfigurationTree > > getChildren(std::map< std::string, std::string > filterMap=std::map< std::string, std::string >(), bool byPriority=false, bool onlyStatusTrue=false) const
Definition: ConfigurationTree.cc:2411
ots::ConfigurationTree::isDefaultValue
bool isDefaultValue(void) const
boolean info
Definition: ConfigurationTree.cc:797
ots::CorePropertySupervisorBase::extractPermissionsMapFromString
static void extractPermissionsMapFromString(const std::string &permissionsString, std::map< std::string, WebUsers::permissionLevel_t > &permissionsMap)
Definition: CorePropertySupervisorBase.cc:347
ots::CorePropertySupervisorBase::doPermissionsGrantAccess
static bool doPermissionsGrantAccess(std::map< std::string, WebUsers::permissionLevel_t > &permissionLevelsMap, std::map< std::string, WebUsers::permissionLevel_t > &permissionThresholdsMap)
Definition: CorePropertySupervisorBase.cc:378
ots::CorePropertySupervisorBase::GatewaySupervisor
friend class GatewaySupervisor
for access to indicateOtsAlive()
Definition: CorePropertySupervisorBase.h:56
ots::CorePropertySupervisorBase::getRequestUserInfo
void getRequestUserInfo(WebUsers::RequestUserInfo &requestUserInfo)
Definition: CorePropertySupervisorBase.cc:629
ots::DesktopIconTable::getRemoteURL
std::string getRemoteURL(ConfigurationManager *configManager, const std::string &localURL) const
Convert to remote URL assuming port forwarding to primary Gateway Port.
Definition: DesktopIconTableImpl.cc:268
ots::DesktopIconTable::setAllDesktopIcons
void setAllDesktopIcons(const std::vector< DesktopIconTable::DesktopIcon > &newIcons)
overwrite dynamically the init result
Definition: DesktopIconTableImpl.cc:347
ots::FiniteStateMachine::getTimeInState
time_t getTimeInState(void) const
Definition: FiniteStateMachine.cc:79
ots::GatewaySupervisor::setSupervisorPropertyDefaults
virtual void setSupervisorPropertyDefaults(void) override
override to control supervisor specific defaults
ots::GatewaySupervisor::stateHalted
void stateHalted(toolbox::fsm::FiniteStateMachine &fsm) override
ots::GatewaySupervisor::forceSupervisorPropertyValues
virtual void forceSupervisorPropertyValues(void) override
override to force supervisor property values (and ignore user settings)
ots::GatewaySupervisor::statePaused
void statePaused(toolbox::fsm::FiniteStateMachine &fsm) override
ots::GatewaySupervisor::stateRunning
void stateRunning(toolbox::fsm::FiniteStateMachine &fsm) override
ots::GatewaySupervisor::stateConfigured
void stateConfigured(toolbox::fsm::FiniteStateMachine &fsm) override
ots::GatewaySupervisor::stateInitial
void stateInitial(toolbox::fsm::FiniteStateMachine &fsm) override
ots::HttpXmlDocument::outputXmlDocument
void outputXmlDocument(std::ostringstream *out, bool dispStdOut=false, bool allowWhiteSpace=false)
Definition: HttpXmlDocument.cc:236
ots::ProgressBar::readPercentageString
std::string readPercentageString()
return percentage complete as std::string
Definition: ProgressBar.cc:136
ots::RunControlStateMachine::runControlMessageHandler
xoap::MessageReference runControlMessageHandler(xoap::MessageReference message)
Run Control Messages.
Definition: RunControlStateMachine.cc:333
ots::SOAPCommand::getCommand
const std::string & getCommand(void) const
Getters.
Definition: SOAPCommand.cc:44
ots::SOAPMessenger::send
std::string send(XDAQ_CONST_CALL xdaq::ApplicationDescriptor *d, xoap::MessageReference message)
Definition: SOAPMessenger.cc:39
ots::SupervisorInfo::getDescriptor
XDAQ_CONST_CALL xdaq::ApplicationDescriptor * getDescriptor(void) const
Getters ----------------—.
Definition: SupervisorInfo.h:90
ots::TableGroupKey::getFullGroupString
static std::string getFullGroupString(const std::string &groupName, const TableGroupKey &key, const std::string &preKey="_v", const std::string &postKey="")
Definition: TableGroupKey.cc:174
ots::WebUsers::addSystemMessage
void addSystemMessage(const std::string &targetUsersCSV, const std::string &message)
Definition: WebUsers.cc:3495
ots::WebUsers::getSecurity
const std::string & getSecurity(void)
WebUsers::getSecurity.
Definition: WebUsers.cc:3900
ots::WebUsers::getGenericPreference
std::string getGenericPreference(uint64_t uid, const std::string &preferenceName, HttpXmlDocument *xmldoc=0) const
Definition: WebUsers.cc:3063
ots::WebUsers::setUserWithLock
bool setUserWithLock(uint64_t actingUid, bool lock, const std::string &username)
Definition: WebUsers.cc:3168
ots::WebUsers::silenceAllUserTooltips
static void silenceAllUserTooltips(const std::string &username)
Definition: WebUsers.cc:2867
ots::WebUsers::getActiveUserCount
size_t getActiveUserCount(void)
Definition: WebUsers.cc:3396
ots::WebUsers::getPermissionsForUser
std::map< std::string, WebUsers::permissionLevel_t > getPermissionsForUser(uint64_t uid)
from Gateway, use public version which considers remote users
Definition: WebUsers.cc:2568
ots::WebUsers::attemptActiveSession
uint64_t attemptActiveSession(const std::string &uuid, std::string &jumbledUser, const std::string &jumbledPw, std::string &newAccountCode, const std::string &ip)
Definition: WebUsers.cc:1106
ots::WebUsers::setGenericPreference
void setGenericPreference(uint64_t uid, const std::string &preferenceName, const std::string &preferenceValue)
Definition: WebUsers.cc:3022
ots::WebUsers::getAllSystemMessages
std::string getAllSystemMessages(void)
Definition: WebUsers.cc:3754
ots::WebUsers::cleanupExpiredEntries
void cleanupExpiredEntries(std::vector< std::string > *loggedOutUsernames=0)
Definition: WebUsers.cc:2308
ots::WebUsers::changeSettingsForUser
void changeSettingsForUser(uint64_t uid, const std::string &bgcolor, const std::string &dbcolor, const std::string &wincolor, const std::string &layout, const std::string &syslayout)
WebUsers::changeSettingsForUser.
Definition: WebUsers.cc:3119
ots::WebUsers::isCookieCodeActiveForLogin
uint64_t isCookieCodeActiveForLogin(const std::string &uuid, std::string &cookieCode, std::string &username)
Definition: WebUsers.cc:1858
ots::WebUsers::createNewLoginSession
std::string createNewLoginSession(const std::string &uuid, const std::string &ip)
Definition: WebUsers.cc:2429
ots::WebUsers::getActiveUsersString
std::string getActiveUsersString(void)
Definition: WebUsers.cc:3408
ots::WebUsers::modifyAccountSettings
void modifyAccountSettings(uint64_t actingUid, uint8_t cmd_type, const std::string &username, const std::string &displayname, const std::string &email, const std::string &permissions)
WebUsers::modifyAccountSettings.
Definition: WebUsers.cc:3236
ots::WebUsers::remoteLoginVerificationPort_
int remoteLoginVerificationPort_
Port of remote Gateway to be used for login verification.
Definition: WebUsers.h:651
ots::WebUsers::isUserIdActive
bool isUserIdActive(uint64_t uid) const
Definition: WebUsers.cc:1625
ots::WebUsers::saveActiveSessions
void saveActiveSessions(void)
Definition: WebUsers.cc:402
ots::WebUsers::remoteLoginVerificationEnabled_
static std::atomic< bool > remoteLoginVerificationEnabled_
true if this supervisor is under control of a remote supervisor
Definition: WebUsers.h:649
ots::WebUsers::getUsersUsername
std::string getUsersUsername(uint64_t uid)
from Gateway, use public version which considers remote users
Definition: WebUsers.cc:2038
ots::WebUsers::xmlRequestOnGateway
bool xmlRequestOnGateway(cgicc::Cgicc &cgi, std::ostringstream *out, HttpXmlDocument *xmldoc, WebUsers::RequestUserInfo &userInfo)
Definition: WebUsers.cc:175
ots::WebUsers::cookieCodeLogout
uint64_t cookieCodeLogout(const std::string &cookieCode, bool logoutOtherUserSessions, uint64_t *uid=0, const std::string &ip="0")
Definition: WebUsers.cc:2057
ots::WebUsers::getSystemMessage
std::string getSystemMessage(const std::string &targetUser)
Definition: WebUsers.cc:3786
ots::WebUsers::getActiveSessionCountForUser
uint64_t getActiveSessionCountForUser(uint64_t uid)
Definition: WebUsers.cc:1915
ots::WebUsers::resetAllUserTooltips
static void resetAllUserTooltips(const std::string &userNeedle="*")
WebUsers::resetAllUserTooltips.
Definition: WebUsers.cc:2856
ots::WebUsers::tooltipSetNeverShowForUsername
static void tooltipSetNeverShowForUsername(const std::string &username, HttpXmlDocument *xmldoc, const std::string &srcFile, const std::string &srcFunc, const std::string &srcId, bool doNeverShow, bool temporarySilence)
Definition: WebUsers.cc:2733
ots::WebUsers::getUsersDisplayName
std::string getUsersDisplayName(uint64_t uid)
from Gateway, use public version which considers remote users
Definition: WebUsers.cc:2028
ots::WebUsers::getLastSystemMessage
std::pair< std::string, time_t > getLastSystemMessage(void)
Definition: WebUsers.cc:3733
ots::WebUsers::attemptActiveSessionWithCert
uint64_t attemptActiveSessionWithCert(const std::string &uuid, std::string &jumbledEmail, std::string &cookieCode, std::string &username, const std::string &ip)
Definition: WebUsers.cc:1298
ots::WebUsers::tooltipCheckForUsername
static void tooltipCheckForUsername(const std::string &username, HttpXmlDocument *xmldoc, const std::string &srcFile, const std::string &srcFunc, const std::string &srcId)
Definition: WebUsers.cc:2791
ots::WebUsers::remoteGatewaySelfName_
std::string remoteGatewaySelfName_
IP of remote Gateway to be used for login verification.
Definition: WebUsers.h:650
ots::WebUsers::cookieCodeIsActiveForRequest
bool cookieCodeIsActiveForRequest(std::string &cookieCode, std::map< std::string, WebUsers::permissionLevel_t > *userPermissions=0, uint64_t *uid=0, const std::string &ip="0", bool refresh=true, bool doNotGoRemote=false, std::string *userWithLock=0, uint64_t *userSessionIndex=0)
Definition: WebUsers.cc:2130
ots::WebUsers::insertSettingsForUser
void insertSettingsForUser(uint64_t uid, HttpXmlDocument *xmldoc, bool includeAccounts=false)
Definition: WebUsers.cc:2902
ots::WorkLoopManager::processRequest
HttpXmlDocument processRequest(cgicc::Cgicc &cgi)
Definition: WorkLoopManager.cc:28
ots::XmlDocument::addTextElementToParent
xercesc::DOMElement * addTextElementToParent(const std::string &childName, const std::string &childText, xercesc::DOMElement *parent)
Definition: XmlDocument.cc:190
ots::INIT_MF
void INIT_MF(const char *name)
Definition: MessageFacility.h:13
ots::makeRunInfo
RunInfoVInterface * makeRunInfo(const std::string &runInfoPluginName, const std::string &runInfoUID)
Definition: MakeRunInfo.cc:6
ots::StringMacros::getTimestampString
static std::string getTimestampString(const std::string &linuxTimeInSeconds)
Definition: StringMacros.cc:636
ots::StringMacros::extractXmlField
static std::string extractXmlField(const std::string &xml, const std::string &field, uint32_t occurrence, size_t after, size_t *returnFindPos=nullptr, const std::string &valueField="value=", const std::string &quoteType="'")
Definition: StringMacros.cc:1477
ots::StringMacros::getVectorFromString
static void getVectorFromString(const std::string &inputString, std::vector< std::string > &listToReturn, const std::set< char > &delimiter={',', '|', '&'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'}, std::vector< char > *listOfDelimiters=0, bool decodeURIComponents=false)
Definition: StringMacros.cc:749
ots::StringMacros::exec
static std::string exec(const char *cmd)
Definition: StringMacros.cc:1340
ots::StringMacros::setToString
static std::string setToString(const std::set< T > &setToReturn, const std::string &delimeter=", ")
setToString ~
ots::StringMacros::vectorToString
static std::string vectorToString(const std::vector< T > &setToReturn, const std::string &delimeter=", ")
vectorToString ~
ots::StringMacros::mapToString
static std::string mapToString(const std::map< std::string, T > &mapToReturn, const std::string &primaryDelimeter=", ", const std::string &secondaryDelimeter=": ")
ots::StringMacros::getMapFromString
static void getMapFromString(const std::string &inputString, std::map< S, T > &mapToReturn, const std::set< char > &pairPairDelimiter={',', '|', '&'}, const std::set< char > &nameValueDelimiter={'=', ':'}, const std::set< char > &whitespace={' ', '\t', '\n', '\r'})
getMapFromString ~
ots::StringMacros::getTimeDurationString
static std::string getTimeDurationString(const time_t durationInSeconds=time(0))
Definition: StringMacros.cc:661
ots::StringMacros::decodeURIComponent
static std::string decodeURIComponent(const std::string &data)
Definition: StringMacros.cc:115