Monitoring JSON changes in DAQ (80X)

DQMServices/Components/src/DQMFileSaver.cc

@@ -291,7 +291,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
   }
   // The availability test of the FastMonitoringService was done in the ctor.
   bpt::ptree data;
-  bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination;
+  bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination, hltErrorEvents;
 
   processedEvents.put("", nProcessed); // Processed events
   acceptedEvents.put("", nProcessed); // Accepted events, same as processed for our purposes
@@ -303,6 +303,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
   inputFiles.put("", ""); // We do not care about input files!
   fileAdler32.put("", -1); // placeholder to match output json definition
   transferDestination.put("", transferDestinationStr); // SM Transfer destination field
+  hltErrorEvents.put("", 0); // Error events
 
   data.push_back(std::make_pair("", processedEvents));
   data.push_back(std::make_pair("", acceptedEvents));
@@ -313,6 +314,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
   data.push_back(std::make_pair("", inputFiles));
   data.push_back(std::make_pair("", fileAdler32));
   data.push_back(std::make_pair("", transferDestination));
+  data.push_back(std::make_pair("", hltErrorEvents));
 
   pt.add_child("data", data);
 

EventFilter/Utilities/interface/FastMonitoringService.h

@@ -112,8 +112,8 @@ namespace evf{
       FastMonitoringService(const edm::ParameterSet&,edm::ActivityRegistry&);
       ~FastMonitoringService();
 
-      std::string makePathLegenda();
-      std::string makeModuleLegenda();
+      std::string makePathLegendaJson();
+      std::string makeModuleLegendaJson();
 
       void preallocate(edm::service::SystemBounds const&);
       void jobFailure();
@@ -258,8 +258,11 @@ namespace evf{
       std::atomic<unsigned long> totalEventsProcessed_;
 
       std::string moduleLegendFile_;
+      std::string moduleLegendFileJson_;
       std::string pathLegendFile_;
+      std::string pathLegendFileJson_;
       bool pathLegendWritten_ = false;
+      unsigned int nOutputModules_ =0;
 
       std::atomic<bool> monInit_;
       bool exception_detected_ = false;

EventFilter/Utilities/plugins/RecoEventOutputModuleForFU.h

@@ -61,6 +61,7 @@ namespace evf {
     jsoncollector::StringJ inputFiles_;
     jsoncollector::IntJ fileAdler32_; 
     jsoncollector::StringJ transferDestination_; 
+    jsoncollector::IntJ hltErrorEvents_; 
     boost::shared_ptr<jsoncollector::FastMonitor> jsonMonitor_;
     evf::FastMonitoringService *fms_;
     jsoncollector::DataPointDefinition outJsonDef_;
@@ -85,6 +86,7 @@ namespace evf {
     inputFiles_(),
     fileAdler32_(1),
     transferDestination_(),
+    hltErrorEvents_(0),
     outBuf_(new unsigned char[1024*1024])
   {
     std::string baseRunDir = edm::Service<evf::EvFDaqDirector>()->baseRunDir();
@@ -122,6 +124,7 @@ namespace evf {
     inputFiles_.setName("InputFiles");
     fileAdler32_.setName("FileAdler32");
     transferDestination_.setName("TransferDestination");
+    hltErrorEvents_.setName("HLTErrorEvents");
 
     outJsonDef_.setDefaultGroup("data");
     outJsonDef_.addLegendItem("Processed","integer",jsoncollector::DataPointDefinition::SUM);
@@ -133,6 +136,7 @@ namespace evf {
     outJsonDef_.addLegendItem("InputFiles","string",jsoncollector::DataPointDefinition::CAT);
     outJsonDef_.addLegendItem("FileAdler32","integer",jsoncollector::DataPointDefinition::ADLER32);
     outJsonDef_.addLegendItem("TransferDestination","string",jsoncollector::DataPointDefinition::SAME);
+    outJsonDef_.addLegendItem("HLTErrorEvents","integer",jsoncollector::DataPointDefinition::SUM);
     std::stringstream tmpss,ss;
     tmpss << baseRunDir << "/open/" << "output_" << getpid() << ".jsd";
     ss << baseRunDir << "/" << "output_" << getpid() << ".jsd";
@@ -161,6 +165,7 @@ namespace evf {
     jsonMonitor_->registerGlobalMonitorable(&inputFiles_,false);
     jsonMonitor_->registerGlobalMonitorable(&fileAdler32_,false);
     jsonMonitor_->registerGlobalMonitorable(&transferDestination_,false);
+    jsonMonitor_->registerGlobalMonitorable(&hltErrorEvents_,false);
     jsonMonitor_->commit(nullptr);
 
   }

EventFilter/Utilities/plugins/output.jsd

@@ -44,6 +44,11 @@
          "name" : "TransferDestination",
          "operation" : "same",
          "type" : "string"
+      },
+      {
+         "name" : "HLTErrorEvents",
+         "operation" : "sum",
+         "type" : "integer"
       }
 
    ]

EventFilter/Utilities/src/FastMonitoringService.cc

@@ -21,6 +21,10 @@ using namespace jsoncollector;
 
 constexpr double throughputFactor() {return (1000000)/double(1024*1024);}
 
+static const int nReservedModules = 33;
+static const int nSpecialModules = 7;
+static const int nReservedPaths = 1;
+
 namespace evf{
 
   const std::string FastMonitoringService::macroStateNames[FastMonitoringThread::MCOUNT] = 
@@ -33,7 +37,7 @@ namespace evf{
   FastMonitoringService::FastMonitoringService(const edm::ParameterSet& iPS, 
 				       edm::ActivityRegistry& reg) : 
     MicroStateService(iPS,reg)
-    ,encModule_(33)
+    ,encModule_(nReservedModules)
     ,nStreams_(0)//until initialized
     ,sleepTime_(iPS.getUntrackedParameter<int>("sleepTime", 1))
     ,fastMonIntervals_(iPS.getUntrackedParameter<unsigned int>("fastMonIntervals", 1))
@@ -81,26 +85,34 @@ namespace evf{
   }
 
 
-  std::string FastMonitoringService::makePathLegenda(){
-    //taken from first stream
-    std::ostringstream ost;
-    for(int i = 0;
-	i < encPath_[0].current_;
-	i++)
-      ost<<i<<"="<<*((std::string *)(encPath_[0].decode(i)))<<" ";
-    return ost.str();
+  std::string FastMonitoringService::makePathLegendaJson() {
+    Json::Value legendaVector(Json::arrayValue);
+    for(int i = 0; i < encPath_[0].current_; i++)
+      legendaVector.append(Json::Value(*((const std::string *)(encPath_[0].decode(i)))));
+    Json::Value valReserved(nReservedPaths);
+    Json::Value pathLegend;
+    pathLegend["names"]=legendaVector;
+    pathLegend["reserved"]=valReserved;
+    Json::StyledWriter writer;
+    return writer.write(pathLegend);
   }
 
-
-  std::string FastMonitoringService::makeModuleLegenda()
-  {  
-    std::ostringstream ost;
+  std::string FastMonitoringService::makeModuleLegendaJson(){
+    Json::Value legendaVector(Json::arrayValue);
     for(int i = 0; i < encModule_.current_; i++)
-      ost<<i<<"="<<((const edm::ModuleDescription *)(encModule_.decode(i)))->moduleLabel()<<" ";
-    return ost.str();
+       legendaVector.append(Json::Value(((const edm::ModuleDescription *)(encModule_.decode(i)))->moduleLabel()));
+    Json::Value valReserved(nReservedModules);
+    Json::Value valSpecial(nSpecialModules);
+    Json::Value valOutputModules(nOutputModules_);
+    Json::Value moduleLegend;
+    moduleLegend["names"]=legendaVector;
+    moduleLegend["reserved"]=valReserved;
+    moduleLegend["special"]=valSpecial;
+    moduleLegend["output"]=valOutputModules;
+    Json::StyledWriter writer;
+    return writer.write(moduleLegend);
   }
 
-
   void FastMonitoringService::preallocate(edm::service::SystemBounds const & bounds)
   {
 
@@ -133,12 +145,18 @@ namespace evf{
     fastPath_ = fast.string();
 
     std::ostringstream moduleLegFile;
+    std::ostringstream moduleLegFileJson;
     moduleLegFile << "microstatelegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".leg";
+    moduleLegFileJson << "microstatelegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".jsn";
     moduleLegendFile_  = (workingDirectory_/moduleLegFile.str()).string();
+    moduleLegendFileJson_  = (workingDirectory_/moduleLegFileJson.str()).string();
 
     std::ostringstream pathLegFile;
+    std::ostringstream pathLegFileJson;
     pathLegFile << "pathlegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".leg";
     pathLegendFile_  = (workingDirectory_/pathLegFile.str()).string();
+    pathLegFileJson << "pathlegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".jsn";
+    pathLegendFileJson_  = (workingDirectory_/pathLegFileJson.str()).string();
 
     LogDebug("FastMonitoringService") << "Initializing FastMonitor with microstate def path -: "
 			                  << microstateDefPath_;
@@ -267,16 +285,18 @@ namespace evf{
     //build a map of modules keyed by their module description address
     //here we need to treat output modules in a special way so they can be easily singled out
     if(desc.moduleName() == "Stream" || desc.moduleName() == "ShmStreamConsumer" || desc.moduleName() == "EvFOutputModule" ||
-       desc.moduleName() == "EventStreamFileWriter" || desc.moduleName() == "PoolOutputModule")
+       desc.moduleName() == "EventStreamFileWriter" || desc.moduleName() == "PoolOutputModule") {
       encModule_.updateReserved((void*)&desc);
+      nOutputModules_++;
+    }
     else
       encModule_.update((void*)&desc);
   }
 
   void FastMonitoringService::postBeginJob()
   {
-    std::string && moduleLegStr = makeModuleLegenda();
-    FileIO::writeStringToFile(moduleLegendFile_, moduleLegStr);
+    std::string && moduleLegStrJson = makeModuleLegendaJson();
+    FileIO::writeStringToFile(moduleLegendFileJson_, moduleLegStrJson);
 
     macrostate_ = FastMonitoringThread::sJobReady;
 
@@ -465,8 +485,8 @@ namespace evf{
 	collectedPathList_[sc.streamID()]->store(true,std::memory_order_seq_cst);
 	fmt_.m_data.ministateBins_=encPath_[sc.streamID()].vecsize();
 	if (!pathLegendWritten_) {
-	  std::string pathLegendStr =  makePathLegenda();
-	  FileIO::writeStringToFile(pathLegendFile_, pathLegendStr);
+	  std::string pathLegendStrJson =  makePathLegendaJson();
+	  FileIO::writeStringToFile(pathLegendFileJson_, pathLegendStrJson);
 	  pathLegendWritten_=true;
 	}
       }

HLTrigger/JSONMonitoring/plugins/TriggerJSONMonitoring.cc

@@ -650,6 +650,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
     IntJ daqJsnAccepted    = daqJsnProcessed;
     IntJ daqJsnErrorEvents = 0;                  
     IntJ daqJsnRetCodeMask = 0;                 
+    IntJ daqJsnHLTErrorEvents = 0;                  
 
     //write out HLT metadata jsn
     Json::Value hltDaqJsn;
@@ -665,6 +666,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
     hltDaqJsn[DataPoint::DATA].append(hltJsnInputFiles.value());
     hltDaqJsn[DataPoint::DATA].append(hltJsnFileAdler32);
     hltDaqJsn[DataPoint::DATA].append(iSummary->streamHLTDestination);
+    hltDaqJsn[DataPoint::DATA].append((unsigned int)daqJsnHLTErrorEvents.value());
 
     result = writer.write(hltDaqJsn);
 
@@ -690,6 +692,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
     l1DaqJsn[DataPoint::DATA].append(l1JsnInputFiles.value());
     l1DaqJsn[DataPoint::DATA].append(l1JsnFileAdler32);
     l1DaqJsn[DataPoint::DATA].append(iSummary->streamL1Destination);
+    l1DaqJsn[DataPoint::DATA].append((unsigned int)daqJsnHLTErrorEvents.value());
 
     result = writer.write(l1DaqJsn);