otsdaq  v2_05_02_indev
DatabaseConfigurationInterface.cc
1 #include "otsdaq/ConfigurationInterface/DatabaseConfigurationInterface.h"
2 #include "otsdaq/Macros/CoutMacros.h"
3 #include "otsdaq/MessageFacility/MessageFacility.h"
4 
5 #include <algorithm>
6 #include <iostream>
7 #include <iterator>
8 #include <string>
9 
10 #include "artdaq-database/BasicTypes/basictypes.h"
11 #include "artdaq-database/ConfigurationDB/configurationdbifc.h"
12 #include "otsdaq/TableCore/TableBase.h"
13 
14 #include "artdaq-database/ConfigurationDB/configuration_common.h"
15 #include "artdaq-database/ConfigurationDB/dispatch_common.h"
16 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb.h"
17 #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h"
18 
19 using namespace ots;
20 
21 using artdaq::database::basictypes::FhiclData;
22 using artdaq::database::basictypes::JsonData;
23 
24 using ots::DatabaseConfigurationInterface;
25 using config_version_map_t = ots::DatabaseConfigurationInterface::config_version_map_t;
26 
27 namespace db = artdaq::database::configuration;
28 using VersionInfoList_t = db::ConfigurationInterface::VersionInfoList_t;
29 
30 constexpr auto default_dbprovider = "filesystem";
31 constexpr auto default_entity = "OTSROOT";
32 
33 //==============================================================================
34 DatabaseConfigurationInterface::DatabaseConfigurationInterface()
35 {
36 #ifdef DEBUG_ENABLE
37  // to enable debugging
38  if(0)
39  {
40  artdaq::database::configuration::debug::ExportImport();
41  artdaq::database::configuration::debug::ManageAliases();
42  artdaq::database::configuration::debug::ManageConfigs();
43  artdaq::database::configuration::debug::ManageDocuments();
44  artdaq::database::configuration::debug::Metadata();
45 
46  artdaq::database::configuration::debug::detail::ExportImport();
47  artdaq::database::configuration::debug::detail::ManageAliases();
48  artdaq::database::configuration::debug::detail::ManageConfigs();
49  artdaq::database::configuration::debug::detail::ManageDocuments();
50  artdaq::database::configuration::debug::detail::Metadata();
51 
52  artdaq::database::configuration::debug::options::OperationBase();
53  artdaq::database::configuration::debug::options::BulkOperations();
54  artdaq::database::configuration::debug::options::ManageDocuments();
55  artdaq::database::configuration::debug::options::ManageConfigs();
56  artdaq::database::configuration::debug::options::ManageAliases();
57 
58  artdaq::database::configuration::debug::MongoDB();
59  artdaq::database::configuration::debug::UconDB();
60  artdaq::database::configuration::debug::FileSystemDB();
61 
62  artdaq::database::filesystem::index::debug::enable();
63 
64  artdaq::database::filesystem::debug::enable();
65  artdaq::database::mongo::debug::enable();
66 
67  artdaq::database::docrecord::debug::JSONDocumentBuilder();
68  artdaq::database::docrecord::debug::JSONDocument();
69 
70  // debug::registerUngracefullExitHandlers();
71  // artdaq::database::useFakeTime(true);
72  }
73 #endif
74 }
75 
76 //==============================================================================
77 // read configuration from database
78 // version = -1 means latest version
79 void DatabaseConfigurationInterface::fill(TableBase* configuration, TableVersion version) const
80 
81 {
82  auto ifc = db::ConfigurationInterface{default_dbprovider};
83 
84  auto versionstring = version.toString();
85 
86  //__COUTV__(versionstring);
87  // configuration->getViewP()->setUniqueStorageIdentifier(storageUID);
88 
89  auto result = ifc.template loadVersion<decltype(configuration), JsonData>(configuration, versionstring, default_entity);
90 
91  if(result.first)
92  {
93  // make sure version is set.. not clear it was happening in loadVersion
94  configuration->getViewP()->setVersion(version);
95  return;
96  }
97  __SS__ << "\n\nDBI Error while filling '" << configuration->getTableName() << "' version '" << versionstring << "' - are you sure this version exists?\n"
98  << "Here is the error:\n\n"
99  << result.second << __E__;
100  __SS_ONLY_THROW__;
101 } // end fill()
102 
103 //==============================================================================
104 // write configuration to database
105 void DatabaseConfigurationInterface::saveActiveVersion(const TableBase* configuration, bool overwrite) const
106 
107 {
108  auto ifc = db::ConfigurationInterface{default_dbprovider};
109 
110  // configuration->getView().getUniqueStorageIdentifier()
111 
112  auto versionstring = configuration->getView().getVersion().toString();
113  //__COUT__ << "versionstring: " << versionstring << "\n";
114 
115  // auto result =
116  // ifc.template storeVersion<decltype(configuration), JsonData>(configuration,
117  // versionstring, default_entity);
118  auto result = overwrite ? ifc.template overwriteVersion<decltype(configuration), JsonData>(configuration, versionstring, default_entity)
119  : ifc.template storeVersion<decltype(configuration), JsonData>(configuration, versionstring, default_entity);
120 
121  if(result.first)
122  return;
123 
124  __SS__ << "DBI Error:" << result.second << __E__;
125  __SS_THROW__;
126 }
127 
128 //==============================================================================
129 // find the latest configuration version by configuration type
130 TableVersion DatabaseConfigurationInterface::findLatestVersion(const TableBase* table) const noexcept
131 {
132  auto versions = getVersions(table);
133 
134  __COUT__ << "Table Name: " << table->getTableName() << __E__;
135  __SS__ << "All Versions: ";
136  for(auto& v : versions)
137  ss << v << " ";
138  ss << __E__;
139  __COUT__ << ss.str();
140 
141  if(!versions.size())
142  return TableVersion(); // return INVALID
143 
144  return *(versions.rbegin());
145 }
146 
147 //==============================================================================
148 // find all configuration versions by configuration type
149 std::set<TableVersion> DatabaseConfigurationInterface::getVersions(const TableBase* table) const noexcept try
150 {
151  auto ifc = db::ConfigurationInterface{default_dbprovider};
152  auto result = ifc.template getVersions<decltype(table)>(table, default_entity);
153 
154  auto resultSet = std::set<TableVersion>{};
155  for(std::string const& version : result)
156  resultSet.insert(TableVersion(std::stol(version, 0, 10)));
157 
158  // auto to_set = [](auto const& inputList)
159  // {
160  // auto resultSet = std::set<TableVersion>{};
161  // std::for_each(inputList.begin(), inputList.end(),
162  // [&resultSet](std::string const& version)
163  // { resultSet.insert(std::stol(version, 0, 10)); });
164  // return resultSet;
165  // };
166 
167  // auto vs = to_set(result);
168  // for(auto &v:vs)
169  // __COUT__ << "\tversion " << v << __E__;
170 
171  return resultSet; // to_set(result);
172 }
173 catch(std::exception const& e)
174 {
175  __COUT__ << "DBI Exception:" << e.what() << "\n";
176  return {};
177 }
178 
179 //==============================================================================
180 // returns a list of all configuration names
181 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableNames() const try
182 {
183  auto ifc = db::ConfigurationInterface{default_dbprovider};
184 
185  auto collection_name_prefix = std::string{};
186 
187  return ifc.listCollections(collection_name_prefix);
188 }
189 catch(std::exception const& e)
190 {
191  __SS__ << "DBI Exception:" << e.what() << "\n";
192  __SS_THROW__;
193 }
194 catch(...)
195 {
196  __SS__ << "DBI Unknown exception.\n";
197  __SS_THROW__;
198 }
199 
200 //==============================================================================
201 // find all configuration groups in database
202 std::set<std::string /*name*/> DatabaseConfigurationInterface::getAllTableGroupNames(std::string const& filterString) const try
203 {
204  auto ifc = db::ConfigurationInterface{default_dbprovider};
205 
206  if(filterString == "")
207  return ifc.findGlobalConfigurations("*"); // GConfig will return all GConfig*
208  // with filesystem db.. for mongodb
209  // would require reg expr
210  else
211  return ifc.findGlobalConfigurations(filterString + "*"); // GConfig will return
212  // all GConfig* with
213  // filesystem db.. for
214  // mongodb would require
215  // reg expr
216 }
217 catch(std::exception const& e)
218 {
219  __SS__ << "Filter string '" << filterString << "' yielded DBI Exception:" << e.what() << "\n";
220  __SS_THROW__;
221 }
222 catch(...)
223 {
224  __SS__ << "Filter string '" << filterString << "' yielded DBI Unknown exception.\n";
225  __SS_THROW__;
226 }
227 
228 //==============================================================================
229 // find the latest configuration group key by group name
230 // if not found, return invalid
231 TableGroupKey DatabaseConfigurationInterface::findLatestGroupKey(const std::string& groupName) const noexcept
232 {
233  std::set<TableGroupKey> keys = DatabaseConfigurationInterface::getKeys(groupName);
234  if(keys.size()) // if keys exist, bump the last
235  return *(keys.crbegin());
236 
237  // else, return invalid
238  return TableGroupKey();
239 }
240 
241 //==============================================================================
242 // find all configuration groups in database
243 std::set<TableGroupKey /*key*/> DatabaseConfigurationInterface::getKeys(const std::string& groupName) const
244 {
245  std::set<TableGroupKey> retSet;
246  std::set<std::string /*name*/> names = getAllTableGroupNames();
247  for(auto& n : names)
248  if(n.find(groupName) == 0)
249  retSet.insert(TableGroupKey(n));
250  return retSet;
251 }
252 
253 //==============================================================================
254 // return the contents of a configuration group
255 config_version_map_t DatabaseConfigurationInterface::getTableGroupMembers(std::string const& tableGroup, bool includeMetaDataTable) const try
256 {
257  auto ifc = db::ConfigurationInterface{default_dbprovider};
258  auto result = ifc.loadGlobalConfiguration(tableGroup);
259 
260  // for(auto &item:result)
261  // __COUT__ << "====================>" << item.configuration << ": " <<
262  // item.version << __E__;
263 
264  auto to_map = [](auto const& inputList, bool includeMetaDataTable) {
265  auto resultMap = config_version_map_t{};
266 
267  std::for_each(inputList.begin(), inputList.end(), [&resultMap](auto const& info) { resultMap[info.configuration] = std::stol(info.version, 0, 10); });
268 
269  if(!includeMetaDataTable)
270  {
271  // remove special meta data table from member map
272  auto metaTable = resultMap.find(GROUP_METADATA_TABLE_NAME);
273  if(metaTable != resultMap.end())
274  resultMap.erase(metaTable);
275  }
276  return resultMap;
277  };
278 
279  return to_map(result, includeMetaDataTable);
280 } // end getTableGroupMembers()
281 catch(std::exception const& e)
282 {
283  __SS__ << "DBI Exception getting Group's member tables for '" << tableGroup << "':\n\n" << e.what() << "\n";
284  __COUT_ERR__ << ss.str();
285  __SS_THROW__;
286 }
287 catch(...)
288 {
289  __SS__ << "DBI Unknown exception getting Group's member tables for '" << tableGroup << ".'\n";
290  __COUT_ERR__ << ss.str();
291  __SS_THROW__;
292 }
293 
294 //==============================================================================
295 // create a new configuration group from the contents map
296 void DatabaseConfigurationInterface::saveTableGroup(config_version_map_t const& configurationMap, std::string const& configurationGroup) const try
297 {
298  auto ifc = db::ConfigurationInterface{default_dbprovider};
299 
300  auto to_list = [](auto const& inputMap) {
301  auto resultList = VersionInfoList_t{};
302  std::transform(inputMap.begin(), inputMap.end(), std::back_inserter(resultList), [](auto const& mapEntry) {
303  return VersionInfoList_t::value_type{mapEntry.first, mapEntry.second.toString(), default_entity};
304  });
305 
306  return resultList;
307  };
308 
309  auto result = ifc.storeGlobalConfiguration(to_list(configurationMap), configurationGroup);
310 
311  if(result.first)
312  return;
313 
314  __THROW__(result.second);
315 } // end saveTableGroup()
316 catch(std::exception const& e)
317 {
318  __SS__ << "DBI Exception:" << e.what() << "\n";
319  __COUT_ERR__ << ss.str();
320  __SS_THROW__;
321 }
322 catch(...)
323 {
324  __SS__ << "DBI Unknown exception.\n";
325  __COUT_ERR__ << ss.str();
326  __SS_THROW__;
327 }