Tomographer  v5.4
Tomographer C++ Framework Documentation
pymultiproc.h
1 /* This file is part of the Tomographer project, which is distributed under the
2  * terms of the MIT license.
3  *
4  * The MIT License (MIT)
5  *
6  * Copyright (c) 2016 ETH Zurich, Institute for Theoretical Physics, Philippe Faist
7  * Copyright (c) 2017 Caltech, Institute for Quantum Information and Matter, Philippe Faist
8  *
9  * Permission is hereby granted, free of charge, to any person obtaining a copy
10  * of this software and associated documentation files (the "Software"), to deal
11  * in the Software without restriction, including without limitation the rights
12  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13  * copies of the Software, and to permit persons to whom the Software is
14  * furnished to do so, subject to the following conditions:
15  *
16  * The above copyright notice and this permission notice shall be included in
17  * all copies or substantial portions of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25  * SOFTWARE.
26  */
27 
28 #ifndef PYMULTIPROC_H
29 #define PYMULTIPROC_H
30 
31 #include <string>
32 
33 #include <tomographerpy/common.h>
34 
35 #include <tomographer/mhrw.h>
36 #include <tomographer/mhrwtasks.h>
37 
38 #include <tomographerpy/pymhrw.h>
39 #include <tomographerpy/exc.h>
40 
41 
42 namespace tpy {
43 
44 
53 struct TOMOGRAPHER_EXPORT WorkerStatusReport
54 {
56  int worker_id;
57 
59  py::float_ fraction_done;
60 
62  py::str msg;
63 
65  py::dict data;
66 };
67 
74 struct TOMOGRAPHER_EXPORT FullStatusReport {
76  : num_completed(-1), num_total_runs(-1), elapsed(0.0),
77  workers(), total_fraction_done(), human_report()
78  {
79  }
80 
82  py::int_ num_completed;
84  py::int_ num_total_runs;
86  py::float_ elapsed;
87 
91  py::list workers; // list of [WorkerStatusReport or None (for idle)]
92 
94  py::float_ total_fraction_done;
95 
97  py::str human_report;
98 };
99 
100 
110 template<typename TaskType>
111 struct TOMOGRAPHER_EXPORT PyStatusReportAddWorkerDataFields
112 {
116  static inline void addDataFields(py::dict & , const typename TaskType::StatusReportType & ) { }
117 };
118 
126 template<typename CData, typename Rng>
127 struct TOMOGRAPHER_EXPORT PyStatusReportAddWorkerDataFields< Tomographer::MHRWTasks::MHRandomWalkTask<CData, Rng> >
128 {
130  static inline void addDataFields(py::dict & d, const TaskStatusReportType & wr) {
131  d["mhrw_params"] = tpy::MHRWParams(
132  pyMHWalkerParamsToDictInvoke(wr.mhrw_params.mhwalker_params),
133  wr.mhrw_params.n_sweep,
134  wr.mhrw_params.n_therm,
135  wr.mhrw_params.n_run);
136  d["acceptance_ratio"] = wr.acceptance_ratio;
137  d["kstep"] = wr.kstep;
138  d["n_total_iters"] = wr.n_total_iters;
139  }
140 };
141 
150 template<typename TaskType, typename IntType = int>
153  )
154 {
156  r.num_completed = report.num_completed;
157  r.num_total_runs = report.num_total_runs;
158  r.elapsed = report.elapsed;
160  r.human_report = report.getHumanReport();
161  r.workers = py::list();
162  for (std::size_t k = 0; k < report.workers_reports.size(); ++k) {
163  if (!report.workers_running[k]) {
164  r.workers.append(py::none());
165  continue;
166  }
167  // and prepare the report object
168  const auto& rr = report.workers_reports[k];
169  tpy::WorkerStatusReport wreport;
170  // generic worker status report fields
171  wreport.worker_id = (int)k;
172  wreport.fraction_done = rr.fraction_done;
173  wreport.msg = rr.msg;
174  // fields specific to MHRWValueHistogramTasks
176  // add this report
177  r.workers.append(wreport);
178  }
179  return r;
180 }
181 
182 
201 template<typename TaskDispatcher>
202 inline void setTasksStatusReportPyCallback(TaskDispatcher & tasks, py::object progress_fn,
203  int progress_interval_ms, bool require_gil_acquisition = false)
204 {
205  typedef typename TaskDispatcher::TaskType TaskType;
206 
207  //
208  // NOTE: always set a progress report handler, even if progress_fn is None. Indeed, we
209  // use this callback to e.g. check for keyboard interrupt signals.
210  //
211 
212  auto fn = [progress_fn](const typename TaskDispatcher::FullStatusReportType & report) {
213 
214  if (PyErr_Occurred() != NULL || PyErr_CheckSignals() == -1) {
215  //fprintf(stderr, "DEBUG:: error set, throwing\n") ;
216  throw tpy::PyFetchedException();
217  }
218  // call the python progress callback:
219  if (!progress_fn.is_none()) {
220  auto r = preparePyTaskStatusReport<TaskType>(report);
221  //fprintf(stderr, "DEBUG:: about to call py callback\n") ;
222  progress_fn(py::cast(r));
223  if (PyErr_Occurred() != NULL || PyErr_CheckSignals() == -1) {
224  fprintf(stderr, "DEBUG:: error set, throwing\n") ;
225  throw tpy::PyFetchedException();
226  }
227  //fprintf(stderr, "DEBUG:: py callback done\n") ;
228  }
229  };
230 
231  auto fn_with_gil = [fn](const typename TaskDispatcher::FullStatusReportType & report) {
232  py::gil_scoped_acquire gil_acquire;
233  fn(report);
234  };
235 
236  if (require_gil_acquisition) {
237  tasks.setStatusReportHandler(fn_with_gil);
238  } else {
239  tasks.setStatusReportHandler(fn);
240  }
241  tasks.requestPeriodicStatusReport(progress_interval_ms);
242 }
243 
244 
245 } // namespace tpy
246 
247 
248 
249 
250 
251 
252 #endif
IterCountIntType kstep
the current iteration number
Definition: mhrw.h:1159
double acceptance_ratio
the current acceptance ratio of the random walk (see Tomographer::MHRandomWalk::acceptanceRatio() ) ...
Definition: mhrw.h:1174
py::float_ fraction_done
Fraction of the job done for this worker (0.0 to 1.0)
Definition: pymultiproc.h:59
Base namespace for the Tomographer project.
Definition: densellh.h:45
py::int_ num_total_runs
Total number of tasks which have been or will be run.
Definition: pymultiproc.h:84
py::dict data
Additional task-specific data; see also PyStatusReportAddWorkerDataFields.
Definition: pymultiproc.h:65
int worker_id
Unique identifier of the worker.
Definition: pymultiproc.h:56
std::vector< TaskStatusReportType, typename Tools::NeedOwnOperatorNew< TaskStatusReportType >::AllocatorType > workers_reports
List with the raw report submitted from each individual thread.
Definition: multiproc.h:134
std::vector< bool > workers_running
List specifying for each worker (e.g. a spawned thread) whether it is active or not.
Definition: multiproc.h:122
TaskCountIntType num_completed
Number of completed tasks.
Definition: multiproc.h:109
std::string getHumanReport() const
Produce a text-based human-readable short representation of the status report.
Definition: multiproc.h:166
py::float_ elapsed
Elapsed time in seconds since the launching of the tasks.
Definition: pymultiproc.h:86
Multiprocessing tasks interface (see Multiprocessing Task Interfaces) for parallel Metropolis-Hasting...
Complete status report for multiple tasks running in parallel.
Definition: pymultiproc.h:74
IterCountIntType n_total_iters
the total number of iterations required for this random walk
Definition: mhrw.h:1182
C++ utility to populate the data field of a WorkerStatusReport for a given TaskType.
Definition: pymultiproc.h:111
py::dict pyMHWalkerParamsToDictInvoke(const MHWalkerParams &p)
Helper for converting any MHWalkerParams into a dictionary, using automatic template parameter deduct...
Definition: pymhrw.h:95
double totalFractionDone() const
The total fraction of the job completed.
Definition: multiproc.h:149
Tomographer::MHRWParams< py::object, IterCountIntType > MHRWParams
The Tomographer::MHRWParams type exposed to Python (the MHWalkerParam can be represented by any Pytho...
Definition: pymhrw.h:44
Helper for catching exceptions in a thread and re-raising them.
Definition: exc.h:115
py::int_ num_completed
Number of tasks which have already completed.
Definition: pymultiproc.h:82
tpy::FullStatusReport preparePyTaskStatusReport(const Tomographer::MultiProc::FullStatusReport< typename TaskType::StatusReportType, IntType > &report)
Utility to prepare a Python status report (FullStatusReport) from a task&#39;s status report...
Definition: pymultiproc.h:151
static void addDataFields(py::dict &, const typename TaskType::StatusReportType &)
Add fields to the given dict, from a status report sent in by a TaskType. The default implementation ...
Definition: pymultiproc.h:116
py::float_ total_fraction_done
Total fraction of work done, as a fraction between 0 and 1.
Definition: pymultiproc.h:94
Status Report structure representing the status of a MHRandomWalk.
Definition: mhrw.h:1142
TaskCountIntType num_total_runs
Total number of tasks to perform.
Definition: multiproc.h:112
py::str human_report
Complete, human-readable summary of the current status of everything.
Definition: pymultiproc.h:97
void setTasksStatusReportPyCallback(TaskDispatcher &tasks, py::object progress_fn, int progress_interval_ms, bool require_gil_acquisition=false)
Set up status reporting for a task dispatcher, using a Python callback for status reports...
Definition: pymultiproc.h:202
py::list workers
A Python list of worker status; either a WorkerStatusReport, or py::none() if the worker is idle...
Definition: pymultiproc.h:91
Routines for performing a Metropolis-Hastings random walk.
Report of the status of a single worker.
Definition: pymultiproc.h:53
C++ Classes and Utilities for Python Modules.
Definition: common.h:89
MHRWParamsType mhrw_params
the parameters of the random walk
Definition: mhrw.h:1169
double elapsed
Number of seconds elapsed since launching the tasks.
Definition: multiproc.h:139
py::str msg
Human-readable message summarizing the status of this worker.
Definition: pymultiproc.h:62
A complete status report, abstract version.
Definition: multiproc.h:97