My Project
Loading...
Searching...
No Matches
FlowMain.hpp
1/*
2 Copyright 2013, 2014, 2015 SINTEF ICT, Applied Mathematics.
3 Copyright 2014 Dr. Blatt - HPC-Simulation-Software & Services
4 Copyright 2015 IRIS AS
5 Copyright 2014 STATOIL ASA.
6
7 This file is part of the Open Porous Media project (OPM).
8
9 OPM is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation, either version 3 of the License, or
12 (at your option) any later version.
13
14 OPM is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with OPM. If not, see <http://www.gnu.org/licenses/>.
21*/
22#ifndef OPM_FLOW_MAIN_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_HEADER_INCLUDED
24
25#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
26#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
27#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
28
30
31#include <opm/simulators/flow/Banners.hpp>
32#include <opm/simulators/flow/FlowUtils.hpp>
33#include <opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp>
34
35#if HAVE_MPI
36#define RESERVOIR_COUPLING_ENABLED
37#endif
38#if HAVE_DUNE_FEM
39#include <dune/fem/misc/mpimanager.hh>
40#else
41#include <dune/common/parallel/mpihelper.hh>
42#endif
43
44#ifdef _OPENMP
45#include <omp.h>
46#endif
47
48#include <charconv>
49#include <cstddef>
50#include <memory>
51
52namespace Opm::Parameters {
53
54// Do not merge parallel output files or warn about them
55struct EnableLoggingFalloutWarning { static constexpr bool value = false; };
56struct OutputInterval { static constexpr int value = 1; };
57
58} // namespace Opm::Parameters
59
60namespace Opm {
61
62 class Deck;
63
64 // The FlowMain class is the black-oil simulator.
65 template <class TypeTag>
67 {
68 public:
76
78
79 FlowMain(int argc, char **argv, bool output_cout, bool output_files )
80 : argc_{argc}, argv_{argv},
81 output_cout_{output_cout}, output_files_{output_files}
82 {
83
84 }
85
86 // Read the command line parameters. Throws an exception if something goes wrong.
87 static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
88 {
89 if (!Parameters::IsRegistrationOpen()) {
90 // We have already successfully run setupParameters_().
91 // For the dynamically chosen runs (as from the main flow
92 // executable) we must run this function again with the
93 // real typetag to be used, as the first time was with the
94 // "FlowEarlyBird" typetag. However, for the static ones (such
95 // as 'flow_onephase_energy') it has already been run with the
96 // correct typetag.
97 return EXIT_SUCCESS;
98 }
99 // register the flow specific parameters
100 Parameters::Register<Parameters::OutputInterval>
101 ("Specify the number of report steps between two consecutive writes of restart data");
102 Parameters::Register<Parameters::EnableLoggingFalloutWarning>
103 ("Developer option to see whether logging was on non-root processors. "
104 "In that case it will be appended to the *.DBG or *.PRT files");
105
106 // register the base parameters
107 registerAllParameters_<TypeTag>(/*finalizeRegistration=*/false);
108
109 Simulator::registerParameters();
110
111 detail::hideUnusedParameters<Scalar>();
112
113 Parameters::endRegistration();
114
115 int mpiRank = comm.rank();
116
117 // read in the command line parameters
119 const_cast<const char**>(argv),
120 /*doRegistration=*/false,
121 /*allowUnused=*/true,
122 /*handleHelp=*/(mpiRank==0));
123 if (status == 0) {
124
125 // deal with unknown parameters.
126
127 int unknownKeyWords = 0;
128 if (mpiRank == 0) {
129 unknownKeyWords = Parameters::printUnused(std::cerr);
130 }
133 if ( unknownKeyWords )
134 {
135 if ( mpiRank == 0 )
136 {
137 std::string msg = "Aborting simulation due to unknown "
138 "parameters. Please query \"flow --help\" for "
139 "supported command line parameters.";
140 if (OpmLog::hasBackend("STREAMLOG"))
141 {
142 OpmLog::error(msg);
143 }
144 else {
145 std::cerr << msg << std::endl;
146 }
147 }
148 return EXIT_FAILURE;
149 }
150
151 // deal with --print-parameters and unknown parameters.
152 if (Parameters::Get<Parameters::PrintParameters>() == 1) {
153 if (mpiRank == 0) {
154 Parameters::printValues(std::cout);
155 }
156 return -1;
157 }
158 }
159
160 // set the maximum limit on OMP threads
161 setMaxThreads();
162
163 return status;
164 }
165
170 {
171 return execute_(&FlowMain::runSimulator, /*cleanup=*/true);
172 }
173
174 int executeInitStep()
175 {
176 return execute_(&FlowMain::runSimulatorInit, /*cleanup=*/false);
177 }
178
179 // Returns true unless "EXIT" was encountered in the schedule
180 // section of the input datafile.
181 int executeStep()
182 {
183 return simulator_->runStep(*simtimer_);
184 }
185
186 // Called from Python to cleanup after having executed the last
187 // executeStep()
188 int executeStepsCleanup()
189 {
190 SimulatorReport report = simulator_->finalize();
191 runSimulatorAfterSim_(report);
192 return report.success.exit_status;
193 }
194
195 ModelSimulator* getSimulatorPtr()
196 {
197 return modelSimulator_.get();
198 }
199
200 SimulatorTimer* getSimTimer()
201 {
202 return simtimer_.get();
203 }
204
207 {
208 return simtimer_->stepLengthTaken();
209 }
210
211 private:
212 // called by execute() or executeInitStep()
213 int execute_(int (FlowMain::* runOrInitFunc)(), bool cleanup)
214 {
215 auto logger = [this](const std::exception& e, const std::string& message_start) {
216 std::ostringstream message;
217 message << message_start << e.what();
218
219 if (this->output_cout_) {
220 // in some cases exceptions are thrown before the logging system is set
221 // up.
222 if (OpmLog::hasBackend("STREAMLOG")) {
223 OpmLog::error(message.str());
224 }
225 else {
226 std::cout << message.str() << "\n";
227 }
228 }
229 detail::checkAllMPIProcesses();
230 return EXIT_FAILURE;
231 };
232
233 try {
234 // deal with some administrative boilerplate
235
236 Dune::Timer setupTimerAfterReadingDeck;
238
239 int status = setupParameters_(this->argc_, this->argv_, FlowGenericVanguard::comm());
240 if (status) {
241 return status;
242 }
243
244 setupParallelism();
245 setupModelSimulator();
247
248 this->deck_read_time_ = modelSimulator_->vanguard().setupTime();
249 this->total_setup_time_ = setupTimerAfterReadingDeck.elapsed() + this->deck_read_time_;
250
251 // if run, do the actual work, else just initialize
252 int exitCode = (this->*runOrInitFunc)();
253 if (cleanup) {
254 executeCleanup_();
255 }
256 return exitCode;
257 }
258 catch (const TimeSteppingBreakdown& e) {
259 auto exitCode = logger(e, "Simulation aborted: ");
260 executeCleanup_();
261 return exitCode;
262 }
263 catch (const std::exception& e) {
264 auto exitCode = logger(e, "Simulation aborted as program threw an unexpected exception: ");
265 executeCleanup_();
266 return exitCode;
267 }
268 }
269
270 void executeCleanup_() {
271 // clean up
272 mergeParallelLogFiles();
273 }
274
275 protected:
276 void setupParallelism()
277 {
278 // determine the rank of the current process and the number of processes
279 // involved in the simulation. MPI must have already been initialized
280 // here. (yes, the name of this method is misleading.)
281 auto comm = FlowGenericVanguard::comm();
282 mpi_rank_ = comm.rank();
283 mpi_size_ = comm.size();
284
285 setMaxThreads();
286 }
287
288 static void setMaxThreads()
289 {
290#if _OPENMP
291 // If openMP is available, default to 2 threads per process unless
292 // OMP_NUM_THREADS is set or command line --threads-per-process used.
293 // Issue a warning if both OMP_NUM_THREADS and --threads-per-process are set,
294 // but let the environment variable take precedence.
295 constexpr int default_threads = 2;
296 const bool isSet = Parameters::IsSet<Parameters::ThreadsPerProcess>();
297 const int requested_threads = Parameters::Get<Parameters::ThreadsPerProcess>();
299
300 const char* env_var = getenv("OMP_NUM_THREADS");
301 if (env_var) {
302 int omp_num_threads = -1;
303 auto result = std::from_chars(env_var, env_var + std::strlen(env_var), omp_num_threads);
304 if (result.ec == std::errc() && omp_num_threads > 0) {
305 // Set threads to omp_num_threads if it was successfully parsed and is positive
307 if (isSet) {
308 OpmLog::warning("Environment variable OMP_NUM_THREADS takes precedence over the --threads-per-process cmdline argument.");
309 }
310 } else {
311 OpmLog::warning("Invalid value for OMP_NUM_THREADS environment variable.");
312 }
313 }
314
315 // Requesting -1 thread will let OMP automatically deduce the number
316 // but setting OMP_NUM_THREADS takes precedence.
317 if (env_var || !(isSet && requested_threads == -1)) {
319 }
320#endif
321
323 TM::init(false);
324 }
325
326 void mergeParallelLogFiles()
327 {
328 // force closing of all log files.
329 OpmLog::removeAllBackends();
330
331 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_ || !modelSimulator_) {
332 return;
333 }
334
335 detail::mergeParallelLogFiles(eclState().getIOConfig().getOutputDir(),
336 Parameters::Get<Parameters::EclDeckFileName>(),
337 Parameters::Get<Parameters::EnableLoggingFalloutWarning>());
338 }
339
340 void setupModelSimulator()
341 {
342 modelSimulator_ = std::make_unique<ModelSimulator>(FlowGenericVanguard::comm(), /*verbose=*/false);
343 modelSimulator_->executionTimer().start();
344 modelSimulator_->model().applyInitialSolution();
345 }
346
347 const EclipseState& eclState() const
348 { return modelSimulator_->vanguard().eclState(); }
349
350 EclipseState& eclState()
351 { return modelSimulator_->vanguard().eclState(); }
352
353 const Schedule& schedule() const
354 { return modelSimulator_->vanguard().schedule(); }
355
356 // Run the simulator.
357 int runSimulator()
358 {
359 return runSimulatorInitOrRun_(&FlowMain::runSimulatorRunCallback_);
360 }
361
362 int runSimulatorInit()
363 {
364 return runSimulatorInitOrRun_(&FlowMain::runSimulatorInitCallback_);
365 }
366
367 private:
368 // Callback that will be called from runSimulatorInitOrRun_().
369 int runSimulatorRunCallback_()
370 {
371#ifdef RESERVOIR_COUPLING_ENABLED
372 SimulatorReport report = simulator_->run(*simtimer_, this->argc_, this->argv_);
373#else
374 SimulatorReport report = simulator_->run(*simtimer_);
375#endif
376 runSimulatorAfterSim_(report);
377 return report.success.exit_status;
378 }
379
380 // Callback that will be called from runSimulatorInitOrRun_().
381 int runSimulatorInitCallback_()
382 {
383#ifdef RESERVOIR_COUPLING_ENABLED
384 simulator_->init(*simtimer_, this->argc_, this->argv_);
385#else
386 simulator_->init(*simtimer_);
387#endif
388 return EXIT_SUCCESS;
389 }
390
391 // Output summary after simulation has completed
392 void runSimulatorAfterSim_(SimulatorReport &report)
393 {
394 if (! this->output_cout_) {
395 return;
396 }
397
398 const int threads
399#if !defined(_OPENMP) || !_OPENMP
400 = 1;
401#else
403#endif
404
405 printFlowTrailer(mpi_size_, threads, total_setup_time_, deck_read_time_, report, simulator_->model().localAccumulatedReports());
406
407 detail::handleExtraConvergenceOutput(report,
408 Parameters::Get<Parameters::OutputExtraConvergenceInfo>(),
409 R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
410 eclState().getIOConfig().getOutputDir(),
411 eclState().getIOConfig().getBaseName());
412 }
413
414 // Run the simulator.
415 int runSimulatorInitOrRun_(int (FlowMain::* initOrRunFunc)())
416 {
417
418 const auto& schedule = this->schedule();
419 auto& ioConfig = eclState().getIOConfig();
420 simtimer_ = std::make_unique<SimulatorTimer>();
421
422 // initialize variables
423 const auto& initConfig = eclState().getInitConfig();
424 simtimer_->init(schedule, static_cast<std::size_t>(initConfig.getRestartStep()));
425
426 if (this->output_cout_) {
427 std::ostringstream oss;
428
429 // This allows a user to catch typos and misunderstandings in the
430 // use of simulator parameters.
431 if (Parameters::printUnused(oss)) {
432 std::cout << "----------------- Unrecognized parameters: -----------------\n";
433 std::cout << oss.str();
434 std::cout << "----------------------------------------------------------------" << std::endl;
435 }
436 }
437
438 if (!ioConfig.initOnly()) {
439 if (this->output_cout_) {
440 std::string msg;
441 msg = "\n\n================ Starting main simulation loop ===============\n";
442 OpmLog::info(msg);
443 }
444
445 return (this->*initOrRunFunc)();
446 }
447 else {
448 if (this->output_cout_) {
449 std::cout << "\n\n================ Simulation turned off ===============\n" << std::flush;
450 }
451 return EXIT_SUCCESS;
452 }
453 }
454
455 protected:
456
458 // Create simulator instance.
459 // Writes to:
460 // simulator_
462 {
463 // Create the simulator instance.
464 simulator_ = std::make_unique<Simulator>(*modelSimulator_);
465 }
466
467 Grid& grid()
468 { return modelSimulator_->vanguard().grid(); }
469
470 private:
471 std::unique_ptr<ModelSimulator> modelSimulator_;
472 int mpi_rank_ = 0;
473 int mpi_size_ = 1;
474 std::any parallel_information_;
475 std::unique_ptr<Simulator> simulator_;
476 std::unique_ptr<SimulatorTimer> simtimer_;
477 int argc_;
478 char **argv_;
479 bool output_cout_;
480 bool output_files_;
481 double total_setup_time_ = 0.0;
482 double deck_read_time_ = 0.0;
483 };
484
485} // namespace Opm
486
487#endif // OPM_FLOW_MAIN_HEADER_INCLUDED
static Parallel::Communication & comm()
Obtain global communicator.
Definition FlowGenericVanguard.hpp:332
Definition FlowMain.hpp:67
double getPreviousReportStepSize()
Get the size of the previous report step.
Definition FlowMain.hpp:206
int execute()
This is the main function of Flow.
Definition FlowMain.hpp:169
void createSimulator()
This is the main function of Flow.
Definition FlowMain.hpp:461
a simulator for the blackoil model
Definition SimulatorFullyImplicitBlackoil.hpp:93
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition blackoilboundaryratevector.hh:37
constexpr auto getPropValue()
get the value data member of a property
Definition propertysystem.hh:242
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type::type GetPropType
get the type alias defined in the property (equivalent to old macro GET_PROP_TYPE(....
Definition propertysystem.hh:235
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type GetProp
get the type of a property (equivalent to old macro GET_PROP(...))
Definition propertysystem.hh:226
Provides convenience routines to bring up the simulation at runtime.
Definition FlowMain.hpp:56