MODFLOW 6 version 6.9.0.dev0
USGS Modular Hydrologic Model
Loading...
Searching...
No Matches
DistributedSim.f90
Go to the documentation of this file.
2 use kindmodule, only: i4b, lgp
14
15 implicit none
16 private
17
18 public :: distributedsimtype
19 public :: get_dsim
20
22 character(len=LENMEMPATH) :: memory_path
23 integer(I4B), pointer :: nr_models !< the total (global) number of models, equals the length of the model block in mfsim.nam
24 integer(I4B), dimension(:), pointer, contiguous :: load_mask => null() !< mask for loading models and exchanges, 1 when active on this processor, else 0
25 integer(I4B), dimension(:), pointer, contiguous :: model_ranks => null() !< load balance: model rank (0,...,nr_procs-1) per global model id (array index)
26 logical(LGP), pointer :: print_ptable !< when true, the partition table is printed to file
27 contains
28 procedure :: create
29 procedure :: get_load_mask
30 procedure :: get_load_balance
31 procedure :: destroy
32 ! private
33 procedure, private :: create_load_mask
34 procedure, private :: set_load_balance_from_input
35 procedure, private :: set_load_balance_default
36 procedure, private :: validate_load_balance
37 procedure, private :: print_load_balance
38 end type
39
40 ! singleton, private member
41 class(distributedsimtype), private, pointer :: dist_sim => null()
42
43contains
44
45 !> @brief Get pointer to the distributed simulation object
46 !<
47 function get_dsim() result(ds)
48 class(distributedsimtype), pointer :: ds
49
50 if (.not. associated(dist_sim)) then
51 allocate (dist_sim)
52 call dist_sim%create()
53 end if
54 ds => dist_sim
55
56 end function get_dsim
57
58 !> Create the distributed simulation object from the simulation input ctx
59 !<
60 subroutine create(this)
61 class(distributedsimtype) :: this
62 ! local
63 character(len=LENMEMPATH) :: input_mempath
64 integer(I4B), pointer :: nmod
65
66 this%memory_path = create_mem_path(component='SIM')
67
68 input_mempath = create_mem_path(component='SIM', context=idm_context)
69 call mem_setptr(nmod, 'NUMMODELS', input_mempath)
70
71 call mem_allocate(this%nr_models, 'NUMMODELS', this%memory_path)
72 this%nr_models = nmod
73
74 call mem_allocate(this%print_ptable, 'PRINT_PTABLE', this%memory_path)
75 this%print_ptable = .false.
76
77 end subroutine create
78
79 !> @brief Return pointer to the load mask for models
80 !!
81 !! Get a load mask to determine which models
82 !! should be loaded by idm on this process. This is in
83 !! sync with models create. The mask array is allocated
84 !! with its size equal to the global number of models.
85 !! It is returned as (1, 1, 0, 0, ... 0) with each entry
86 !! being a load mask for the model at the corresponding
87 !< location in the 'MNAME' array of the IDM.
88 function get_load_mask(this) result(load_mask)
89 class(distributedsimtype) :: this
90 integer(I4B), dimension(:), pointer :: load_mask
91
92 if (.not. associated(this%load_mask)) then
93 call this%create_load_mask()
94 end if
95 load_mask => this%load_mask
96
97 end function get_load_mask
98
99 !> @brief Create a load mask for IDM from the load balance array
100 !<
101 subroutine create_load_mask(this)
102 class(distributedsimtype) :: this
103 ! local
104 integer(I4B), dimension(:), pointer :: model_ranks => null() !< the load balance
105 integer(I4B) :: m_id !< model id
106
107 call mem_allocate(this%load_mask, this%nr_models, 'LOADMASK', &
108 this%memory_path)
109 this%load_mask = 0
110
111 ! get load balance (probably the first call, so creates it)
112 model_ranks => this%get_load_balance()
113
114 ! set mask from balance
115 do m_id = 1, this%nr_models
116 if (model_ranks(m_id) == proc_id) then
117 this%load_mask(m_id) = 1
118 else
119 this%load_mask(m_id) = 0
120 end if
121 end do
122
123 end subroutine create_load_mask
124
125 !> @brief Get the model load balance for the simulation
126 !<
127 function get_load_balance(this) result(mranks)
128 use simvariablesmodule, only: iout
130 class(distributedsimtype) :: this !< this distributed sim instance
131 integer(I4B), dimension(:), pointer :: mranks !< the load balance: array of ranks per model id
132 ! local
133 integer(I4B) :: isize
134 logical(LGP) :: hpc6_present, partitions_present
135 character(len=LENMEMPATH) :: simnam_mempath, hpc_mempath
136 type(utlhpcparamfoundtype) :: found
137
138 ! if load balance available, return here:
139 if (associated(this%model_ranks)) then
140 mranks => this%model_ranks
141 return
142 end if
143
144 call mem_allocate(this%model_ranks, this%nr_models, 'MODELRANKS', &
145 this%memory_path)
146
147 ! check for optional HPC file
148 simnam_mempath = create_mem_path('SIM', 'NAM', idm_context)
149 call get_isize('HPC6_FILENAME', simnam_mempath, isize)
150 hpc6_present = isize > 0
151
152 ! handle serial case
153 if (simulation_mode == 'SEQUENTIAL') then
154 if (hpc6_present) then
155 write (warnmsg, *) "Ignoring PARTITIONS block in HPC file when "// &
156 "running a serial process"
158 end if
159
160 ! single process, everything on cpu 0:
161 this%model_ranks = 0
162 mranks => this%model_ranks
163 return
164 end if
165
166 ! continue for PARALLEL mode only:
167 write (iout, '(/1x,a)') 'PROCESSING HPC DATA'
168
169 hpc_mempath = create_mem_path('UTL', 'HPC', idm_context)
170 ! source optional print input flag
171 call mem_set_value(this%print_ptable, 'PRINT_TABLE', hpc_mempath, &
172 found%print_table)
173 ! check if optional partition block exists
174 call get_isize('MNAME', hpc_mempath, isize)
175 partitions_present = isize > 0
176
177 ! fill model ranks (i.e. the load balance)
178 if (partitions_present) then
179 ! set balance from HPC file
180 call this%set_load_balance_from_input()
181 call this%validate_load_balance()
182 write (iout, '(1x,a)') 'Read partition data from HPC file'
183 else
184 ! no HPC file present, set balance with default algorithm
185 call this%set_load_balance_default()
186 write (iout, '(1x,a)') 'Generate default partition data'
187 end if
188
189 mranks => this%model_ranks
190
191 ! print to listing file
192 if (this%print_ptable) then
193 call this%print_load_balance()
194 end if
195
196 write (iout, '(1x,a)') 'END OF HPC DATA'
197
198 end function get_load_balance
199
200 !> @brief Load load balance from the input configuration
201 !<
203 class(distributedsimtype) :: this !< this distributed sim instance
204 ! local
205 character(len=LENMEMPATH) :: simnam_mempath, hpc_mempath
206 character(len=LENMODELNAME) :: model_name
207 type(characterstringtype), dimension(:), contiguous, pointer :: mnames !< model names (all) from the simulation nam file
208 type(characterstringtype), dimension(:), contiguous, pointer :: mnames_hpc !< model names in the hpc file
209 integer(I4B), dimension(:), contiguous, pointer :: mranks_hpc !< rank numbers in the hpc file
210 integer(I4B) :: i, model_idx
211 integer(I4B) :: target_rank
212 integer(I4B), dimension(:), allocatable :: rank_used
213 type(characterstringtype), dimension(:), contiguous, pointer :: hpc_names !< helper array to get the hpc filename
214 character(len=LINELENGTH) :: hpc_filename !< the HPC option file
215
216 ! set to uninitialized
217 this%model_ranks = -1
218
219 ! from IDM
220 simnam_mempath = create_mem_path('SIM', 'NAM', idm_context)
221 hpc_mempath = create_mem_path('UTL', 'HPC', idm_context)
222 call mem_setptr(mnames, 'MNAME', simnam_mempath)
223 call mem_setptr(mnames_hpc, 'MNAME', hpc_mempath)
224 call mem_setptr(mranks_hpc, 'MRANK', hpc_mempath)
225 call mem_setptr(hpc_names, 'HPC6_FILENAME', simnam_mempath)
226
227 ! FILEIN options give an array, so take the first:
228 hpc_filename = hpc_names(1)
229
230 ! check: valid model names
231 do i = 1, size(mnames_hpc)
232 if (ifind(mnames, mnames_hpc(i)) == -1) then
233 model_name = mnames_hpc(i)
234 write (errmsg, *) "HPC input error: undefined model name (", &
235 trim(model_name), ")"
236 call store_error(errmsg)
237 end if
238 end do
239 ! check: valid ranks
240 do i = 1, size(mranks_hpc)
241 target_rank = mranks_hpc(i)
242 if (target_rank < 0 .or. target_rank > nr_procs - 1) then
243 model_name = mnames_hpc(i)
244 write (errmsg, '(a,i0,2a)') "HPC input error: invalid target rank (", &
245 target_rank, ") for model ", trim(model_name)
246 call store_error(errmsg)
247 end if
248 end do
249 if (count_errors() > 0) then
250 call store_error_filename(hpc_filename)
251 end if
252
253 ! construct rank array
254 do i = 1, size(mnames_hpc)
255 model_idx = ifind(mnames, mnames_hpc(i))
256 this%model_ranks(model_idx) = mranks_hpc(i)
257 end do
258
259 ! check: all models acquired rank
260 do i = 1, size(this%model_ranks)
261 if (this%model_ranks(i) == -1) then
262 model_name = mnames(i)
263 write (errmsg, '(2a)') "HPC input error: no target rank for model ", &
264 trim(model_name)
265 call store_error(errmsg)
266 end if
267 end do
268 if (count_errors() > 0) then
269 call store_error_filename(hpc_filename)
270 end if
271
272 ! check: no idle ranks
273 allocate (rank_used(nr_procs))
274 rank_used = 0
275 do i = 1, size(this%model_ranks)
276 if (this%model_ranks(i) >= 0 .and. this%model_ranks(i) < nr_procs) then
277 rank_used(this%model_ranks(i) + 1) = 1
278 end if
279 end do
280 do i = 1, size(rank_used)
281 if (rank_used(i) == 0) then
282 write (errmsg, '(a,i0,a)') "HPC input error: rank ", i - 1, &
283 " has no models assigned"
284 call store_error(errmsg)
285 end if
286 end do
287 deallocate (rank_used)
288 if (count_errors() > 0) then
289 call store_error_filename(hpc_filename)
290 end if
291
292 end subroutine set_load_balance_from_input
293
294 !> @brief Distribute the models over the available
295 !! processes in a parallel run. Expects an array sized
296 !< to the number of models in the global simulation
298 class(distributedsimtype) :: this !< this distributed sim. instance
299 ! local
300 integer(I4B) :: im, imm, ie, ip, cnt
301 integer(I4B) :: nr_models, nr_gwf_models
302 integer(I4B) :: nr_exchanges
303 integer(I4B) :: min_per_proc, nr_left
304 integer(I4B) :: rank
305 integer(I4B), dimension(:), allocatable :: nr_models_proc
306 character(len=LENPACKAGETYPE) :: model_type_str
307 character(len=LENMEMPATH) :: input_mempath
308 type(characterstringtype), dimension(:), contiguous, &
309 pointer :: mtypes !< model types
310 type(characterstringtype), dimension(:), contiguous, &
311 pointer :: mnames !< model names
312 type(characterstringtype), dimension(:), contiguous, &
313 pointer :: etypes !< exg types
314 type(characterstringtype), dimension(:), contiguous, &
315 pointer :: emnames_a !< model a names
316 type(characterstringtype), dimension(:), contiguous, &
317 pointer :: emnames_b !< model b names
318
319 this%model_ranks = 0
320
321 ! load IDM data
322 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
323 call mem_setptr(mtypes, 'MTYPE', input_mempath)
324 call mem_setptr(mnames, 'MNAME', input_mempath)
325 call mem_setptr(etypes, 'EXGTYPE', input_mempath)
326 call mem_setptr(emnames_a, 'EXGMNAMEA', input_mempath)
327 call mem_setptr(emnames_b, 'EXGMNAMEB', input_mempath)
328
329 ! count flow models
330 nr_models = size(mnames)
331 nr_gwf_models = 0
332 do im = 1, nr_models
333 if (mtypes(im) == 'GWF6') then
334 nr_gwf_models = nr_gwf_models + 1
335 end if
336
337 if (mtypes(im) == 'GWF6' .or. &
338 mtypes(im) == 'GWT6' .or. &
339 mtypes(im) == 'GWE6') then
340 cycle
341 end if
342
343 model_type_str = mtypes(im)
344 write (errmsg, *) 'Model type ', model_type_str, &
345 ' not supported in parallel mode.'
346 call store_error(errmsg, terminate=.true.)
347 end do
348
349 ! calculate nr of flow models for each rank
350 allocate (nr_models_proc(nr_procs))
351 min_per_proc = nr_gwf_models / nr_procs
352 nr_left = nr_gwf_models - nr_procs * min_per_proc
353 cnt = 1
354 do ip = 1, nr_procs
355 rank = ip - 1
356 nr_models_proc(ip) = min_per_proc
357 if (rank < nr_left) then
358 nr_models_proc(ip) = nr_models_proc(ip) + 1
359 end if
360 end do
361
362 ! assign ranks for flow models
363 rank = 0
364 do im = 1, nr_models
365 if (mtypes(im) == 'GWF6') then
366 if (nr_models_proc(rank + 1) == 0) then
367 rank = rank + 1
368 end if
369 this%model_ranks(im) = rank
370 nr_models_proc(rank + 1) = nr_models_proc(rank + 1) - 1
371 end if
372 end do
373
374 ! match other models to flow
375 nr_exchanges = size(etypes)
376 do im = 1, nr_models
377 if (mtypes(im) == 'GWT6') then
378
379 ! find match
380 do ie = 1, nr_exchanges
381 if (etypes(ie) == 'GWF6-GWT6' .and. mnames(im) == emnames_b(ie)) then
382 rank = 0
383 do imm = 1, nr_models
384 if (mnames(imm) == emnames_a(ie)) then
385 rank = this%model_ranks(imm)
386 exit
387 end if
388 end do
389 this%model_ranks(im) = rank
390 exit
391 end if
392 end do
393
394 else if (mtypes(im) == 'GWE6') then
395 do ie = 1, nr_exchanges
396 if (etypes(ie) == 'GWF6-GWE6' .and. mnames(im) == emnames_b(ie)) then
397 rank = 0
398 do imm = 1, nr_models
399 if (mnames(imm) == emnames_a(ie)) then
400 rank = this%model_ranks(imm)
401 exit
402 end if
403 end do
404 this%model_ranks(im) = rank
405 exit
406 end if
407 end do
408
409 else
410 cycle ! e.g., for a flow model
411 end if
412 end do
413
414 ! cleanup
415 deallocate (nr_models_proc)
416
417 end subroutine set_load_balance_default
418
419 !> @brief Check validity of load balance configuration
420 !<
421 subroutine validate_load_balance(this)
422 class(distributedsimtype) :: this
423 ! local
424 character(len=LENMEMPATH) :: input_mempath
425 type(characterstringtype), dimension(:), contiguous, &
426 pointer :: mtypes !< model types
427 type(characterstringtype), dimension(:), contiguous, &
428 pointer :: mnames !< model names
429 type(characterstringtype), dimension(:), contiguous, &
430 pointer :: etypes !< exg types
431 type(characterstringtype), dimension(:), contiguous, &
432 pointer :: emnames_a !< model a names
433 type(characterstringtype), dimension(:), contiguous, &
434 pointer :: emnames_b !< model b names
435 type(characterstringtype), dimension(:), contiguous, &
436 pointer :: hpc_names !< helper array to get hpc filename
437 integer(I4B) :: ie
438 integer(I4B) :: idx_a, idx_b
439 integer(I4B) :: rank_a, rank_b
440 integer(I4B) :: nr_exchanges
441 character(len=LINELENGTH) :: hpc_filename !< the HPC option file
442 character(len=LENMODELNAME) :: name_a, name_b
443 character(len=LINELENGTH) :: exg_type
444
445 ! load IDM data
446 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
447 call mem_setptr(mtypes, 'MTYPE', input_mempath)
448 call mem_setptr(mnames, 'MNAME', input_mempath)
449 call mem_setptr(etypes, 'EXGTYPE', input_mempath)
450 call mem_setptr(emnames_a, 'EXGMNAMEA', input_mempath)
451 call mem_setptr(emnames_b, 'EXGMNAMEB', input_mempath)
452 call mem_setptr(hpc_names, 'HPC6_FILENAME', input_mempath)
453
454 ! FILEIN options give an array, so take the first:
455 hpc_filename = hpc_names(1)
456
457 nr_exchanges = size(etypes)
458
459 ! loop over exchanges
460 do ie = 1, nr_exchanges
461 if (etypes(ie) == 'GWF6-GWT6' .or. etypes(ie) == 'GWF6-GWE6') then
462 idx_a = ifind(mnames, emnames_a(ie))
463 idx_b = ifind(mnames, emnames_b(ie))
464 rank_a = this%model_ranks(idx_a)
465 rank_b = this%model_ranks(idx_b)
466 if (rank_a /= rank_b) then
467 name_a = emnames_a(ie)
468 name_b = emnames_b(ie)
469 exg_type = etypes(ie)
470 write (errmsg, '(7a)') "HPC input error: models ", &
471 trim(name_a), " and ", trim(name_b), " with a ", &
472 trim(exg_type), " coupling have to be assigned to the same rank"
473 call store_error(errmsg)
474 end if
475 end if
476 end do
477
478 if (count_errors() > 0) then
479 call store_error_filename(hpc_filename)
480 end if
481
482 end subroutine validate_load_balance
483
484 !> @brief Print the load balance table to the listing file
485 !<
486 subroutine print_load_balance(this)
487 use tablemodule, only: tabletype, table_cr
490 class(distributedsimtype) :: this
491 ! local
492 type(tabletype), pointer :: inputtab => null()
493 character(len=LINELENGTH) :: tag, term
494 character(len=LENMEMPATH) :: input_mempath
495 type(characterstringtype), dimension(:), contiguous, &
496 pointer :: mtypes !< model types
497 type(characterstringtype), dimension(:), contiguous, &
498 pointer :: mnames !< model names
499 integer(I4B) :: im, nr_models
500
501 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
502
503 call mem_setptr(mtypes, 'MTYPE', input_mempath)
504 call mem_setptr(mnames, 'MNAME', input_mempath)
505
506 ! setup table
507 nr_models = size(mnames)
508 call table_cr(inputtab, 'HPC', 'HPC PARTITION DATA')
509 call inputtab%table_df(nr_models, 5, iout)
510
511 ! add columns
512 tag = 'ID'
513 call inputtab%initialize_column(tag, 8, alignment=tableft)
514 tag = 'NAME'
515 call inputtab%initialize_column(tag, lenmodelname + 4, alignment=tableft)
516 tag = 'TYPE'
517 call inputtab%initialize_column(tag, 8, alignment=tableft)
518 tag = 'RANK'
519 call inputtab%initialize_column(tag, 8, alignment=tableft)
520 tag = 'LOCAL'
521 call inputtab%initialize_column(tag, 8, alignment=tableft)
522
523 do im = 1, nr_models
524 call inputtab%add_term(im)
525 term = mnames(im)
526 call inputtab%add_term(term)
527 term = mtypes(im)
528 call inputtab%add_term(term)
529 call inputtab%add_term(this%model_ranks(im))
530 term = ''
531 if (this%model_ranks(im) == proc_id) term = 'X'
532 call inputtab%add_term(term)
533 end do
534
535 ! deallocate
536 call inputtab%table_da()
537 deallocate (inputtab)
538
539 end subroutine print_load_balance
540
541 !> @brief clean up
542 !<
543 subroutine destroy(this)
544 class(distributedsimtype) :: this
545
546 if (associated(this%load_mask)) then
547 call mem_deallocate(this%load_mask)
548 call mem_deallocate(this%model_ranks)
549 end if
550
551 call mem_deallocate(this%nr_models)
552 call mem_deallocate(this%print_ptable)
553
554 ! delete singleton instance
555 if (associated(dist_sim)) deallocate (dist_sim)
556
557 end subroutine destroy
558
559end module distributedsimmodule
This module contains simulation constants.
Definition Constants.f90:9
integer(i4b), parameter linelength
maximum length of a standard line
Definition Constants.f90:45
@ tabcenter
centered table column
@ tableft
left justified table column
integer(i4b), parameter lenmodelname
maximum length of the model name
Definition Constants.f90:22
integer(i4b), parameter lenpackagetype
maximum length of a package type (DIS6, SFR6, CSUB6, etc.)
Definition Constants.f90:38
integer(i4b), parameter lenmempath
maximum length of the memory path
Definition Constants.f90:27
subroutine set_load_balance_from_input(this)
Load load balance from the input configuration.
subroutine validate_load_balance(this)
Check validity of load balance configuration.
subroutine create_load_mask(this)
Create a load mask for IDM from the load balance array.
class(distributedsimtype) function, pointer, public get_dsim()
Get pointer to the distributed simulation object.
integer(i4b) function, dimension(:), pointer get_load_mask(this)
Return pointer to the load mask for models.
subroutine print_load_balance(this)
Print the load balance table to the listing file.
subroutine create(this)
Create the distributed simulation object from the simulation input ctx.
class(distributedsimtype), pointer, private dist_sim
subroutine set_load_balance_default(this)
Distribute the models over the available processes in a parallel run. Expects an array sized.
integer(i4b) function, dimension(:), pointer get_load_balance(this)
Get the model load balance for the simulation.
subroutine destroy(this)
clean up
This module defines variable data types.
Definition kind.f90:8
character(len=lenmempath) function create_mem_path(component, subcomponent, context)
returns the path to the memory object
subroutine, public mem_print_detailed(iout)
subroutine, public get_isize(name, mem_path, isize)
Get the number of elements for this variable.
This module contains simulation methods.
Definition Sim.f90:10
subroutine, public store_warning(msg, substring)
Store warning message.
Definition Sim.f90:241
subroutine, public store_error(msg, terminate)
Store an error message.
Definition Sim.f90:94
integer(i4b) function, public count_errors()
Return number of errors.
Definition Sim.f90:60
subroutine, public store_error_filename(filename, terminate)
Store the erroring file name.
Definition Sim.f90:208
This module contains simulation variables.
character(len=maxcharlen) errmsg
error message string
character(len=linelength) idm_context
character(len=linelength) simulation_mode
integer(i4b) nr_procs
character(len=maxcharlen) warnmsg
warning message string
integer(i4b), dimension(:), pointer, contiguous model_ranks
all model processor ids (ranks) in the (global) simulation
integer(i4b) iout
file unit number for simulation output
integer(i4b) proc_id
subroutine, public table_cr(this, name, title)
Definition Table.f90:87
This class is used to store a single deferred-length character string. It was designed to work in an ...