MODFLOW 6 version 6.9.0.dev0
USGS Modular Hydrologic Model
Loading...
Searching...
No Matches
distributedsimmodule Module Reference

Data Types

type  distributedsimtype

Functions/Subroutines

class(distributedsimtype) function, pointer, public get_dsim ()
 Get pointer to the distributed simulation object.
subroutine create (this)
 Create the distributed simulation object from the simulation input ctx.
integer(i4b) function, dimension(:), pointer get_load_mask (this)
 Return pointer to the load mask for models.
subroutine create_load_mask (this)
 Create a load mask for IDM from the load balance array.
integer(i4b) function, dimension(:), pointer get_load_balance (this)
 Get the model load balance for the simulation.
subroutine set_load_balance_from_input (this)
 Load load balance from the input configuration.
subroutine set_load_balance_default (this)
 Distribute the models over the available processes in a parallel run. Expects an array sized.
subroutine validate_load_balance (this)
 Check validity of load balance configuration.
subroutine print_load_balance (this)
 Print the load balance table to the listing file.
subroutine destroy (this)
 clean up

Variables

class(distributedsimtype), pointer, private dist_sim => null()

Function/Subroutine Documentation

◆ create()

subroutine distributedsimmodule::create ( class(distributedsimtype) this)
private

Definition at line 60 of file DistributedSim.f90.

61 class(DistributedSimType) :: this
62 ! local
63 character(len=LENMEMPATH) :: input_mempath
64 integer(I4B), pointer :: nmod
65
66 this%memory_path = create_mem_path(component='SIM')
67
68 input_mempath = create_mem_path(component='SIM', context=idm_context)
69 call mem_setptr(nmod, 'NUMMODELS', input_mempath)
70
71 call mem_allocate(this%nr_models, 'NUMMODELS', this%memory_path)
72 this%nr_models = nmod
73
74 call mem_allocate(this%print_ptable, 'PRINT_PTABLE', this%memory_path)
75 this%print_ptable = .false.
76
Here is the call graph for this function:

◆ create_load_mask()

subroutine distributedsimmodule::create_load_mask ( class(distributedsimtype) this)
private

Definition at line 101 of file DistributedSim.f90.

102 class(DistributedSimType) :: this
103 ! local
104 integer(I4B), dimension(:), pointer :: model_ranks => null() !< the load balance
105 integer(I4B) :: m_id !< model id
106
107 call mem_allocate(this%load_mask, this%nr_models, 'LOADMASK', &
108 this%memory_path)
109 this%load_mask = 0
110
111 ! get load balance (probably the first call, so creates it)
112 model_ranks => this%get_load_balance()
113
114 ! set mask from balance
115 do m_id = 1, this%nr_models
116 if (model_ranks(m_id) == proc_id) then
117 this%load_mask(m_id) = 1
118 else
119 this%load_mask(m_id) = 0
120 end if
121 end do
122

◆ destroy()

subroutine distributedsimmodule::destroy ( class(distributedsimtype) this)
private

Definition at line 543 of file DistributedSim.f90.

544 class(DistributedSimType) :: this
545
546 if (associated(this%load_mask)) then
547 call mem_deallocate(this%load_mask)
548 call mem_deallocate(this%model_ranks)
549 end if
550
551 call mem_deallocate(this%nr_models)
552 call mem_deallocate(this%print_ptable)
553
554 ! delete singleton instance
555 if (associated(dist_sim)) deallocate (dist_sim)
556

◆ get_dsim()

class(distributedsimtype) function, pointer, public distributedsimmodule::get_dsim

Definition at line 47 of file DistributedSim.f90.

48 class(DistributedSimType), pointer :: ds
49
50 if (.not. associated(dist_sim)) then
51 allocate (dist_sim)
52 call dist_sim%create()
53 end if
54 ds => dist_sim
55
Here is the caller graph for this function:

◆ get_load_balance()

integer(i4b) function, dimension(:), pointer distributedsimmodule::get_load_balance ( class(distributedsimtype) this)
private
Parameters
thisthis distributed sim instance
Returns
the load balance: array of ranks per model id

Definition at line 127 of file DistributedSim.f90.

128 use simvariablesmodule, only: iout
130 class(DistributedSimType) :: this !< this distributed sim instance
131 integer(I4B), dimension(:), pointer :: mranks !< the load balance: array of ranks per model id
132 ! local
133 integer(I4B) :: isize
134 logical(LGP) :: hpc6_present, partitions_present
135 character(len=LENMEMPATH) :: simnam_mempath, hpc_mempath
136 type(UtlHpcParamFoundType) :: found
137
138 ! if load balance available, return here:
139 if (associated(this%model_ranks)) then
140 mranks => this%model_ranks
141 return
142 end if
143
144 call mem_allocate(this%model_ranks, this%nr_models, 'MODELRANKS', &
145 this%memory_path)
146
147 ! check for optional HPC file
148 simnam_mempath = create_mem_path('SIM', 'NAM', idm_context)
149 call get_isize('HPC6_FILENAME', simnam_mempath, isize)
150 hpc6_present = isize > 0
151
152 ! handle serial case
153 if (simulation_mode == 'SEQUENTIAL') then
154 if (hpc6_present) then
155 write (warnmsg, *) "Ignoring PARTITIONS block in HPC file when "// &
156 "running a serial process"
157 call store_warning(warnmsg)
158 end if
159
160 ! single process, everything on cpu 0:
161 this%model_ranks = 0
162 mranks => this%model_ranks
163 return
164 end if
165
166 ! continue for PARALLEL mode only:
167 write (iout, '(/1x,a)') 'PROCESSING HPC DATA'
168
169 hpc_mempath = create_mem_path('UTL', 'HPC', idm_context)
170 ! source optional print input flag
171 call mem_set_value(this%print_ptable, 'PRINT_TABLE', hpc_mempath, &
172 found%print_table)
173 ! check if optional partition block exists
174 call get_isize('MNAME', hpc_mempath, isize)
175 partitions_present = isize > 0
176
177 ! fill model ranks (i.e. the load balance)
178 if (partitions_present) then
179 ! set balance from HPC file
180 call this%set_load_balance_from_input()
181 call this%validate_load_balance()
182 write (iout, '(1x,a)') 'Read partition data from HPC file'
183 else
184 ! no HPC file present, set balance with default algorithm
185 call this%set_load_balance_default()
186 write (iout, '(1x,a)') 'Generate default partition data'
187 end if
188
189 mranks => this%model_ranks
190
191 ! print to listing file
192 if (this%print_ptable) then
193 call this%print_load_balance()
194 end if
195
196 write (iout, '(1x,a)') 'END OF HPC DATA'
197
This module contains simulation variables.
character(len=linelength) idm_context
character(len=linelength) simulation_mode
character(len=maxcharlen) warnmsg
warning message string
integer(i4b) iout
file unit number for simulation output
Here is the call graph for this function:

◆ get_load_mask()

integer(i4b) function, dimension(:), pointer distributedsimmodule::get_load_mask ( class(distributedsimtype) this)
private

Get a load mask to determine which models should be loaded by idm on this process. This is in sync with models create. The mask array is allocated with its size equal to the global number of models. It is returned as (1, 1, 0, 0, ... 0) with each entry being a load mask for the model at the corresponding

Definition at line 88 of file DistributedSim.f90.

89 class(DistributedSimType) :: this
90 integer(I4B), dimension(:), pointer :: load_mask
91
92 if (.not. associated(this%load_mask)) then
93 call this%create_load_mask()
94 end if
95 load_mask => this%load_mask
96

◆ print_load_balance()

subroutine distributedsimmodule::print_load_balance ( class(distributedsimtype) this)
private

Definition at line 486 of file DistributedSim.f90.

487 use tablemodule, only: tabletype, table_cr
490 class(DistributedSimType) :: this
491 ! local
492 type(TableType), pointer :: inputtab => null()
493 character(len=LINELENGTH) :: tag, term
494 character(len=LENMEMPATH) :: input_mempath
495 type(CharacterStringType), dimension(:), contiguous, &
496 pointer :: mtypes !< model types
497 type(CharacterStringType), dimension(:), contiguous, &
498 pointer :: mnames !< model names
499 integer(I4B) :: im, nr_models
500
501 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
502
503 call mem_setptr(mtypes, 'MTYPE', input_mempath)
504 call mem_setptr(mnames, 'MNAME', input_mempath)
505
506 ! setup table
507 nr_models = size(mnames)
508 call table_cr(inputtab, 'HPC', 'HPC PARTITION DATA')
509 call inputtab%table_df(nr_models, 5, iout)
510
511 ! add columns
512 tag = 'ID'
513 call inputtab%initialize_column(tag, 8, alignment=tableft)
514 tag = 'NAME'
515 call inputtab%initialize_column(tag, lenmodelname + 4, alignment=tableft)
516 tag = 'TYPE'
517 call inputtab%initialize_column(tag, 8, alignment=tableft)
518 tag = 'RANK'
519 call inputtab%initialize_column(tag, 8, alignment=tableft)
520 tag = 'LOCAL'
521 call inputtab%initialize_column(tag, 8, alignment=tableft)
522
523 do im = 1, nr_models
524 call inputtab%add_term(im)
525 term = mnames(im)
526 call inputtab%add_term(term)
527 term = mtypes(im)
528 call inputtab%add_term(term)
529 call inputtab%add_term(this%model_ranks(im))
530 term = ''
531 if (this%model_ranks(im) == proc_id) term = 'X'
532 call inputtab%add_term(term)
533 end do
534
535 ! deallocate
536 call inputtab%table_da()
537 deallocate (inputtab)
538
This module contains simulation constants.
Definition Constants.f90:9
@ tabcenter
centered table column
@ tableft
left justified table column
integer(i4b), parameter lenmodelname
maximum length of the model name
Definition Constants.f90:22
integer(i4b) proc_id
subroutine, public table_cr(this, name, title)
Definition Table.f90:87
Here is the call graph for this function:

◆ set_load_balance_default()

subroutine distributedsimmodule::set_load_balance_default ( class(distributedsimtype) this)
private
Parameters
thisthis distributed sim. instance

Definition at line 297 of file DistributedSim.f90.

298 class(DistributedSimType) :: this !< this distributed sim. instance
299 ! local
300 integer(I4B) :: im, imm, ie, ip, cnt
301 integer(I4B) :: nr_models, nr_gwf_models
302 integer(I4B) :: nr_exchanges
303 integer(I4B) :: min_per_proc, nr_left
304 integer(I4B) :: rank
305 integer(I4B), dimension(:), allocatable :: nr_models_proc
306 character(len=LENPACKAGETYPE) :: model_type_str
307 character(len=LENMEMPATH) :: input_mempath
308 type(CharacterStringType), dimension(:), contiguous, &
309 pointer :: mtypes !< model types
310 type(CharacterStringType), dimension(:), contiguous, &
311 pointer :: mnames !< model names
312 type(CharacterStringType), dimension(:), contiguous, &
313 pointer :: etypes !< exg types
314 type(CharacterStringType), dimension(:), contiguous, &
315 pointer :: emnames_a !< model a names
316 type(CharacterStringType), dimension(:), contiguous, &
317 pointer :: emnames_b !< model b names
318
319 this%model_ranks = 0
320
321 ! load IDM data
322 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
323 call mem_setptr(mtypes, 'MTYPE', input_mempath)
324 call mem_setptr(mnames, 'MNAME', input_mempath)
325 call mem_setptr(etypes, 'EXGTYPE', input_mempath)
326 call mem_setptr(emnames_a, 'EXGMNAMEA', input_mempath)
327 call mem_setptr(emnames_b, 'EXGMNAMEB', input_mempath)
328
329 ! count flow models
330 nr_models = size(mnames)
331 nr_gwf_models = 0
332 do im = 1, nr_models
333 if (mtypes(im) == 'GWF6') then
334 nr_gwf_models = nr_gwf_models + 1
335 end if
336
337 if (mtypes(im) == 'GWF6' .or. &
338 mtypes(im) == 'GWT6' .or. &
339 mtypes(im) == 'GWE6') then
340 cycle
341 end if
342
343 model_type_str = mtypes(im)
344 write (errmsg, *) 'Model type ', model_type_str, &
345 ' not supported in parallel mode.'
346 call store_error(errmsg, terminate=.true.)
347 end do
348
349 ! calculate nr of flow models for each rank
350 allocate (nr_models_proc(nr_procs))
351 min_per_proc = nr_gwf_models / nr_procs
352 nr_left = nr_gwf_models - nr_procs * min_per_proc
353 cnt = 1
354 do ip = 1, nr_procs
355 rank = ip - 1
356 nr_models_proc(ip) = min_per_proc
357 if (rank < nr_left) then
358 nr_models_proc(ip) = nr_models_proc(ip) + 1
359 end if
360 end do
361
362 ! assign ranks for flow models
363 rank = 0
364 do im = 1, nr_models
365 if (mtypes(im) == 'GWF6') then
366 if (nr_models_proc(rank + 1) == 0) then
367 rank = rank + 1
368 end if
369 this%model_ranks(im) = rank
370 nr_models_proc(rank + 1) = nr_models_proc(rank + 1) - 1
371 end if
372 end do
373
374 ! match other models to flow
375 nr_exchanges = size(etypes)
376 do im = 1, nr_models
377 if (mtypes(im) == 'GWT6') then
378
379 ! find match
380 do ie = 1, nr_exchanges
381 if (etypes(ie) == 'GWF6-GWT6' .and. mnames(im) == emnames_b(ie)) then
382 rank = 0
383 do imm = 1, nr_models
384 if (mnames(imm) == emnames_a(ie)) then
385 rank = this%model_ranks(imm)
386 exit
387 end if
388 end do
389 this%model_ranks(im) = rank
390 exit
391 end if
392 end do
393
394 else if (mtypes(im) == 'GWE6') then
395 do ie = 1, nr_exchanges
396 if (etypes(ie) == 'GWF6-GWE6' .and. mnames(im) == emnames_b(ie)) then
397 rank = 0
398 do imm = 1, nr_models
399 if (mnames(imm) == emnames_a(ie)) then
400 rank = this%model_ranks(imm)
401 exit
402 end if
403 end do
404 this%model_ranks(im) = rank
405 exit
406 end if
407 end do
408
409 else
410 cycle ! e.g., for a flow model
411 end if
412 end do
413
414 ! cleanup
415 deallocate (nr_models_proc)
416
Here is the call graph for this function:

◆ set_load_balance_from_input()

subroutine distributedsimmodule::set_load_balance_from_input ( class(distributedsimtype) this)
private
Parameters
thisthis distributed sim instance

Definition at line 202 of file DistributedSim.f90.

203 class(DistributedSimType) :: this !< this distributed sim instance
204 ! local
205 character(len=LENMEMPATH) :: simnam_mempath, hpc_mempath
206 character(len=LENMODELNAME) :: model_name
207 type(CharacterStringType), dimension(:), contiguous, pointer :: mnames !< model names (all) from the simulation nam file
208 type(CharacterStringType), dimension(:), contiguous, pointer :: mnames_hpc !< model names in the hpc file
209 integer(I4B), dimension(:), contiguous, pointer :: mranks_hpc !< rank numbers in the hpc file
210 integer(I4B) :: i, model_idx
211 integer(I4B) :: target_rank
212 integer(I4B), dimension(:), allocatable :: rank_used
213 type(CharacterStringType), dimension(:), contiguous, pointer :: hpc_names !< helper array to get the hpc filename
214 character(len=LINELENGTH) :: hpc_filename !< the HPC option file
215
216 ! set to uninitialized
217 this%model_ranks = -1
218
219 ! from IDM
220 simnam_mempath = create_mem_path('SIM', 'NAM', idm_context)
221 hpc_mempath = create_mem_path('UTL', 'HPC', idm_context)
222 call mem_setptr(mnames, 'MNAME', simnam_mempath)
223 call mem_setptr(mnames_hpc, 'MNAME', hpc_mempath)
224 call mem_setptr(mranks_hpc, 'MRANK', hpc_mempath)
225 call mem_setptr(hpc_names, 'HPC6_FILENAME', simnam_mempath)
226
227 ! FILEIN options give an array, so take the first:
228 hpc_filename = hpc_names(1)
229
230 ! check: valid model names
231 do i = 1, size(mnames_hpc)
232 if (ifind(mnames, mnames_hpc(i)) == -1) then
233 model_name = mnames_hpc(i)
234 write (errmsg, *) "HPC input error: undefined model name (", &
235 trim(model_name), ")"
236 call store_error(errmsg)
237 end if
238 end do
239 ! check: valid ranks
240 do i = 1, size(mranks_hpc)
241 target_rank = mranks_hpc(i)
242 if (target_rank < 0 .or. target_rank > nr_procs - 1) then
243 model_name = mnames_hpc(i)
244 write (errmsg, '(a,i0,2a)') "HPC input error: invalid target rank (", &
245 target_rank, ") for model ", trim(model_name)
246 call store_error(errmsg)
247 end if
248 end do
249 if (count_errors() > 0) then
250 call store_error_filename(hpc_filename)
251 end if
252
253 ! construct rank array
254 do i = 1, size(mnames_hpc)
255 model_idx = ifind(mnames, mnames_hpc(i))
256 this%model_ranks(model_idx) = mranks_hpc(i)
257 end do
258
259 ! check: all models acquired rank
260 do i = 1, size(this%model_ranks)
261 if (this%model_ranks(i) == -1) then
262 model_name = mnames(i)
263 write (errmsg, '(2a)') "HPC input error: no target rank for model ", &
264 trim(model_name)
265 call store_error(errmsg)
266 end if
267 end do
268 if (count_errors() > 0) then
269 call store_error_filename(hpc_filename)
270 end if
271
272 ! check: no idle ranks
273 allocate (rank_used(nr_procs))
274 rank_used = 0
275 do i = 1, size(this%model_ranks)
276 if (this%model_ranks(i) >= 0 .and. this%model_ranks(i) < nr_procs) then
277 rank_used(this%model_ranks(i) + 1) = 1
278 end if
279 end do
280 do i = 1, size(rank_used)
281 if (rank_used(i) == 0) then
282 write (errmsg, '(a,i0,a)') "HPC input error: rank ", i - 1, &
283 " has no models assigned"
284 call store_error(errmsg)
285 end if
286 end do
287 deallocate (rank_used)
288 if (count_errors() > 0) then
289 call store_error_filename(hpc_filename)
290 end if
291
Here is the call graph for this function:

◆ validate_load_balance()

subroutine distributedsimmodule::validate_load_balance ( class(distributedsimtype) this)
private

Definition at line 421 of file DistributedSim.f90.

422 class(DistributedSimType) :: this
423 ! local
424 character(len=LENMEMPATH) :: input_mempath
425 type(CharacterStringType), dimension(:), contiguous, &
426 pointer :: mtypes !< model types
427 type(CharacterStringType), dimension(:), contiguous, &
428 pointer :: mnames !< model names
429 type(CharacterStringType), dimension(:), contiguous, &
430 pointer :: etypes !< exg types
431 type(CharacterStringType), dimension(:), contiguous, &
432 pointer :: emnames_a !< model a names
433 type(CharacterStringType), dimension(:), contiguous, &
434 pointer :: emnames_b !< model b names
435 type(CharacterStringType), dimension(:), contiguous, &
436 pointer :: hpc_names !< helper array to get hpc filename
437 integer(I4B) :: ie
438 integer(I4B) :: idx_a, idx_b
439 integer(I4B) :: rank_a, rank_b
440 integer(I4B) :: nr_exchanges
441 character(len=LINELENGTH) :: hpc_filename !< the HPC option file
442 character(len=LENMODELNAME) :: name_a, name_b
443 character(len=LINELENGTH) :: exg_type
444
445 ! load IDM data
446 input_mempath = create_mem_path('SIM', 'NAM', idm_context)
447 call mem_setptr(mtypes, 'MTYPE', input_mempath)
448 call mem_setptr(mnames, 'MNAME', input_mempath)
449 call mem_setptr(etypes, 'EXGTYPE', input_mempath)
450 call mem_setptr(emnames_a, 'EXGMNAMEA', input_mempath)
451 call mem_setptr(emnames_b, 'EXGMNAMEB', input_mempath)
452 call mem_setptr(hpc_names, 'HPC6_FILENAME', input_mempath)
453
454 ! FILEIN options give an array, so take the first:
455 hpc_filename = hpc_names(1)
456
457 nr_exchanges = size(etypes)
458
459 ! loop over exchanges
460 do ie = 1, nr_exchanges
461 if (etypes(ie) == 'GWF6-GWT6' .or. etypes(ie) == 'GWF6-GWE6') then
462 idx_a = ifind(mnames, emnames_a(ie))
463 idx_b = ifind(mnames, emnames_b(ie))
464 rank_a = this%model_ranks(idx_a)
465 rank_b = this%model_ranks(idx_b)
466 if (rank_a /= rank_b) then
467 name_a = emnames_a(ie)
468 name_b = emnames_b(ie)
469 exg_type = etypes(ie)
470 write (errmsg, '(7a)') "HPC input error: models ", &
471 trim(name_a), " and ", trim(name_b), " with a ", &
472 trim(exg_type), " coupling have to be assigned to the same rank"
473 call store_error(errmsg)
474 end if
475 end if
476 end do
477
478 if (count_errors() > 0) then
479 call store_error_filename(hpc_filename)
480 end if
481
Here is the call graph for this function:

Variable Documentation

◆ dist_sim

class(distributedsimtype), pointer, private distributedsimmodule::dist_sim => null()
private

Definition at line 41 of file DistributedSim.f90.