Ticket #2011: suite.rc

File suite.rc, 7.8 KB (added by markmuetz, 4 years ago)
Line 
1#!jinja2
2{% macro format_final(run_periods, fmt='PT1S') -%}
3    {%- set ENDYR = '+P%dY'%(run_periods[0]) if run_periods[0] > 0 else '' -%}
4    {%- set ENDMO = '+P%dM'%(run_periods[1]) if run_periods[1] > 0 else '' -%}
5    {%- set ENDDA = '+P%dD'%(run_periods[2]) if run_periods[2] > 0 else '' -%}
6    {%- set ENDHR = '+PT%dH'%(run_periods[3]) if run_periods[3] > 0 else '' -%}
7    {%- set ENDMI = '+PT%dM'%(run_periods[4]) if run_periods[4] > 0 else '' -%}
8    {{ENDYR}}{{ENDMO}}{{ENDDA}}{{ENDHR}}{{ENDMI}}-{{fmt}}
9{%- endmacro %}
10
11{% if BASIS[0] == 0 %}
12  {% set CYCLE_INIT = '00000101' %}
13{% else %}
14  {% set CYCLE_INIT = '%04d%02d%02dT%02d%02d' % (BASIS[0],BASIS[1],BASIS[2],BASIS[3],BASIS[4]) %}
15{% endif %}
16{% set FMT = 'P' + ('T' if CYCLE == 'm' or CYCLE == 'H' else '') + RESUB|string + CYCLE|upper %}
17
18[cylc]
19    UTC mode = True
20    # Timeout handlers
21    [[event hooks]]
22        timeout handler = "rose suite-hook --mail --shutdown"
23        timeout = P5D # 5 Days
24
25[scheduling]
26    cycling mode        = {{CALENDAR}}
27    initial cycle point = {{CYCLE_INIT}}
28    final cycle point   = {{ format_final(RUNLEN, FMT) if RUN else CYCLE_INIT }}
29    [[dependencies]]
30        [[[ R1 ]]]
31            graph = """
32{{ 'fcm_make => fcm_make2' + (' => recon' if RECON else ' => atmos' if RUN else '') if BUILD_UM else '' }}
33{{ 'recon' + (' => atmos' if RUN else '') if RECON else '' }}
34{% if POST_PROC %}
35post_proc
36{% endif %}
37"""
38        [[[ {{FMT}} ]]]
39            graph = """
40{% if RUN %}
41atmos[-{{FMT}}] => atmos
42{% endif %}
43"""
44        # Run once at the final cycle point, see:
45        # https://cylc.github.io/cylc/html/single/cug-html.html#9.3.4
46        [[[ R1/P0Y ]]]
47            graph = """
48{% if RUN %}
49    {% if ARCHIVE %}
50    atmos => archive {% if POST_PROC %} => post_proc {% endif %}
51    {% endif %}
52{% else %}
53    {% if ARCHIVE %}
54    archive {% if POST_PROC %} => post_proc {% endif %}
55    {$ elif POST_PROC %}
56    post_proc
57    {% endif %}
58{% endif %}
59"""
60
61[runtime]
62# Root, inherited by everything
63    [[root]]
64        initial scripting = """
65export CYLC_VERSION={{CYLC_VERSION}}
66export ROSE_VERSION={{ROSE_VERSION}}
67"""
68        [[[event hooks]]]
69            succeeded handler = "rose suite-hook"
70            failed handler = "rose suite-hook"
71            retry handler = "rose suite-hook --mail"
72            submission failed handler = "rose suite-hook --mail"
73            submission timeout = PT12H # 12 hours
74            submission timeout handler = "rose suite-hook --mail"
75            execution timeout  =  PT3H # 3 hours
76            execution timeout handler = "rose suite-hook --mail"
77        [[[environment]]]
78            MODELBASIS = '{{ BASIS | join(',') }}'
79            lcal360    = {{'.true.' if CALENDAR == '360day' else '.false.'}}
80            CUMFDIR=$CYLC_TASK_WORK_PATH
81            DATAW=$CYLC_TASK_WORK_DIR
82            DATAM=$CYLC_SUITE_SHARE_DIR/data/{{DATAM}}
83            ROSE_ORIG_HOST={{ ROSE_ORIG_HOST }}
84            INPUT_DATA=$UMDIR/standard_jobs/inputs/vn{{VN}}
85            VN = {{ VN }}
86            NUM_LEVS = 70
87            NUM_LEVS_M1 = 69
88            NUM_LEVS_M2 = 68
89
90     [[XC30]]
91        command scripting = "rose task-run --verbose"
92        environment scripting = "eval $(rose task-env)"
93        initial scripting = "export HOME=$DATADIR"
94        pre-command scripting = """
95module load cray-netcdf/4.4.0
96module load cray-hdf5/1.8.16
97module list 2>&1             
98                                """
99        [[[directives]]]
100            -W umask = 0022
101            -A = {{ HPC_ACCOUNT }}
102        [[[job submission]]]
103            method = pbs
104        [[[remote]]]
105            host = {{ HPC_HOST }}
106{% if HPC_USER is defined %}
107            owner = {{ HPC_USER }}
108{% endif %}         
109
110     [[LINUX]]
111        command scripting = "rose task-run --verbose"
112        environment scripting = "eval $(rose task-env)"
113        [[[remote]]]
114            host = {{ ROSE_ORIG_HOST }}
115        [[[job submission]]]
116            method = at
117
118# UM high OpenMP
119    # Extract, Mirror
120    [[fcm_make]]
121        inherit = LINUX
122        [[[environment]]]
123            PREBUILD = ''
124
125    # Pre-process, Build
126    [[fcm_make2]]
127        inherit = XC30
128        [[[directives]]]
129            -l walltime=10:00:00
130            -l select=serial=true:ncpus=1
131        [[[environment]]]
132            UM_INSTALL_DIR  = /work/y07/y07/umshared
133            PREBUILD = ''
134            ROSE_TASK_N_JOBS     = 6
135            ROSE_TASK_OPTIONS=--ignore-lock
136
137    # Reconfiguration 1x2 - using shared node with mpiexec launcher
138    # NOTE - If the number of tasks is changed to be 32 or greater then
139    # the job launcher should be changed to aprun, see atmos example below
140    [[recon]]
141        inherit = XC30
142        [[[directives]]]
143            -l select=1
144            -l walltime=00:20:00
145{% if HPC_QUEUE is defined %}
146            -q = {{ HPC_QUEUE }}
147{% endif %}
148        [[[environment]]]
149            UM_INSTALL_DIR  = /work/y07/y07/umshared
150            ROSE_TASK_APP    = um
151            ASTART=../recon/atmos.astart
152            RCF_NPROCX = 4
153            RCF_NPROCY = 6
154            OMP_NUM_THREADS = 1
155            MPI_TASKS_PER_NODE = 24
156            TASKS_PER_NUMA = 12
157            TOTAL_MPI_TASKS = 24
158            ROSE_LAUNCHER_PREOPTS="""-ss -n $TOTAL_MPI_TASKS
159                                   -N $MPI_TASKS_PER_NODE
160                                   -S $TASKS_PER_NUMA
161                                   -d $OMP_NUM_THREADS"""
162            TASKSTART = $MODELBASIS
163            TASKEND   = $MODELBASIS
164
165    # Atmosphere Model Run 4x8 - using parallel node (i.e. full node, not shared) aprun launcher
166    [[atmos]]
167       inherit = XC30
168       [[[directives]]]
169          -l select=8
170          -l walltime=03:00:00
171{% if HPC_QUEUE is defined %}
172          -q = {{ HPC_QUEUE }}
173{% endif %}
174       [[[environment]]]
175          CONTINUE = $(if [[ $CYLC_TASK_CYCLE_POINT == $CYLC_SUITE_INITIAL_CYCLE_POINT ]] && [[ $CYLC_TASK_TRY_NUMBER -eq 1 ]]; then echo ""; else echo "true"; fi)
176          UM_INSTALL_DIR  = /work/y07/y07/umshared
177          ROSE_TASK_APP    = um
178          ASTART=../recon/atmos.astart
179          UM_ATM_NPROCX = {{ATM_PROC_X}}
180          UM_ATM_NPROCY = {{ATM_PROC_Y}}
181          FLUME_IOS_NPROC = 0
182          HYPERTHREADS = 1
183          OMP_NUM_THREADS = 1
184          MPI_TASKS_PER_NODE = 24
185          TASKS_PER_NUMA = 12
186          TOTAL_MPI_TASKS = $((UM_ATM_NPROCX * UM_ATM_NPROCY))
187          TASKSTART   = $( rose date $CYLC_TASK_CYCLE_POINT --print-format %Y,%m,%d,%H,%M )
188          TASKLENGTH  = $( rose date 0000 0000 --calendar {{CALENDAR}} -s -{{FMT}} -f y,m,d,h,M,s )
189          TASKEND         = $( rose date $CYLC_SUITE_INITIAL_CYCLE_POINT $CYLC_TASK_CYCLE_POINT \
190--calendar {{CALENDAR}} --offset -{{FMT}} --print-format y,m,d,h,M,s )
191          ROSE_LAUNCHER_PREOPTS="""-ss -n $TOTAL_MPI_TASKS
192                                   -N $MPI_TASKS_PER_NODE
193                                   -S $TASKS_PER_NUMA
194                                   -d $OMP_NUM_THREADS"""
195    [[archive]]
196        command scripting = "cp -r /work/n02/n02/mmuetz/cylc-run/iUM_L70_vertlev_MC_on /nerc/n02/n02/mmuetz/um10.5_runs/20day/iUM_L70_vertlev_MC_on"
197        [[[job submission]]]
198            method = pbs
199        [[[directives]]]
200           -A = {{ HPC_ACCOUNT }}
201           -l walltime=01:00:00
202           -l select=serial=true:ncpus=1
203        [[[remote]]]
204            host = {{ HPC_HOST }}
205{% if HPC_USER is defined %}
206            owner = {{ HPC_USER }}
207{% endif %}         
208    [[post_proc]]
209        pre-command scripting = """
210module load anaconda
211export PATH=$PATH:/nerc/n02/n02/mmuetz/.local/bin/
212cd /home/n02/n02/mmuetz/omnis/iUM_moisture_budget
213                                """
214        command scripting = "omni process -b batch2"
215        [[[job submission]]]
216            method = pbs
217        [[[directives]]]
218           -A = {{ HPC_ACCOUNT }}
219           -l walltime=05:00:00
220           -l select=serial=true:ncpus=1
221        [[[remote]]]
222            host = {{ HPC_HOST }}
223{% if HPC_USER is defined %}
224            owner = {{ HPC_USER }}
225{% endif %}