<core_client_version>7.16.6</core_client_version>
<![CDATA[
<message>
Disk usage limit exceeded</message>
<stderr_txt>
10.000
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:1631460:1631460] [20221225:212923:1671964163:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
21:30:32 STEP 768 H= 768:00 +CPU= 55.913
21:31:07 STEP 769 H= 769:00 +CPU= 32.348
21:31:43 STEP 770 H= 770:00 +CPU= 33.280
21:32:29 STEP 771 H= 771:00 +CPU= 43.041
21:33:01 STEP 772 H= 772:00 +CPU= 29.396
21:33:33 STEP 773 H= 773:00 +CPU= 29.715
21:34:20 STEP 774 H= 774:00 +CPU= 43.600
21:34:50 STEP 775 H= 775:00 +CPU= 27.310
21:35:24 STEP 776 H= 776:00 +CPU= 31.687
21:36:11 STEP 777 H= 777:00 +CPU= 43.478
21:36:42 STEP 778 H= 778:00 +CPU= 28.678
21:37:17 STEP 779 H= 779:00 +CPU= 32.101
21:38:11 STEP 780 H= 780:00 +CPU= 50.547
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000780
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000780
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000780
21:38:43 STEP 781 H= 781:00 +CPU= 28.743
21:39:17 STEP 782 H= 782:00 +CPU= 31.674
21:40:06 STEP 783 H= 783:00 +CPU= 44.882
21:40:39 STEP 784 H= 784:00 +CPU= 30.900
21:41:09 STEP 785 H= 785:00 +CPU= 28.270
21:41:58 STEP 786 H= 786:00 +CPU= 45.338
21:42:31 STEP 787 H= 787:00 +CPU= 30.735
21:43:04 STEP 788 H= 788:00 +CPU= 31.211
21:43:58 STEP 789 H= 789:00 +CPU= 51.472
21:44:31 STEP 790 H= 790:00 +CPU= 31.209
21:45:05 STEP 791 H= 791:00 +CPU= 31.598
21:46:06 STEP 792 H= 792:00 +CPU= 55.734
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000792
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000792
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000792
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000768
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000768
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000768
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000780
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000780
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000780
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_32.zip
Uploading the intermediate file: upload_file_32.zip
21:46:38 STEP 793 H= 793:00 +CPU= 29.277
21:47:08 STEP 794 H= 794:00 +CPU= 27.494
21:47:57 STEP 795 H= 795:00 +CPU= 45.426
21:48:30 STEP 796 H= 796:00 +CPU= 31.137
21:49:04 STEP 797 H= 797:00 +CPU= 32.729
21:49:55 STEP 798 H= 798:00 +CPU= 47.829
21:50:24 STEP 799 H= 799:00 +CPU= 28.141
21:50:56 STEP 800 H= 800:00 +CPU= 29.293
21:51:44 STEP 801 H= 801:00 +CPU= 44.843
21:52:12 STEP 802 H= 802:00 +CPU= 26.210
21:52:43 STEP 803 H= 803:00 +CPU= 28.028
21:53:38 STEP 804 H= 804:00 +CPU= 50.537
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000804
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000804
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000804
21:54:07 STEP 805 H= 805:00 +CPU= 27.162
21:54:40 STEP 806 H= 806:00 +CPU= 30.606
21:55:30 STEP 807 H= 807:00 +CPU= 47.386
21:56:01 STEP 808 H= 808:00 +CPU= 28.273
21:56:31 STEP 809 H= 809:00 +CPU= 27.472
21:57:17 STEP 810 H= 810:00 +CPU= 42.561
21:57:51 STEP 811 H= 811:00 +CPU= 31.412
21:58:27 STEP 812 H= 812:00 +CPU= 34.193
21:59:18 STEP 813 H= 813:00 +CPU= 48.050
21:59:49 STEP 814 H= 814:00 +CPU= 29.233
22:00:21 STEP 815 H= 815:00 +CPU= 29.657
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
22:01:51 STEP 816 H= 816:00 +CPU= 52.752
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000816
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000816
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000816
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000792
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000792
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000792
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000804
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000804
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000804
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_33.zip
Uploading the intermediate file: upload_file_33.zip
22:02:23 STEP 817 H= 817:00 +CPU= 30.181
22:02:58 STEP 818 H= 818:00 +CPU= 31.980
22:03:47 STEP 819 H= 819:00 +CPU= 45.364
22:04:18 STEP 820 H= 820:00 +CPU= 28.244
22:04:50 STEP 821 H= 821:00 +CPU= 28.729
22:05:38 STEP 822 H= 822:00 +CPU= 43.772
22:06:07 STEP 823 H= 823:00 +CPU= 26.394
22:06:40 STEP 824 H= 824:00 +CPU= 29.649
22:07:30 STEP 825 H= 825:00 +CPU= 46.001
22:08:01 STEP 826 H= 826:00 +CPU= 28.905
22:08:34 STEP 827 H= 827:00 +CPU= 29.851
22:09:25 STEP 828 H= 828:00 +CPU= 47.418
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000828
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000828
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000828
22:09:54 STEP 829 H= 829:00 +CPU= 25.504
22:10:28 STEP 830 H= 830:00 +CPU= 27.106
22:11:22 STEP 831 H= 831:00 +CPU= 47.321
22:11:57 STEP 832 H= 832:00 +CPU= 31.947
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
22:12:40 STEP 833 H= 833:00 +CPU= 30.359
22:13:29 STEP 834 H= 834:00 +CPU= 44.345
22:13:57 STEP 835 H= 835:00 +CPU= 25.735
22:14:30 STEP 836 H= 836:00 +CPU= 28.774
22:15:24 STEP 837 H= 837:00 +CPU= 48.679
22:15:58 STEP 838 H= 838:00 +CPU= 32.737
22:16:35 STEP 839 H= 839:00 +CPU= 34.241
22:17:34 STEP 840 H= 840:00 +CPU= 53.038
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000840
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000840
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000840
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000816
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000816
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000816
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000828
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000828
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000828
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_34.zip
Uploading the intermediate file: upload_file_34.zip
22:18:02 STEP 841 H= 841:00 +CPU= 25.291
22:18:32 STEP 842 H= 842:00 +CPU= 27.908
22:19:29 STEP 843 H= 843:00 +CPU= 51.092
22:20:05 STEP 844 H= 844:00 +CPU= 34.333
22:20:41 STEP 845 H= 845:00 +CPU= 33.508
22:21:35 STEP 846 H= 846:00 +CPU= 51.170
22:22:07 STEP 847 H= 847:00 +CPU= 30.424
22:22:38 STEP 848 H= 848:00 +CPU= 28.289
22:23:23 STEP 849 H= 849:00 +CPU= 39.851
22:23:56 STEP 850 H= 850:00 +CPU= 28.467
22:24:31 STEP 851 H= 851:00 +CPU= 30.076
22:25:34 STEP 852 H= 852:00 +CPU= 57.968
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000852
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000852
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000852
22:26:08 STEP 853 H= 853:00 +CPU= 31.872
22:26:42 STEP 854 H= 854:00 +CPU= 31.753
22:27:27 STEP 855 H= 855:00 +CPU= 41.645
22:27:58 STEP 856 H= 856:00 +CPU= 27.182
22:28:31 STEP 857 H= 857:00 +CPU= 29.054
22:29:24 STEP 858 H= 858:00 +CPU= 47.400
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2012050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0812
(argv4) batchid: 981
(argv5) wuid: 12198456
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/5
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_770f5718358bcbc84930f57f9f295a8b to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
ic_ancil_file: ic_ancil_12198456
ifsdata_file: ifsdata_12198456
climate_data_file: clim_data_12198456
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_fe74f80180ad967f040aba5e9dfbc2b0 to: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_84b2bb678ff7074049ef5de525f1005a to: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Checking for progress XML file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
Opened progress file ok : /var/lib/boinc-client/slots/5/progress_file_12198456.xml
-- Model is restarting --
Adjusting last_iter, 858, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
last_cpu_time: 29424
upload_file_number: 35
last_iter: 839
last_upload: 3024000
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467
The child process has been launched with process id: 3382839
Executing the command: /var/lib/boinc-client/slots/5/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221225:223036:1671967836:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
22:31:39 STEP 840 H= 840:00 +CPU= 51.785
22:32:14 STEP 841 H= 841:00 +CPU= 29.938
22:32:49 STEP 842 H= 842:00 +CPU= 31.869
22:33:36 STEP 843 H= 843:00 +CPU= 44.003
22:34:09 STEP 844 H= 844:00 +CPU= 29.807
22:34:42 STEP 845 H= 845:00 +CPU= 30.898
22:35:29 STEP 846 H= 846:00 +CPU= 43.852
22:35:59 STEP 847 H= 847:00 +CPU= 27.490
22:36:34 STEP 848 H= 848:00 +CPU= 31.334
22:37:22 STEP 849 H= 849:00 +CPU= 44.485
22:37:54 STEP 850 H= 850:00 +CPU= 29.822
22:38:29 STEP 851 H= 851:00 +CPU= 32.313
22:39:23 STEP 852 H= 852:00 +CPU= 50.736
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000852
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000852
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000852
22:39:57 STEP 853 H= 853:00 +CPU= 31.035
22:40:29 STEP 854 H= 854:00 +CPU= 29.405
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
22:41:49 STEP 855 H= 855:00 +CPU= 45.132
22:42:22 STEP 856 H= 856:00 +CPU= 30.338
22:42:53 STEP 857 H= 857:00 +CPU= 29.119
22:43:43 STEP 858 H= 858:00 +CPU= 45.776
22:44:16 STEP 859 H= 859:00 +CPU= 30.671
22:44:49 STEP 860 H= 860:00 +CPU= 30.932
22:45:42 STEP 861 H= 861:00 +CPU= 48.592
22:46:18 STEP 862 H= 862:00 +CPU= 34.361
11:19:00 STEP 863 H= 863:00 +CPU= 29.658
11:20:07 STEP 864 H= 864:00 +CPU= 51.051
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000864
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000864
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000864
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000840
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000840
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000840
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000852
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000852
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000852
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_35.zip
Uploading the intermediate file: upload_file_35.zip
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
11:21:04 STEP 865 H= 865:00 +CPU= 30.043
11:21:41 STEP 866 H= 866:00 +CPU= 31.936
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.116] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#15 (SIGTERM) :: 2550MB (heap), 4411MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#15 was caused by unrecognized si_code [memaddr=0x1], nsigs = 1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#15, nsigs = 1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 2598 MB (maxheap), 4411 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.119] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WVCOUPLE
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WAVEMDL
[EC_DRHOOK:Lian:1:1:3382839:3382839] [20221226:112229:1672014149:46313.120] [c_drhook_print_@/h(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2012050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0812
(argv4) batchid: 981
(argv5) wuid: 12198456
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/5
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_770f5718358bcbc84930f57f9f295a8b to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
ic_ancil_file: ic_ancil_12198456
ifsdata_file: ifsdata_12198456
climate_data_file: clim_data_12198456
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_fe74f80180ad967f040aba5e9dfbc2b0 to: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_84b2bb678ff7074049ef5de525f1005a to: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Checking for progress XML file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
Opened progress file ok : /var/lib/boinc-client/slots/5/progress_file_12198456.xml
-- Model is restarting --
Adjusting last_iter, 867, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
last_cpu_time: 30445
upload_file_number: 36
last_iter: 863
last_upload: 3110400
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467
The child process has been launched with process id: 684229
Executing the command: /var/lib/boinc-client/slots/5/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:195951:1672131591:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
20:00:47 STEP 864 H= 864:00 +CPU= 45.732
20:01:17 STEP 865 H= 865:00 +CPU= 25.628
20:01:46 STEP 866 H= 866:00 +CPU= 26.130
20:02:30 STEP 867 H= 867:00 +CPU= 38.459
20:03:00 STEP 868 H= 868:00 +CPU= 26.062
20:03:30 STEP 869 H= 869:00 +CPU= 26.029
20:04:14 STEP 870 H= 870:00 +CPU= 39.316
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:04:58 STEP 871 H= 871:00 +CPU= 25.686
20:05:28 STEP 872 H= 872:00 +CPU= 26.645
20:06:12 STEP 873 H= 873:00 +CPU= 38.839
20:06:41 STEP 874 H= 874:00 +CPU= 26.178
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:07:29 STEP 875 H= 875:00 +CPU= 25.693
20:08:17 STEP 876 H= 876:00 +CPU= 42.579
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000876
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000876
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000876
20:08:46 STEP 877 H= 877:00 +CPU= 25.800
20:09:16 STEP 878 H= 878:00 +CPU= 26.342
20:10:00 STEP 879 H= 879:00 +CPU= 39.009
20:10:30 STEP 880 H= 880:00 +CPU= 26.249
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:11:35 STEP 881 H= 881:00 +CPU= 25.788
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:13:06 STEP 882 H= 882:00 +CPU= 38.892
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:13:46 STEP 883 H= 883:00 +CPU= 25.797
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:14:27 STEP 884 H= 884:00 +CPU= 26.696
20:15:11 STEP 885 H= 885:00 +CPU= 38.901
20:15:40 STEP 886 H= 886:00 +CPU= 25.965
20:16:10 STEP 887 H= 887:00 +CPU= 26.573
20:17:01 STEP 888 H= 888:00 +CPU= 45.460
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000888
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000888
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000888
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000864
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000864
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000864
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000876
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000876
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000876
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_36.zip
Uploading the intermediate file: upload_file_36.zip
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:17:44 STEP 889 H= 889:00 +CPU= 23.988
20:18:14 STEP 890 H= 890:00 +CPU= 26.766
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:21:32 STEP 891 H= 891:00 +CPU= 38.597
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:22:17 STEP 892 H= 892:00 +CPU= 25.899
20:22:46 STEP 893 H= 893:00 +CPU= 25.686
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:25:06 STEP 894 H= 894:00 +CPU= 41.695
20:25:34 STEP 895 H= 895:00 +CPU= 26.843
20:26:05 STEP 896 H= 896:00 +CPU= 28.879
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:27:06 STEP 897 H= 897:00 +CPU= 40.909
20:27:35 STEP 898 H= 898:00 +CPU= 28.245
20:28:04 STEP 899 H= 899:00 +CPU= 27.416
20:28:52 STEP 900 H= 900:00 +CPU= 45.442
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000900
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000900
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000900
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:30:15 STEP 901 H= 901:00 +CPU= 28.559
20:30:48 STEP 902 H= 902:00 +CPU= 28.713
20:31:38 STEP 903 H= 903:00 +CPU= 44.252
20:32:12 STEP 904 H= 904:00 +CPU= 30.761
20:32:45 STEP 905 H= 905:00 +CPU= 29.185
20:33:34 STEP 906 H= 906:00 +CPU= 45.762
20:34:05 STEP 907 H= 907:00 +CPU= 28.308
20:34:37 STEP 908 H= 908:00 +CPU= 30.110
20:35:25 STEP 909 H= 909:00 +CPU= 45.646
20:35:57 STEP 910 H= 910:00 +CPU= 30.885
20:36:30 STEP 911 H= 911:00 +CPU= 31.224
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.410] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#15 (SIGTERM) :: 3996MB (heap), 4419MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#15 was caused by unrecognized si_code [memaddr=0x1], nsigs = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#15, nsigs = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 3996 MB (maxheap), 4419 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : STEPO
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : SCAN2M
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL_HEAP
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : EC_PHYS_DRV
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : >OMP-PHYSICS CLDPP T/S (1002)
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : EC_PHYS
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CALLPAR
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GWDRAG_LAYER
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203703:1672133823:2232.414] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GWDRAG
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203704:1672133824:2233.416] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#15, nsigs = 1
[EC_DRHOOK:Lian:1:1:684229:684229] [20221227:203704:1672133824:2233.416] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#15, nsigs = 1
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001CCAFDC Unknown Unknown Unknown
oifs_43r3_model.e 000000000108F566 gwdrag_ 287 gwdrag.F90
oifs_43r3_model.e 000000000108DD4F gwdrag_layer_ 81 gwdrag_layer.F90
oifs_43r3_model.e 0000000000F0B558 callpar_ 851 callpar.F90
oifs_43r3_model.e 0000000000E4940F ec_phys_ 670 ec_phys.F90
oifs_43r3_model.e 0000000000E3610C ec_phys_drv_ 599 ec_phys_drv.F90
oifs_43r3_model.e 0000000000BFA3D3 gp_model_ 613 gp_model.F90
oifs_43r3_model.e 00000000012E9482 gp_model_heap_ 74 gp_model_heap.F90
oifs_43r3_model.e 0000000000BBC1CE scan2m_ 535 scan2m.F90
oifs_43r3_model.e 000000000057A664 stepo_ 327 stepo.F90
oifs_43r3_model.e 000000000055DEEC cnt4_ 1133 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2012050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0812
(argv4) batchid: 981
(argv5) wuid: 12198456
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/5
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_770f5718358bcbc84930f57f9f295a8b to: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/5/oifs_43r3_ps_0812_2012050100_123_981_12198456.zip
ic_ancil_file: ic_ancil_12198456
ifsdata_file: ifsdata_12198456
climate_data_file: clim_data_12198456
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_fe74f80180ad967f040aba5e9dfbc2b0 to: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/5/ic_ancil_12198456.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_84b2bb678ff7074049ef5de525f1005a to: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/5/ifsdata/ifsdata_12198456.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/5/159l_2/clim_data_12198456.zip
Checking for progress XML file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
Opened progress file ok : /var/lib/boinc-client/slots/5/progress_file_12198456.xml
-- Model is restarting --
Adjusting last_iter, 912, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/5/progress_file_12198456.xml
last_cpu_time: 32010
upload_file_number: 37
last_iter: 911
last_upload: 3196800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467
The child process has been launched with process id: 4565
Executing the command: /var/lib/boinc-client/slots/5/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Lian:1:1:4565:4565] [20221227:203825:1672133905:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
20:39:09 STEP 912 H= 912:00 +CPU= 37.036
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000912
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000912
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000912
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000888
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000888
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000888
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000900
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000900
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000900
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_37.zip
Uploading the intermediate file: upload_file_37.zip
20:39:34 STEP 913 H= 913:00 +CPU= 21.781
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:40:03 STEP 914 H= 914:00 +CPU= 22.037
20:40:44 STEP 915 H= 915:00 +CPU= 38.475
20:41:09 STEP 916 H= 916:00 +CPU= 24.513
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:41:44 STEP 917 H= 917:00 +CPU= 23.004
20:42:16 STEP 918 H= 918:00 +CPU= 30.831
20:42:36 STEP 919 H= 919:00 +CPU= 19.660
20:42:57 STEP 920 H= 920:00 +CPU= 20.415
20:43:32 STEP 921 H= 921:00 +CPU= 34.322
20:43:57 STEP 922 H= 922:00 +CPU= 24.588
20:44:19 STEP 923 H= 923:00 +CPU= 21.895
20:44:54 STEP 924 H= 924:00 +CPU= 33.318
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000924
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000924
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000924
20:45:13 STEP 925 H= 925:00 +CPU= 19.198
20:45:33 STEP 926 H= 926:00 +CPU= 19.485
20:46:02 STEP 927 H= 927:00 +CPU= 28.436
20:46:24 STEP 928 H= 928:00 +CPU= 20.920
20:46:46 STEP 929 H= 929:00 +CPU= 21.245
20:47:21 STEP 930 H= 930:00 +CPU= 33.966
20:47:45 STEP 931 H= 931:00 +CPU= 23.658
20:48:09 STEP 932 H= 932:00 +CPU= 23.361
20:48:47 STEP 933 H= 933:00 +CPU= 36.588
20:49:13 STEP 934 H= 934:00 +CPU= 24.612
20:49:34 STEP 935 H= 935:00 +CPU= 20.448
20:50:11 STEP 936 H= 936:00 +CPU= 36.492
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMGGhq0f+000936
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMSHhq0f+000936
Moving to projects directory: /var/lib/boinc-client/slots/5/ICMUAhq0f+000936
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000912
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000912
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000912
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMGGhq0f+000924
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMSHhq0f+000924
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12198456/ICMUAhq0f+000924
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0812_2012050100_123_981_12198456_0_r1897811467_38.zip
Uploading the intermediate file: upload_file_38.zip
20:50:33 STEP 937 H= 937:00 +CPU= 20.781
20:50:58 STEP 938 H= 938:00 +CPU= 24.323
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
20:51:46 STEP 939 H= 939:00 +CPU= 35.495
20:52:11 STEP 940 H= 940:00 +CPU= 23.885
20:52:35 STEP 941 H= 941:00 +CPU= 22.875
20:53:11 STEP 942 H= 942:00 +CPU= 34.977
Abort request received from BOINC client, ending the child process
..Failed, model did not complete successfully
</stderr_txt>
]]>
