Name QrGKDm9OjbvnShfckohDCDFpABFKDmABFKDmykOXDmABFKDm45cmln_1
Workunit 1943636
Created 7 Oct 2019, 17:15:57 UTC
Sent 8 Oct 2019, 0:45:50 UTC
Report deadline 15 Oct 2019, 0:45:50 UTC
Received 8 Oct 2019, 22:05:02 UTC
Server state Over
Outcome Success
Client state Done
Exit status 0 (0x00000000)
Computer ID 3911
Run time 6 hours 53 min 25 sec
CPU time 1 days 2 hours 43 min 43 sec
Validate state Valid
Credit 668.17
Device peak FLOPS 11.64 GFLOPS
Application version ATLAS Simulation v0.74 (native_mt)
x86_64-pc-linux-gnu
Peak working set size 1.81 GB
Peak swap size 2.58 GB
Peak disk usage 676.30 MB

Stderr output

<core_client_version>7.7.0</core_client_version>
<![CDATA[
<stderr_txt>
17:11:25 (31039): wrapper (7.7.26015): starting
17:11:25 (31039): wrapper: running run_atlas (--nthreads 4)
2019-10-08 17:11:25,307: singularity image is /cvmfs/atlas.cern.ch/repo/containers/images/singularity/x86_64-centos7.img
2019-10-08 17:11:25,307: sys.argv = ['run_atlas', '--nthreads', '4']
2019-10-08 17:11:25,308: THREADS=4
2019-10-08 17:11:25,310: Checking for CVMFS
2019-10-08 17:11:27,059: CVMFS is installed
2019-10-08 17:11:27,059: Checking Singularity...
2019-10-08 17:11:27,666: Singularity is installed, version singularity version 3.2.1-1
2019-10-08 17:11:27,667: Testing the function of Singularity...
2019-10-08 17:11:27,667: Checking singularity with cmd:singularity exec -B /cvmfs /cvmfs/atlas.cern.ch/repo/containers/images/singularity/x86_64-centos7.img hostname
2019-10-08 17:11:40,444: Singularity Works...
2019-10-08 17:11:40,445: copy /home/grid/slots/0/shared/ATLAS.root_0
2019-10-08 17:11:41,043: copy /home/grid/slots/0/shared/input.tar.gz
2019-10-08 17:11:41,044: copy /home/grid/slots/0/shared/RTE.tar.gz
2019-10-08 17:11:41,045: copy /home/grid/slots/0/shared/start_atlas.sh
2019-10-08 17:11:41,045: export ATHENA_PROC_NUMBER=4;
2019-10-08 17:11:41,085: start atlas job with PandaID=4502006949
2019-10-08 17:11:41,086: cmd = singularity exec --pwd /home/grid/slots/0 -B /cvmfs,/home /cvmfs/atlas.cern.ch/repo/containers/images/singularity/x86_64-centos7.img sh start_atlas.sh > runtime_log 2> runtime_log.err
2019-10-09 00:04:46,702: running cmd return value is 0
2019-10-09 00:04:46,706: *****************The last 200 lines of the pilot log******************
2019-10-09 00:04:47,142:         ], 
        "type": "EVNT"
      }
    ], 
    "output": [
      {
        "argName": "outputHITSFile", 
        "dataset": null, 
        "subFiles": [
          {
            "file_guid": null, 
            "file_size": null, 
            "name": "HITS.19056345._005910.pool.root.1_000", 
            "nentries": null
          }
        ], 
        "type": "HITS"
      }
    ]
  }, 
  "name": "Sim_tf", 
  "reportVersion": "2.0.7", 
  "resource": {
    "dbDataTotal": 8542136, 
    "dbTimeTotal": 112.08, 
    "executor": {
      "EVNTtoHITS": {
        "cpuTime": 24073, 
        "cpuTimePerWorker": 6018.25, 
        "dbData": 4271068, 
        "dbTime": 56.04, 
        "memory": {
          "Avg": {
            "avgPSS": 1412364, 
            "avgRSS": 5194722, 
            "avgSwap": 0, 
            "avgVMEM": 7537361, 
            "rateRBYTES": 216917, 
            "rateRCHAR": 0, 
            "rateWBYTES": 0, 
            "rateWCHAR": 0
          }, 
          "Max": {
            "maxPSS": 2572504, 
            "maxRSS": 9357572, 
            "maxSwap": 0, 
            "maxVMEM": 13453644, 
            "totRBYTES": 2263642112, 
            "totRCHAR": 691471085, 
            "totWBYTES": 160927744, 
            "totWCHAR": 160453191
          }
        }, 
        "mpworkers": 4, 
        "postExe": {
          "cpuTime": 130, 
          "wallTime": 1
        }, 
        "preExe": {
          "cpuTime": 0, 
          "wallTime": 1
        }, 
        "total": {
          "cpuTime": null, 
          "wallTime": null
        }, 
        "validation": {
          "cpuTime": null, 
          "wallTime": null
        }, 
        "wallTime": 24617
      }
    }, 
    "machine": {
      "linux_distribution": [
        "CentOS Linux", 
        "7.6.1810", 
        "Core"
      ], 
      "node": "dcameron04.cern.ch", 
      "platform": "Linux-3.10.0-957.el7.x86_64-x86_64-with-centos-7.6.1810-Core"
    }, 
    "transform": {
      "cpuEfficiency": 0.2444, 
      "cpuPWEfficiency": 0.0002, 
      "cpuTime": 5, 
      "cpuTimeTotal": 0, 
      "externalCpuTime": 131, 
      "trfPredata": null, 
      "wallTime": 24625
    }
  }
}
2019-10-08 22:04:08,248 | DEBUG    | queue_monitor       | pilot.util.auxiliary.4502006949  | update_server             | xml:will send fileinfo
2019-10-08 22:04:08,248 | INFO     | queue_monitor       | pilot.control.job.4502006949     | send_state                | pilot will not update the server (heartbeat message will be written to file)
2019-10-08 22:04:08,248 | INFO     | queue_monitor       | pilot.control.job.4502006949     | send_state                | job 4502006949 has failed - writing final server update
2019-10-08 22:04:08,248 | INFO     | queue_monitor       | pilot.control.job.4502006949     | verify_error_code         | verified error code
2019-10-08 22:04:08,253 | WARNING  | queue_monitor       | pilot.api.analytics              | get_fitted_data           | wrong length of table data, x=[1570547569.0, 1570547630.0, 1570547691.0, 1570547752.0, 1570547813.0, 1570547874.0, 1570547935.0, 1570547996.0, 1570548057.0, 1570548118.0, 1570548179.0, 1570548240.0, 1570548301.0, 1570548362.0, 1570548423.0, 1570548484.0, 1570548545.0, 1570548606.0, 1570548667.0, 1570548728.0, 1570548789.0, 1570548850.0, 1570548911.0, 1570548972.0, 1570549033.0, 1570549094.0, 1570549155.0, 1570549216.0, 1570549277.0, 1570549338.0, 1570549399.0, 1570549460.0, 1570549521.0, 1570549582.0, 1570549643.0, 1570549704.0, 1570549765.0, 1570549826.0, 1570549887.0, 1570549948.0, 1570550009.0, 1570550070.0, 1570550131.0, 1570550192.0, 1570550253.0, 1570550314.0, 1570550375.0, 1570550436.0, 1570550497.0, 1570550558.0, 1570550619.0, 1570550680.0, 1570550741.0, 1570550802.0, 1570550863.0, 1570550924.0, 1570550985.0, 1570551046.0, 1570551107.0, 1570551168.0, 1570551229.0, 1570551290.0, 1570551351.0, 1570551412.0, 1570551473.0, 1570551534.0, 1570551595.0, 1570551656.0, 1570551717.0, 1570551778.0, 1570551839.0, 1570551900.0, 1570551961.0, 1570552022.0, 1570552083.0, 1570552144.0, 1570552205.0, 1570552266.0, 1570552327.0, 1570552388.0, 1570552449.0, 1570552510.0, 1570552571.0, 1570552632.0, 1570552693.0, 1570552754.0, 1570552815.0, 1570552876.0, 1570552937.0, 1570552998.0, 1570553059.0, 1570553120.0, 1570553181.0, 1570553242.0, 1570553303.0, 1570553364.0, 1570553425.0, 1570553486.0, 1570553547.0, 1570553608.0, 1570553669.0, 1570553730.0, 1570553791.0, 1570553852.0, 1570553913.0, 1570553974.0, 1570554035.0, 1570554096.0, 1570554157.0, 1570554218.0, 1570554279.0, 1570554340.0, 1570554401.0, 1570554462.0, 1570554523.0, 1570554584.0, 1570554645.0, 1570554706.0, 1570554767.0, 1570554828.0, 1570554889.0, 1570554950.0, 1570555011.0, 1570555072.0, 1570555133.0, 1570555194.0, 1570555255.0, 1570555316.0, 1570555377.0, 1570555438.0, 1570555499.0, 1570555560.0, 1570555621.0, 1570555682.0, 1570555743.0, 1570555804.0, 1570555865.0, 1570555926.0, 1570555987.0, 1570556048.0, 1570556109.0, 1570556170.0, 1570556231.0, 1570556292.0, 1570556353.0, 1570556414.0, 1570556475.0, 1570556536.0, 1570556597.0, 1570556658.0, 1570556719.0, 1570556780.0, 1570556841.0, 1570556902.0, 1570556963.0, 1570557024.0, 1570557085.0, 1570557146.0, 1570557207.0, 1570557268.0, 1570557329.0, 1570557390.0, 1570557451.0, 1570557512.0, 1570557573.0, 1570557634.0, 1570557695.0, 1570557756.0, 1570557817.0, 1570557878.0, 1570557939.0, 1570558000.0, 1570558061.0, 1570558122.0, 1570558183.0, 1570558244.0, 1570558305.0, 1570558366.0, 1570558427.0, 1570558488.0, 1570558549.0, 1570558610.0, 1570558671.0, 1570558732.0, 1570558793.0, 1570558854.0, 1570558915.0, 1570558976.0, 1570559037.0, 1570559098.0, 1570559159.0, 1570559220.0, 1570559281.0, 1570559342.0, 1570559403.0, 1570559464.0, 1570559525.0, 1570559586.0, 1570559647.0, 1570559708.0, 1570559769.0, 1570559830.0, 1570559891.0, 1570559952.0, 1570560013.0, 1570560074.0, 1570560135.0, 1570560196.0, 1570560257.0, 1570560318.0, 1570560379.0, 1570560440.0, 1570560501.0, 1570560562.0, 1570560623.0, 1570560684.0, 1570560745.0, 1570560806.0, 1570560867.0, 1570560928.0, 1570560989.0, 1570561050.0, 1570561111.0, 1570561172.0, 1570561233.0, 1570561294.0, 1570561355.0, 1570561416.0, 1570561477.0, 1570561538.0, 1570561599.0, 1570561660.0, 1570561721.0, 1570561782.0, 1570561843.0, 1570561904.0, 1570561965.0, 1570562026.0, 1570562087.0, 1570562148.0, 1570562209.0, 1570562270.0, 1570562331.0, 1570562392.0, 1570562453.0, 1570562514.0, 1570562575.0, 1570562636.0, 1570562697.0, 1570562758.0, 1570562819.0, 1570562880.0, 1570562941.0, 1570563002.0, 1570563063.0, 1570563124.0, 1570563185.0, 1570563246.0, 1570563307.0, 1570563368.0, 1570563429.0, 1570563490.0, 1570563551.0, 1570563612.0, 1570563673.0, 1570563734.0, 1570563795.0, 1570563856.0, 1570563917.0, 1570563978.0, 1570564039.0, 1570564100.0, 1570564161.0, 1570564222.0, 1570564283.0, 1570564344.0, 1570564405.0, 1570564466.0, 1570564527.0, 1570564588.0, 1570564649.0, 1570564710.0, 1570564771.0, 1570564832.0, 1570564893.0, 1570564954.0, 1570565015.0, 1570565076.0, 1570565137.0, 1570565198.0, 1570565259.0, 1570565320.0, 1570565381.0, 1570565442.0, 1570565503.0, 1570565564.0, 1570565625.0, 1570565686.0, 1570565747.0, 1570565808.0, 1570565869.0, 1570565930.0, 1570565991.0, 1570566052.0, 1570566113.0, 1570566174.0, 1570566235.0, 1570566296.0, 1570566357.0, 1570566418.0, 1570566479.0, 1570566540.0, 1570566601.0, 1570566662.0, 1570566723.0, 1570566784.0, 1570566845.0, 1570566906.0, 1570566967.0, 1570567028.0, 1570567089.0, 1570567150.0, 1570567211.0, 1570567272.0, 1570567333.0, 1570567394.0, 1570567455.0, 1570567516.0, 1570567577.0, 1570567638.0, 1570567699.0, 1570567760.0, 1570567821.0, 1570567882.0, 1570567943.0, 1570568004.0, 1570568065.0, 1570568126.0, 1570568187.0, 1570568248.0, 1570568309.0, 1570568370.0, 1570568431.0, 1570568492.0, 1570568553.0, 1570568614.0, 1570568675.0, 1570568736.0, 1570568797.0, 1570568858.0, 1570568919.0, 1570568980.0, 1570569041.0, 1570569102.0, 1570569163.0, 1570569224.0, 1570569285.0, 1570569346.0, 1570569407.0, 1570569468.0, 1570569529.0, 1570569590.0, 1570569651.0, 1570569712.0, 1570569773.0, 1570569834.0, 1570569895.0, 1570569956.0, 1570570017.0, 1570570078.0, 1570570139.0, 1570570200.0, 1570570261.0, 1570570322.0, 1570570383.0, 1570570444.0, 1570570505.0, 1570570566.0, 1570570627.0, 1570570688.0, 1570570749.0, 1570570810.0, 1570570871.0, 1570570932.0, 1570570993.0, 1570571054.0, 1570571115.0, 1570571176.0, 1570571237.0, 1570571298.0, 1570571359.0, 1570571420.0, 1570571481.0, 1570571542.0, 1570571603.0, 1570571664.0, 1570571725.0, 1570571786.0, 1570571847.0, 1570571908.0, 1570571969.0, 1570572030.0, 1570572091.0, 1570572152.0, 1570572213.0], y=[] (must be same and length>=2)
2019-10-08 22:04:08,253 | DEBUG    | queue_monitor       | pilot.util.auxiliary.4502006949  | get_job_metrics           | job metrics="coreCount=4 workDirSize=31597351"
2019-10-08 22:04:08,253 | INFO     | queue_monitor       | pilot.control.job.4502006949     | get_data_structure        | payload/TRF did not report the number of read events
  File "/home/grid/slots/0/pilot2/pilot/common/exception.py", line 413, in run
    self._Thread__target(**self._Thread__kwargs)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 1676, in queue_monitor
    update_server(job, args)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 1718, in update_server
    send_state(job, args, job.state, xml=dumps(job.fileinfo), metadata=metadata)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 226, in send_state
    data = get_data_structure(job, state, args, xml=xml, metadata=metadata)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 491, in get_data_structure
    data['cpuConsumptionUnit'] = job.cpuconsumptionunit + "+" + get_cpu_model()
  File "/home/grid/slots/0/pilot2/pilot/util/workernode.py", line 186, in get_cpu_model
    with open("/proc/cpuinfo", "r") as f:
exception caught by thread run() function: (<type 'exceptions.IOError'>, IOError(2, 'No such file or directory'), <traceback object at 0x7fe78f293d88>)
Traceback (most recent call last):
  File "/home/grid/slots/0/pilot2/pilot/common/exception.py", line 413, in run
    self._Thread__target(**self._Thread__kwargs)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 1676, in queue_monitor
    update_server(job, args)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 1718, in update_server
    send_state(job, args, job.state, xml=dumps(job.fileinfo), metadata=metadata)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 226, in send_state
    data = get_data_structure(job, state, args, xml=xml, metadata=metadata)
  File "/home/grid/slots/0/pilot2/pilot/control/job.py", line 491, in get_data_structure
    data['cpuConsumptionUnit'] = job.cpuconsumptionunit + "+" + get_cpu_model()
  File "/home/grid/slots/0/pilot2/pilot/util/workernode.py", line 186, in get_cpu_model
    with open("/proc/cpuinfo", "r") as f:
IOError: [Errno 2] No such file or directory: '/proc/cpuinfo'

None
exception has been put in bucket queue belonging to thread 'queue_monitor'
setting graceful stop in 10 s since there is no point in continuing
2019-10-08 22:04:08,792 | WARNING  | job                 | pilot.control.job                | control                   | thread 'queue_monitor' received an exception from bucket: [Errno 2] No such file or directory: '/proc/cpuinfo'
2019-10-08 22:04:18,326 | INFO     | monitor             | pilot.control.monitor            | control                   | [monitor] control thread has ended
2019-10-08 22:04:18,330 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 15 threads
2019-10-08 22:04:18,330 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(job, started 140632491009792)>, <ExcThread(copytool_in, started 140631859001088)>, <ExcThread(failed_post, started 140631825430272)>, <ExcThread(retrieve, started 140632236476160)>, <ExcThread(validate_post, started 140631850608384)>, <ExcThread(execute_payloads, started 140631817037568)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(create_data_payload, started 140632253261568)>, <ExcThread(validate_pre, started 140632228083456)>, <ExcThread(payload, started 140632261654272)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(data, started 140632474224384)>, <ExcThread(validate, started 140632482617088)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:18,492 | DEBUG    | retrieve            | pilot.control.job                | retrieve                  | [job] retrieve thread has finished
2019-10-08 22:04:18,536 | DEBUG    | job                 | pilot.control.job                | control                   | job control ending since graceful_stop has been set
2019-10-08 22:04:18,537 | DEBUG    | job                 | pilot.control.job                | control                   | [job] control thread has finished
2019-10-08 22:04:18,697 | WARNING  | copytool_out        | pilot.util.common                | should_abort              | data:copytool_out:received graceful stop - abort after this iteration
2019-10-08 22:04:18,732 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 13 threads
2019-10-08 22:04:18,732 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(copytool_in, started 140631859001088)>, <ExcThread(failed_post, started 140631825430272)>, <ExcThread(validate_post, started 140631850608384)>, <ExcThread(execute_payloads, started 140631817037568)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(create_data_payload, started 140632253261568)>, <ExcThread(validate_pre, started 140632228083456)>, <ExcThread(payload, started 140632261654272)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(data, started 140632474224384)>, <ExcThread(validate, started 140632482617088)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:18,758 | INFO     | execute_payloads    | pilot.control.payload            | execute_payloads          | [payload] execute_payloads thread has finished
2019-10-08 22:04:18,759 | DEBUG    | create_data_payload | pilot.control.job                | create_data_payload       | [job] create_data_payload thread has finished
2019-10-08 22:04:18,890 | DEBUG    | validate            | pilot.control.job                | validate                  | [job] validate thread has finished
2019-10-08 22:04:18,898 | INFO     | failed_post         | pilot.control.payload            | failed_post               | [payload] failed_post thread has finished
2019-10-08 22:04:18,939 | DEBUG    | data                | pilot.control.data               | control                   | data control ending since graceful_stop has been set
2019-10-08 22:04:18,939 | DEBUG    | data                | pilot.control.data               | control                   | [data] control thread has finished
2019-10-08 22:04:19,033 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 8 threads
2019-10-08 22:04:19,034 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(copytool_in, started 140631859001088)>, <ExcThread(validate_post, started 140631850608384)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(validate_pre, started 140632228083456)>, <ExcThread(payload, started 140632261654272)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:19,124 | INFO     | validate_post       | pilot.control.payload            | validate_post             | [payload] validate_post thread has finished
2019-10-08 22:04:19,235 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 7 threads
2019-10-08 22:04:19,235 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(copytool_in, started 140631859001088)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(validate_pre, started 140632228083456)>, <ExcThread(payload, started 140632261654272)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:19,265 | DEBUG    | copytool_in         | pilot.control.data               | copytool_in               | [data] copytool_in thread has finished
2019-10-08 22:04:19,337 | DEBUG    | payload             | pilot.control.payload            | control                   | payload control ending since graceful_stop has been set
2019-10-08 22:04:19,337 | DEBUG    | payload             | pilot.control.payload            | control                   | [payload] control thread has finished
2019-10-08 22:04:19,338 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 5 threads
2019-10-08 22:04:19,338 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(validate_pre, started 140632228083456)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:19,491 | INFO     | validate_pre        | pilot.control.payload            | validate_pre              | [payload] validate_pre thread has finished
2019-10-08 22:04:19,539 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 4 threads
2019-10-08 22:04:19,540 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(queue_monitoring, started 140632219690752)>, <ExcThread(copytool_out, started 140632244868864)>]
2019-10-08 22:04:19,698 | DEBUG    | copytool_out        | pilot.control.data               | copytool_out              | [data] copytool_out thread has finished
2019-10-08 22:04:19,741 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 3 threads
2019-10-08 22:04:19,741 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(job_monitor, started 140631842215680)>, <ExcThread(queue_monitoring, started 140632219690752)>]
2019-10-08 22:04:21,195 | WARNING  | queue_monitoring    | pilot.util.common                | should_abort              | data:queue_monitoring:received graceful stop - abort after this iteration
2019-10-08 22:04:24,196 | DEBUG    | queue_monitoring    | pilot.control.data               | queue_monitoring          | [data] queue_monitor thread has finished
2019-10-08 22:04:24,273 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | thread count now at 2 threads
2019-10-08 22:04:24,274 | DEBUG    | MainThread          | pilot.workflow.generic           | run                       | enumerate: [<_MainThread(MainThread, started 140632652642112)>, <ExcThread(job_monitor, started 140631842215680)>]
2019-10-08 22:04:43,752 | INFO     | job_monitor         | pilot.util.auxiliary.4502006949  | job_monitor               | monitor loop #297: job 0:4502006949 is in state 'failed'
2019-10-08 22:04:43,752 | INFO     | job_monitor         | pilot.util.auxiliary.4502006949  | job_monitor               | aborting job monitoring since job state=failed
2019-10-08 22:04:43,752 | DEBUG    | job_monitor         | pilot.control.job                | job_monitor               | [job] job monitor thread has finished
2019-10-08 22:04:43,813 | INFO     | MainThread          | pilot.workflow.generic           | run                       | end of generic workflow (traces error code: 0)
2019-10-08 22:04:43,813 | INFO     | MainThread          | root                             | wrap_up                   | traces error code: 0
2019-10-08 22:04:43,813 | INFO     | MainThread          | root                             | wrap_up                   | pilot has finished
2019-10-08 22:04:43 UTC [wrapper] ==== pilot stdout END ====
2019-10-08 22:04:43 UTC [wrapper] ==== wrapper stdout RESUME ====
2019-10-08 22:04:43 UTC [wrapper] Pilot exit status: 0
2019-10-08 22:04:43 UTC [wrapper] STATUSCODE: 0
2019-10-08 22:04:43 UTC [wrapper] apfmon messages muted
---- find pandaIDs.out ----
total 56
-rw-------.  1 grid grid 11357 Jul 25 16:38 LICENSE
-rw-------.  1 grid grid    20 Sep  9 13:04 MANIFEST.IN
-rw-------.  1 grid grid    11 Oct  8 17:12 pandaIDs.out
drwx------. 14 grid grid   216 Oct  8 17:12 pilot
-rwx------.  1 grid grid 20136 Sep  9 13:04 pilot.py
-rw-------.  1 grid grid     9 Sep  9 13:04 PILOTVERSION
-rw-------.  1 grid grid  2251 Jul 25 16:38 README.md
-rw-------.  1 grid grid   760 Aug 22 11:01 setup.py
-rw-------.  1 grid grid   221 Jul 25 16:38 TODO.md

2019-10-08 22:04:43 UTC [wrapper] pandaIDs.out files:
-rw-------. 1 grid grid 11 Oct  8 17:12 /home/grid/slots/0/pilot2/pandaIDs.out
2019-10-08 22:04:43 UTC [wrapper] pandaIDs.out content:
4502006949

2019-10-08 22:04:43 UTC [wrapper] Test setup, not cleaning
2019-10-08 22:04:43 UTC [wrapper] ==== wrapper stdout END ====
2019-10-08 22:04:43 UTC [wrapper] ==== wrapper stderr END ====
2019-10-08 22:04:43 UTC [wrapper] wrapper wrapperexiting ec=0, duration=24774
2019-10-08 22:04:43 UTC [wrapper] apfmon messages muted
2019-10-09 00:04:47,147: ***************diag file************
2019-10-09 00:04:47,147: runtimeenvironments=APPS/HEP/ATLAS-SITE;
Processors=1
WallTime=24774.72s
KernelTime=520.24s
UserTime=24239.55s
CPUUsage=99%
MaxResidentMemory=1907568kB
AverageResidentMemory=0kB
AverageTotalMemory=0kB
AverageUnsharedMemory=0kB
AverageUnsharedStack=0kB
AverageSharedMemory=0kB
PageSize=4096B
MajorPageFaults=12015
MinorPageFaults=16726152
Swaps=0
ForcedSwitches=2854759
WaitSwitches=10689901
Inputs=4185568
Outputs=192112
SocketReceived=0
SocketSent=0
Signals=0

nodename=David_Cameron@dcameron04.cern.ch
exitcode=0
2019-10-09 00:04:47,163: ******************************WorkDir***********************
2019-10-09 00:04:47,163: total 233436
drwxrwx--x. 8 grid grid      4096 Oct  9 00:04 .
drwxr-x--x. 6 grid grid        42 Sep 26 14:51 ..
-rw-------. 1 grid grid   7613799 Oct  8 17:12 agis_ddmendpoints.json
-rw-------. 1 grid grid   3935747 Oct  8 17:12 agis_schedconf.cvmfs.json
drwx------. 2 grid grid         6 Oct  8 17:11 .alrb
drwxr-xr-x. 3 grid grid        17 Oct  8 17:11 APPS
-rw-------. 1 grid grid       538 Oct  8 17:11 .asetup
-rw-------. 1 grid grid      4503 Oct  8 17:12 .asetup.save
drwx------. 2 grid grid         6 Oct  8 17:12 .asetup-sysbin_5616
-rw-rw-r--. 1 grid grid         0 Oct  8 17:11 boinc_lockfile
-rw-rw-r--. 1 grid grid      8192 Oct  9 00:04 boinc_mmap_file
-rw-rw-r--. 1 grid grid       537 Oct  9 00:03 boinc_task_state.xml
-rw-rw-r--. 1 grid grid 223648985 Oct  8 17:11 EVNT.18605806._000073.pool.root.1
-rw-------. 1 grid grid       607 Oct  8 23:51 heartbeat.json
-rw-rw-r--. 1 grid grid      5865 Oct  8 17:11 init_data.xml
-rw-rw-r--. 1 grid grid    267506 Oct  8 17:11 input.tar.gz
-rw-rw-r--. 1 grid grid       112 Oct  8 17:11 job.xml
-rw-------. 1 grid grid   1235641 Oct  9 00:04 log.19056345._005910.job.log.1
-rw-------. 1 grid grid    588279 Oct  9 00:04 log.19056345._005910.job.log.tgz.1
-rw-------. 1 grid grid       789 Oct  9 00:03 memory_monitor_summary.json
-rw-------. 1 grid grid       241 Oct  9 00:04 output.list
-rw-------. 1 grid grid      2866 Oct  7 16:42 pandaJobData.out
drwxrwx---. 5 grid grid      4096 Oct  9 00:04 PanDA_Pilot-4502006949
drwx------. 3 grid grid       229 Oct  8 17:12 pilot2
-rw-r--r--. 1 grid grid    259319 Oct  7 16:34 pilot2.tar.gz
-rw-------. 1 grid grid   1209967 Oct  9 00:04 pilotlog.txt
-rw-------. 1 grid grid       515 Oct  9 00:04 QrGKDm9OjbvnShfckohDCDFpABFKDmABFKDmykOXDmABFKDm45cmln.diag
-rw-r--r--. 1 grid grid      4480 Oct  7 16:43 queuedata.json
-rw-rw-r--. 1 grid grid       815 Oct  8 17:11 RTE.tar.gz
-rwxr-xr-x. 1 grid grid      7950 Oct  8 17:11 run_atlas
-rwx------. 1 grid grid     12762 Oct  7 16:43 runpilot2-wrapper.sh
-rw-rw-r--. 1 grid grid       692 Oct  9 00:04 runtime_log
-rw-rw-r--. 1 grid grid      6747 Oct  9 00:04 runtime_log.err
drwxrwx--x. 2 grid grid       107 Oct  9 00:04 shared
-rw-rw-r--. 1 grid grid      8688 Oct  8 17:11 start_atlas.sh
-rw-rw-r--. 1 grid grid     24315 Oct  9 00:04 stderr.txt
-rw-rw-r--. 1 grid grid       107 Oct  8 17:11 wrapper_26015_x86_64-pc-linux-gnu
-rw-rw-r--. 1 grid grid        28 Oct  9 00:04 wrapper_checkpoint.txt

2019-10-09 00:04:47,163: running start_atlas return value is 0
2019-10-09 00:04:47,163: Parent exit 0
2019-10-09 00:04:47,165: child process exit 0
00:04:48 (31039): run_atlas exited; CPU time 24252.057305
00:04:48 (31039): called boinc_finish(0)

</stderr_txt>
]]>


©2024 CERN