Untitled

mail@pastecode.io avatar
unknown
plain_text
3 years ago
6.9 kB
4
Indexable
Never
Traceback (most recent call last):
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 535, in run_task
    gang.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 249, in run
    sub_gang.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 148, in run
    g.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 119, in run
    p.start()
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 291, in _Popen
    return Popen(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 35, in __init__
    super().__init__(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 47, in _launch
    reduction.dump(process_obj, buf)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'GangMemberJob.run.<locals>.start_process'
E0802 02:21:59.907000 203975 workflow_main.py:280] Exception thrown from workflow binary:
Traceback (most recent call last):
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 535, in run_task
    gang.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 249, in run
    sub_gang.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 148, in run
    g.run()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 119, in run
    p.start()
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 291, in _Popen
    return Popen(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 35, in __init__
    super().__init__(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 47, in _launch
    reduction.dump(process_obj, buf)
  File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'GangMemberJob.run.<locals>.start_process'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 276, in main
    return DebugMode.run(lambda: cmd.handler(rest))
  File "/tmp/jetter.3t94a9aj/fblearner/flow/util/debugmode.py", line 66, in run
    return func()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 276, in <lambda>
    return DebugMode.run(lambda: cmd.handler(rest))
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 160, in launch_workflow
    return main(args)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 145, in main
    bootstrap_workflow_launch(new_args, flow_server_port)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 158, in bootstrap_workflow_launch
    workflow_run_id, scheduler_task_id = launch_workflow_run(args, flow_server_port)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 395, in launch_workflow_run
    run_dag_compilation(cmd_args)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow/entry.py", line 229, in run_dag_compilation
    register_and_schedule_operators(operators, args, operator_entry)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/operator/entry.py", line 385, in register_and_schedule_operators
    schedule_operators(args, worker_main)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/main/operator/entry.py", line 296, in schedule_operators
    driver_client().schedule_operators_locally(
  File "/tmp/jetter.3t94a9aj/fblearner/flow/py_workflow/lib/driver_client/thrift.py", line 637, in schedule_operators_locally
    schedule_operators_locally(
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduling.py", line 897, in schedule_operators_locally
    return schedule_operators(
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduling.py", line 804, in schedule_operators
    scheduler.run_tasks()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/basescheduler.py", line 200, in run_tasks
    self.run_task(task)
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 543, in run_task
    gang.terminate()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 245, in terminate
    sub_gang.terminate()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 198, in terminate
    g.terminate()
  File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 84, in terminate
    if self.process.exitcode is None:
AttributeError: 'NoneType' object has no attribute 'exitcode'
I0802 02:21:59.910000 203975 error_reporting.py:301] Will format exception: 'NoneType' object has no attribute 'exitcode'
E0802 02:21:59.911000 203975 structured_diagnostic.py:120] Error log path is not set
I0802 02:21:59.911000 203975 workflow_main.py:292] Exiting with no retries exit code
I0802 02:22:01.412029 203975 ClientSingletonManager.cpp:99] Shutting down Manifold ClientSingletonManager
I0802 02:22:01.717000 203892 jetter.py:326] Done running [PosixPath('/tmp/jetter.3t94a9aj/invoke_xar_via_trampoline'), '--local', 'exec/bin/flow.py', 'mobile_vision.detectron2go.core.workflow.e2e_workflow', '--run_mode', '6', '--hyper_params', '/tmp/tmp_hyper_params_g7vdpxhs', '--name', 'mobile_face_landmark/jlp_fbnetv3/train', '--local_data_basedir', '/var/tmp/fblearner_flow_local_378951', '--driver_port', '44059', '--db_tier', 'sqlite:////dev/shm/fblearner_flow_ndhingra/mylocaldb.sqlite.1d2e50cacb6358