Traceback (most recent call last):
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 535, in run_task
gang.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 249, in run
sub_gang.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 148, in run
g.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 119, in run
p.start()
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/process.py", line 121, in start
self._popen = self._Popen(self)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 291, in _Popen
return Popen(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 35, in __init__
super().__init__(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 47, in _launch
reduction.dump(process_obj, buf)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'GangMemberJob.run.<locals>.start_process'
E0802 02:21:59.907000 203975 workflow_main.py:280] Exception thrown from workflow binary:
Traceback (most recent call last):
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 535, in run_task
gang.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 249, in run
sub_gang.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 148, in run
g.run()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 119, in run
p.start()
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/process.py", line 121, in start
self._popen = self._Popen(self)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/context.py", line 291, in _Popen
return Popen(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 35, in __init__
super().__init__(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/popen_forkserver.py", line 47, in _launch
reduction.dump(process_obj, buf)
File "/usr/local/fbcode/platform009/lib/python3.8/multiprocessing/reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'GangMemberJob.run.<locals>.start_process'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 276, in main
return DebugMode.run(lambda: cmd.handler(rest))
File "/tmp/jetter.3t94a9aj/fblearner/flow/util/debugmode.py", line 66, in run
return func()
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 276, in <lambda>
return DebugMode.run(lambda: cmd.handler(rest))
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow_main.py", line 160, in launch_workflow
return main(args)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 145, in main
bootstrap_workflow_launch(new_args, flow_server_port)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 158, in bootstrap_workflow_launch
workflow_run_id, scheduler_task_id = launch_workflow_run(args, flow_server_port)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/launch_workflow_run/entry.py", line 395, in launch_workflow_run
run_dag_compilation(cmd_args)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/workflow/entry.py", line 229, in run_dag_compilation
register_and_schedule_operators(operators, args, operator_entry)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/operator/entry.py", line 385, in register_and_schedule_operators
schedule_operators(args, worker_main)
File "/tmp/jetter.3t94a9aj/fblearner/flow/main/operator/entry.py", line 296, in schedule_operators
driver_client().schedule_operators_locally(
File "/tmp/jetter.3t94a9aj/fblearner/flow/py_workflow/lib/driver_client/thrift.py", line 637, in schedule_operators_locally
schedule_operators_locally(
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduling.py", line 897, in schedule_operators_locally
return schedule_operators(
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduling.py", line 804, in schedule_operators
scheduler.run_tasks()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/basescheduler.py", line 200, in run_tasks
self.run_task(task)
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 543, in run_task
gang.terminate()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 245, in terminate
sub_gang.terminate()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 198, in terminate
g.terminate()
File "/tmp/jetter.3t94a9aj/fblearner/flow/driver/scheduler/localscheduler.py", line 84, in terminate
if self.process.exitcode is None:
AttributeError: 'NoneType' object has no attribute 'exitcode'
I0802 02:21:59.910000 203975 error_reporting.py:301] Will format exception: 'NoneType' object has no attribute 'exitcode'
E0802 02:21:59.911000 203975 structured_diagnostic.py:120] Error log path is not set
I0802 02:21:59.911000 203975 workflow_main.py:292] Exiting with no retries exit code
I0802 02:22:01.412029 203975 ClientSingletonManager.cpp:99] Shutting down Manifold ClientSingletonManager
I0802 02:22:01.717000 203892 jetter.py:326] Done running [PosixPath('/tmp/jetter.3t94a9aj/invoke_xar_via_trampoline'), '--local', 'exec/bin/flow.py', 'mobile_vision.detectron2go.core.workflow.e2e_workflow', '--run_mode', '6', '--hyper_params', '/tmp/tmp_hyper_params_g7vdpxhs', '--name', 'mobile_face_landmark/jlp_fbnetv3/train', '--local_data_basedir', '/var/tmp/fblearner_flow_local_378951', '--driver_port', '44059', '--db_tier', 'sqlite:////dev/shm/fblearner_flow_ndhingra/mylocaldb.sqlite.1d2e50cacb6358