Untitled

mail@pastecode.io avatarunknown
plain_text
a month ago
8.3 kB
1
Indexable
Never
InvalidArgumentError                      Traceback (most recent call last)
Cell In[12], line 8
      1 testing = data.load_data_v1(
      2     test_path,
      3     data_shape=data_shape,
      4     subset=subset,
      5     group_same_observations=True,
      6 )
      7 testing = testing.map(normalize_data)
----> 8 model.evaluate(
      9     testing,
     10     callbacks=[
     11         keras.callbacks.TensorBoard(
     12             log_dir=f'outputs/{exp_name}_{runtime}_1st_board',
     13         ),
     14     ])
     18 # Obtain performance value at different thresholds.

File ~/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/utils/traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     65 except Exception as e:  # pylint: disable=broad-except
     66   filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67   raise e.with_traceback(filtered_tb) from None
     68 finally:
     69   del filtered_tb

File ~/.conda/envs/tc_quan/lib/python3.10/site-packages/tensorflow/python/eager/execute.py:54, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     52 try:
     53   ctx.ensure_initialized()
---> 54   tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
     55                                       inputs, attrs, num_outputs)
     56 except core._NotOkStatusException as e:
     57   if name is not None:

InvalidArgumentError: Graph execution error:

Detected at node 'assert_greater_equal/Assert/AssertGuard/Assert' defined at (most recent call last):
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/runpy.py", line 196, in _run_module_as_main
      return _run_code(code, main_globals, None,
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/runpy.py", line 86, in _run_code
      exec(code, run_globals)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel_launcher.py", line 17, in <module>
      app.launch_new_instance()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/traitlets/config/application.py", line 1043, in launch_instance
      app.start()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/kernelapp.py", line 725, in start
      self.io_loop.start()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/tornado/platform/asyncio.py", line 215, in start
      self.asyncio_loop.run_forever()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
      self._run_once()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
      handle._run()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/asyncio/events.py", line 80, in _run
      self._context.run(self._callback, *self._args)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 513, in dispatch_queue
      await self.process_one()
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 502, in process_one
      await dispatch(*args)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 409, in dispatch_shell
      await result
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 729, in execute_request
      reply_content = await reply_content
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 422, in do_execute
      res = shell.run_cell(
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 540, in run_cell
      return super().run_cell(*args, **kwargs)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3006, in run_cell
      result = self._run_cell(
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3061, in _run_cell
      result = runner(coro)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
      coro.send(None)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3266, in run_cell_async
      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3445, in run_ast_nodes
      if await self.run_code(code, result, async_=asy):
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
      exec(code_obj, self.user_global_ns, self.user_ns)
    File "/tmp/ipykernel_17728/124998052.py", line 2, in <module>
      first_stage_history = model.fit(
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1445, in fit
      val_logs = self.evaluate(
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1756, in evaluate
      tmp_logs = self.test_function(iterator)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1557, in test_function
      return step_function(self, iterator)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1546, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1535, in run_step
      outputs = model.test_step(data)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 1502, in test_step
      return self.compute_metrics(x, y, y_pred, sample_weight)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/training.py", line 987, in compute_metrics
      self.compiled_metrics.update_state(y, y_pred, sample_weight)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/engine/compile_utils.py", line 501, in update_state
      metric_obj.update_state(y_t, y_p, sample_weight=mask)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/utils/metrics_utils.py", line 70, in decorated
      update_op = update_state_fn(*args, **kwargs)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/metrics/base_metric.py", line 140, in update_state_fn
      return ag_update_state(*args, **kwargs)
    File "/geode2/home/u100/trihnguy/Carbonate/tc-prediction-tri/tc_formation/tf_metrics.py", line 17, in update_state
      return super().update_state(y_true, y_pred, sample_weight)
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/metrics/metrics.py", line 949, in update_state
      return metrics_utils.update_confusion_matrix_variables(
    File "/N/u/trihnguy/Carbonate/.conda/envs/tc_quan/lib/python3.10/site-packages/keras/utils/metrics_utils.py", line 602, in update_confusion_matrix_variables
      tf.debugging.assert_greater_equal(
Node: 'assert_greater_equal/Assert/AssertGuard/Assert'
assertion failed: [predictions must be >= 0] [Condition x >= y did not hold element-wise:] [x (Sigmoid:0) = ] [[0.0195365865][-nan][0.0402007438]...] [y (Cast_8/x:0) = ] [0]
	 [[{{node assert_greater_equal/Assert/AssertGuard/Assert}}]] [Op:__inference_test_function_8259]