Untitled
plain_text
a month ago
4.2 kB
3
Indexable
Never
Traceback (most recent call last): File "/home/tin/Documents/Github/FractalMessagePassing/lightning_train.py", line 174, in <module> trainer.fit(lightning_model) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit call._call_and_handle_interrupt( File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt return trainer_fn(*args, **kwargs) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl self._run(model, ckpt_path=ckpt_path) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run results = self._run_stage() File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1016, in _run_stage self._run_sanity_check() File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1045, in _run_sanity_check val_loop.run() File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator return loop_run(self, *args, **kwargs) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 108, in run batch, batch_idx, dataloader_idx = next(data_fetcher) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 136, in __next__ self._fetch_next_batch(self.dataloader_iter) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 150, in _fetch_next_batch batch = next(iterator) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 284, in __next__ out = next(self._iterator) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 123, in __next__ out = next(self.iterators[0]) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 633, in __next__ data = self._next_data() File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1345, in _next_data return self._process_data(data) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1371, in _process_data data.reraise() File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/_utils.py", line 644, in reraise raise exception RuntimeError: Caught RuntimeError in DataLoader worker process 0. Original Traceback (most recent call last): File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop data = fetcher.fetch(index) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch return self.collate_fn(data) File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch_geometric/loader/dataloader.py", line 20, in __call__ return Batch.from_data_list(batch, self.follow_batch, File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch_geometric/data/batch.py", line 76, in from_data_list batch, slice_dict, inc_dict = collate( File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch_geometric/data/collate.py", line 85, in collate value, slices, incs = _collate(attr, values, data_list, stores, File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch_geometric/data/collate.py", line 147, in _collate storage = elem.untyped_storage()._new_shared( File "/home/tin/anaconda3/envs/Fractal/lib/python3.10/site-packages/torch/storage.py", line 220, in _new_shared return cls(size, device=device) RuntimeError: CUDA error: initialization error Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.