Hey, I ran into this error during training—just wondering if you’ve seen it before or have any idea what might be going on?
LLVM ERROR: pthread_join failed: Invalid argument
LLVM ERROR: pthread_join failed: Invalid argument
LLVM ERROR: pthread_join failed: Invalid argument
{'loss': 1.9891, 'grad_norm': 9.454951905087402, 'learning_rate': 4.9287719354932735e-06, 'epoch': 1.04}
[52%| | 7676/14778 [7:03:13<30:30:23, 15.46s/it]
[52%| | 7676/14778 [7:03:13<30:30:23, 15.46s/it]
[rank1]: Traceback (most recent call last):
[rank1]: File "/workspace/Gemma3-Finetune-master/src/training/train.py", line 225, in <module>
[rank1]: train()
[rank1]: File "/workspace/Gemma3-Finetune-master/src/training/train.py", line 198, in train
[rank1]: trainer.train(resume_from_checkpoint=True)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2245, in train
[rank1]: return inner_training_loop(
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2556, in _inner_training_loop
[rank1]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 3718, in training_step
[rank1]: loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 3783, in compute_loss
[rank1]: outputs = model(**inputs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
[rank1]: ret_val = func(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/runtime/engine.py", line 1987, in forward
[rank1]: loss = self.module(*inputs, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank1]: return inner()
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1790, in inner
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/workspace/Gemma3-Finetune-master/src/training/monkey_patch_forward.py", line 104, in gemma3_mixed_modality_forward_with_flce
[rank1]: outputs = self.language_model.model(
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank1]: return inner()
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1790, in inner
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/models/gemma3/modeling_gemma3.py", line 742, in forward
[rank1]: layer_outputs = self._gradient_checkpointing_func(
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/_compile.py", line 32, in inner
[rank1]: return disable_fn(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py", line 632, in _fn
[rank1]: return fn(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 489, in checkpoint
[rank1]: return CheckpointFunction.apply(function, preserve, *args)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py", line 575, in apply
[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc]
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 264, in forward
[rank1]: outputs = run_function(*args)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank1]: return inner()
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1790, in inner
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/models/gemma3/modeling_gemma3.py", line 445, in forward
[rank1]: hidden_states, self_attn_weights = self.self_attn(
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank1]: return inner()
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1790, in inner
[rank1]: result = forward_call(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/transformers/models/gemma3/modeling_gemma3.py", line 329, in forward
[rank1]: query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank1]: return inner()
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1779, in inner
[rank1]: args_result = hook(self, args)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
[rank1]: ret_val = func(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/runtime/zero/parameter_offload.py", line 292, in _pre_forward_module_hook
[rank1]: self.pre_sub_module_forward_function(module)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
[rank1]: return func(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/runtime/zero/parameter_offload.py", line 467, in pre_sub_module_forward_function
[rank1]: param_coordinator.fetch_sub_module(sub_module, forward=True)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py", line 632, in _fn
[rank1]: return fn(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
[rank1]: ret_val = func(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
[rank1]: return func(*args, **kwargs)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/deepspeed/runtime/zero/partitioned_param_coordinator.py", line 343, in fetch_sub_module
[rank1]: get_accelerator().current_stream().wait_stream(self.__allgather_stream)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/cuda/streams.py", line 69, in wait_stream
[rank1]: self.wait_event(stream.record_event())
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/cuda/streams.py", line 55, in wait_event
[rank1]: event.wait(self)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/cuda/streams.py", line 197, in wait
[rank1]: super().wait(stream)
[rank1]: File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler
[rank1]: _error_if_any_worker_fails()
[rank1]: RuntimeError: DataLoader worker (pid 1968601) is killed by signal: Aborted.
Hey, I ran into this error during training—just wondering if you’ve seen it before or have any idea what might be going on?