{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "a0863295", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 9, "id": "cccf4b4c", "metadata": {}, "outputs": [], "source": [ "import contextlib\n", "import time\n", "import random\n", "\n", "import tqdm\n", "import numpy as np\n", "import torch\n", "import nemo.collections.asr as nemo_asr\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.display import suppress_logging\n", "from suno_utils.utils.text import normalize_whitespace\n", "from suno_utils.ctcdecode.decoder import build_ctcdecoder\n", "\n", "if (\n", " torch.cuda.is_available()\n", " and hasattr(torch.cuda, 'amp')\n", " and hasattr(torch.cuda.amp, 'autocast')\n", "):\n", " autocast = torch.cuda.amp.autocast\n", "else:\n", " @contextlib.contextmanager\n", " def autocast():\n", " yield\n", " \n", "SAMPLE_RATE = 16_000\n", " \n", "def gpu_stats(clear_cache=True):\n", " if clear_cache:\n", " torch.cuda.empty_cache()\n", " from pynvml import nvmlInit, nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo\n", " t = torch.cuda.get_device_properties(0).total_memory\n", " r = torch.cuda.memory_reserved(0)\n", " a = torch.cuda.memory_allocated(0)\n", " print(\"torch:\")\n", " print(round(t / 1e9, 1), \"Gb total\")\n", " print(round(r / 1e9, 1), \"Gb reserved\")\n", " print(round(a / 1e9, 1), \"Gb allocated\")\n", " print()\n", " nvmlInit()\n", " h = nvmlDeviceGetHandleByIndex(0)\n", " info = nvmlDeviceGetMemoryInfo(h)\n", " print(\"nvidia:\")\n", " print(round(info.total / 1e9, 1), \"Gb total\")\n", " print(round(info.used / 1e9, 1), \"Gb used\")\n", " print(round(info.free / 1e9, 1), \"Gb free\") \n", " \n", "def load_model(model_name=\"stt_en_conformer_ctc_large\"):\n", " with suppress_logging():\n", " if model_name[0] == \"/\":\n", " model = nemo_asr.models.EncDecCTCModelBPE.load_from_checkpoint(model_name)\n", " else:\n", " model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name=model_name)\n", " model.preprocessor.featurizer.dither = 0.0\n", " model.preprocessor.featurizer.pad_to = 0\n", " model.eval()\n", " model.encoder.freeze()\n", " model.decoder.freeze()\n", " if torch.cuda.is_available():\n", " torch.cuda.synchronize()\n", " torch.cuda.empty_cache()\n", " return model\n", "\n", "def get_tokens_from_logits(vocab, logits):\n", " probs = np.exp(logits)\n", " argmax_preds = probs.argmax(axis=1)\n", " tokens = [vocab[idx] for idx in argmax_preds]\n", " return tokens\n", "\n", "def get_text_from_tokens(tokens):\n", " squashed_tokens = []\n", " prev_token = None\n", " for token in tokens:\n", " if token != prev_token:\n", " squashed_tokens.append(token)\n", " prev_token = token\n", " text = \"\".join(squashed_tokens).replace(\"▁\", \" \")\n", " text = normalize_whitespace(text)\n", " return text\n", "\n", "def decode_logits(vocab, logits):\n", " tokens = get_tokens_from_logits(vocab, logits)\n", " text = get_text_from_tokens(tokens)\n", " return text\n", "\n", "def _collate_features(batch, pad_id=0):\n", " \"\"\"collate batch of audio sig, audio len\"\"\"\n", " packed_batch = list(zip(*batch))\n", " if len(packed_batch) == 2:\n", " _, audio_lengths = packed_batch\n", " else:\n", " raise ValueError(\"Expects 2 tensors in the batch!\")\n", " max_audio_len = 0\n", " has_audio = audio_lengths[0] is not None\n", " if not has_audio:\n", " return None, None\n", " max_audio_len = max(audio_lengths).item()\n", " audio_signal = []\n", " for sig, sig_len in batch:\n", " sig_len = sig_len.item()\n", " if sig_len < max_audio_len:\n", " pad = (0, int(max_audio_len - sig_len))\n", " sig = torch.nn.functional.pad(sig, pad)\n", " audio_signal.append(sig)\n", " audio_signal = torch.stack(audio_signal)\n", " audio_lengths = torch.stack(audio_lengths)\n", " return audio_signal, audio_lengths\n", "\n", "def _get_features(model, audio_signal, audio_lengths):\n", " processed_signal, processed_signal_length = model.preprocessor(\n", " input_signal=audio_signal, length=audio_lengths,\n", " )\n", " return processed_signal, processed_signal_length\n", "\n", "def _infer(model, processed_signal, processed_signal_length):\n", " logits, logits_len, labels = model.forward(\n", " processed_signal=processed_signal, processed_signal_length=processed_signal_length,\n", " )\n", " return logits, logits_len, labels\n", "\n", "MIN_PREDICT_DURATION_MS = 10 # this is to avoid model errors\n", "MAX_PREDICT_DURATION_MS = 60_100 # this is to avoid gpu oom\n", "\n", "def predict_logits(model, audio_arr_list):\n", " if len(audio_arr_list) == 0:\n", " return []\n", " # alert if contains an array that is too short or too long\n", " if any([\n", " (\n", " arr.shape[0] < int(MIN_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE) or \n", " arr.shape[0] > int(MAX_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE)\n", " ) for arr in audio_arr_list\n", " ]):\n", " raise ValueError(\"array size error\")\n", " with torch.inference_mode(), torch.no_grad(), autocast():\n", " arr_list_gpu = [torch.from_numpy(arr).to(model.device) for arr in audio_arr_list]\n", " arr_len_list_gpu = [torch.tensor(arr.shape[-1]).to(model.device) for arr in audio_arr_list]\n", " batch = list(zip(arr_list_gpu, arr_len_list_gpu))\n", " audio_signal, audio_lengths = _collate_features(batch)\n", " processed_signal, processed_signal_length = _get_features(model, audio_signal, audio_lengths)\n", " logits, logits_len, labels = _infer(model, processed_signal, processed_signal_length)\n", " logits_list = [l[:idx].detach().cpu().numpy().squeeze() for idx, l in zip(logits_len, logits)]\n", " # labels_list = [l[:idx].detach().cpu().numpy().squeeze() for idx, l in zip(logits_len, labels)]\n", " del (\n", " arr_list_gpu, arr_len_list_gpu, batch, audio_signal, audio_lengths, \n", " processed_signal, processed_signal_length, logits, logits_len, labels\n", " )\n", " if torch.cuda.is_available():\n", " torch.cuda.synchronize()\n", " return logits_list\n", "\n", "def _find_break_idx(tokens):\n", " best_idx = None\n", " # break at last new word\n", " for n, t in enumerate(tokens[::-1]):\n", " if t.startswith(\"▁\"):\n", " best_idx = len(tokens) - n - 1\n", " break\n", " # if nothing found then break at an early blank\n", " for n, t in enumerate(tokens[-5:]):\n", " if t == \"\":\n", " best_idx = len(tokens) + n - 5\n", " break\n", " # if nothing found then hard break\n", " if best_idx is None:\n", " best_idx = max(0, len(tokens) - 5)\n", " return best_idx\n", "\n", "def _find_break_idx_from_guess(tokens, idx_guess, decoded_words=None):\n", " best_idx = idx_guess\n", " all_break_idx = [n for n, t in enumerate(tokens) if t.startswith(\"▁\")]\n", " if len(all_break_idx) == 0:\n", " return best_idx\n", " # find break index that is closest to use as a best guess\n", " best_idx = all_break_idx[np.argsort([np.abs(idx - best_idx) for idx in all_break_idx])[0]]\n", " # refine index in word space\n", " if decoded_words is None:\n", " decoded_words = []\n", " if len(decoded_words) == 0:\n", " best_idx = 0\n", " return best_idx\n", " # user words to finetune selection\n", " # if we already did a good job then return\n", " discarded_words = get_text_from_tokens(tokens[:best_idx]).split()\n", " if len(discarded_words) > 0 and discarded_words[-1] == decoded_words[-1]:\n", " return best_idx\n", " # check if there is word overlap (up to 2)\n", " later_break_idx = [idx for idx in all_break_idx if idx > best_idx][:2]\n", " for break_idx in later_break_idx[::-1]:\n", " extra_words = get_text_from_tokens(tokens[best_idx:break_idx]).split()\n", " if extra_words == decoded_words[-len(extra_words):]:\n", " return break_idx\n", " # check earlier break index incase we missed words (up to 2)\n", " earlier_break_idx = [idx for idx in all_break_idx if idx < best_idx][-2:]\n", " for break_idx in earlier_break_idx[::-1]:\n", " discarded_words = get_text_from_tokens(tokens[:break_idx]).split()\n", " if discarded_words == decoded_words[-len(discarded_words):]:\n", " return break_idx\n", " return best_idx" ] }, { "cell_type": "code", "execution_count": 4, "id": "2af4ba6d", "metadata": {}, "outputs": [], "source": [ "# notebook visualization stuff\n", "import html\n", "from IPython.display import display, HTML, Javascript\n", "\n", "def display_boxes(n_users):\n", " for n_user in range(n_users):\n", " display(HTML(\"User \" + str(n_user) + \":
\"))\n", " \n", "def display_text(n_user, text):\n", " display(Javascript(\n", " \"var el = document.getElementsByClassName('asr_user_\" + str(n_user) + \"');\"\n", " \"for (var i = 0; i < el.length; ++i) {el[i].innerHTML = '\" + html.escape(text) + \"';}\"\n", " ))" ] }, { "cell_type": "code", "execution_count": 5, "id": "930ff6ad", "metadata": {}, "outputs": [], "source": [ "import os\n", "import tempfile\n", "from ipywebrtc import AudioRecorder, CameraStream\n", "\n", "\n", "def transcribe_recorder(recorder, model, vocab):\n", " with tempfile.TemporaryDirectory() as tmp_dir:\n", " tmp_audio_fp = os.path.join(tmp_dir, \"audio.webm\")\n", " with open(tmp_audio_fp, \"wb\") as f:\n", " f.write(recorder.audio.value)\n", " recorded_audio = Audio.from_file(tmp_audio_fp, sample_rate=16_000, byte_width=2)\n", " audio_arr_list = [recorded_audio.array_float]\n", " logits_list = predict_logits(model, audio_arr_list)\n", " text = decode_logits(vocab, logits_list[0])\n", " return text\n", "\n", "camera_stream = CameraStream(constraints={\"audio\": True, \"video\": False})" ] }, { "cell_type": "code", "execution_count": 6, "id": "ffd965dd", "metadata": {}, "outputs": [], "source": [ "ckpt_dir = \"/mnt/data-ssd-2/nemo_training/checkpoints/Conformer-CTC-BPE/2022-11-26_22-56-06/checkpoints/\"\n", "ckpt_fp = ckpt_dir + \"Conformer-CTC-BPE--val_wer=0.1798-epoch=2.ckpt\"\n", "asr_model = load_model(ckpt_fp)\n", "asr_vocab = {n: c for n, c in enumerate(list(asr_model.decoder.vocabulary) + [\"\"])}\n", "russia_audio = Audio.from_file(\"/home/georg/data/sample_audio/russia.wav\", sample_rate=16_000, byte_width=2)" ] }, { "cell_type": "code", "execution_count": 10, "id": "84bf3809", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'President Joe Biden, backed by the full symbolic power of the Western Alliance, is locked in a showdown with Russian President of Vladimir Putin who is using Ukraine as a hostage to try to force the US. to renegotiate the settled outcome of the Cold War.'" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "audio_arr_list = [russia_audio.array_float]\n", "logits_list = predict_logits(asr_model, audio_arr_list)\n", "decode_logits(asr_vocab, logits_list[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "1beb15ca", "metadata": {}, "outputs": [], "source": [ "# recorder = AudioRecorder(stream=camera_stream)\n", "# recorder" ] }, { "cell_type": "code", "execution_count": null, "id": "01487b19", "metadata": {}, "outputs": [], "source": [ "# transcribe_recorder(recorder, model=asr_model, vocab=asr_vocab)" ] }, { "cell_type": "code", "execution_count": null, "id": "70a59748", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d9134d35", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "dadc2a69", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 16, "id": "e9c69135", "metadata": {}, "outputs": [], "source": [ "# import gradio as gr\n", "# gr.close_all()\n", "# time.sleep(10)" ] }, { "cell_type": "code", "execution_count": 12, "id": "4eac774d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running on local URL: http://127.0.0.1:7860\n", "\n", "To create a public link, set `share=True` in `launch()`.\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 11:00:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:20 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:22 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:24 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:26 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 11:00:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:16:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:17:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:17:20 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:20 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:22 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:24 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:26:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:26:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:34 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:38 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:27:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:27:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:28:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:42 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:44 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:28:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:29:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:29:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:30:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:30:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:30:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:20 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:26 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:28 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:30 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:34 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:38 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:31:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:31:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:32:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:38 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:40 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:42 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:44 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:32:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:33:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:30 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:34 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:33:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:34:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:34:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:35:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:12 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:18 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:20 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:22 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:38 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:40 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:42 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:44 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:35:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:35:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:01 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:24 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:26 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:36:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:36:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:14 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:16 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:37:30 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:34 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:38 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:40 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:42 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:44 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:37:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:38:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:38:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:03 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:05 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:07 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:09 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:39:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:26 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:28 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:30 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:39:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:40:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:28 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:30 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:32 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:34 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:36 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:47 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:40:49 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:51 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:53 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:55 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:57 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:40:59 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:13 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:15 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:17 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:19 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:21 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:23 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:25 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:27 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:29 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:31 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:33 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:35 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:37 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-12-07 16:41:39 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:41 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:43 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:45 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:46 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:48 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:50 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:52 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:54 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:56 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:41:58 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:00 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:02 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:04 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:06 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:08 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:10 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n", "[NeMo W 2022-12-07 16:42:11 nemo_logging:349] /home/georg/venvs/ml/lib/python3.8/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n", " warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n", " \n" ] } ], "source": [ "import gradio as gr\n", "import time\n", "\n", "max_context = 16000*10\n", "def gradio_transcribe(audio_fp, state=None):\n", " time.sleep(0.5)\n", " recorded_audio = Audio.from_file(audio_fp, sample_rate=16_000, byte_width=2)\n", " arr = recorded_audio.array_float\n", " if state is not None:\n", " arr = np.hstack([state, arr])\n", " logits_list = predict_logits(asr_model, [arr[-max_context:]])\n", " text = decode_logits(asr_vocab, logits_list[0])\n", " return text, arr\n", "\n", "# with gr.Blocks(css='.gradio-container {font-family: monospace !important; }')\n", "gradio_interface = gr.Interface(\n", " fn=gradio_transcribe, \n", " inputs=[\n", " gr.Audio(source=\"microphone\", type=\"filepath\", streaming=True), \n", " \"state\"\n", " ],\n", " outputs=[\n", " \"textbox\",\n", " \"state\"\n", " ],\n", " live=True)\n", "gradio_interface.launch(server_port=7860)" ] }, { "cell_type": "code", "execution_count": null, "id": "685ceebc", "metadata": {}, "outputs": [], "source": [ "# multi/single model?\n", "\n", "# transcribe english\n", "# translate multilingual\n", "# transducer?\n", "# stochastic train data?\n", "# probability\n", "# emotion tag\n", "# val/arousal - high/low\n", "# tags (speaker change, applause, music, cough, laughter, hesitation, ...)\n", "# transcribe music" ] }, { "cell_type": "code", "execution_count": null, "id": "3b29874c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ca4af53b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6d4e13ad", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "87467ee6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a3dc982a", "metadata": {}, "source": [ "## Run transcribing in background" ] }, { "cell_type": "code", "execution_count": 35, "id": "eee101fa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n", "A LLO maybe you don't have one. have What?ecause my mom had it.\n" ] } ], "source": [ "import time\n", "STEP_DURATION_MS = 500\n", "t0 = time.time()\n", "n_step = 0\n", "for _ in range(10):\n", " print(transcribe_recorder(recorder, asr_model, asr_vocab))\n", " sleep_duration_s = STEP_DURATION_MS / 1_000 * (n_step + 1) - (time.time() - t0)\n", " if sleep_duration_s > 0.01:\n", " time.sleep(sleep_duration_s)\n", " n_step += 1" ] }, { "cell_type": "code", "execution_count": null, "id": "d109ae15", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8996e099", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "79582146", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4972ec54", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "22ae07fb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "cba0c46d", "metadata": {}, "source": [ "## Realtime" ] }, { "cell_type": "markdown", "id": "929bcabc", "metadata": {}, "source": [ "## Sliding window inference (single user)" ] }, { "cell_type": "code", "execution_count": 364, "id": "572f62e0", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "president joe biden backed by the full symbolic power of the western alliance is locked in a showdown with the russian president of vladimir putin who is using ukraine as a hostage to try to force the u s to renegotiate the settled outcome of the cold war\r" ] } ], "source": [ "# we are pretending that user upload the audio in USER_AUDIO_ARRAYS\n", "# they do so continuously such that we can realtime process chunks of STEP_DURATION_MS\n", "# end of stream is known here but needs to be communicated separately in a real scenario\n", "STEP_DURATION_MS = 500\n", "MAX_CONTEXT_DURATION_S = 5\n", "RESPECT_REALTIME = True\n", "\n", "USER_AUDIO_ARRAY = USER_AUDIO_ARRAYS[0]\n", "\n", "# caluclate some basics\n", "step_n_array = int(STEP_DURATION_MS / 1_000 * SAMPLE_RATE)\n", "n_array_per_logit = 8000 / 13 # TODO: is there a better estimate of this?\n", "max_context_n_array = int(MAX_CONTEXT_DURATION_S * SAMPLE_RATE)\n", "\n", "# store user info\n", "user_words = []\n", "user_prev_audio_start_idx = 0\n", "user_prev_audio_end_idx = 0\n", "user_prev_token_end_idx = 0\n", "\n", "n_step = 0\n", "t0 = time.time()\n", "while True:\n", " # sleep if necessary to simulate realtime\n", " if RESPECT_REALTIME:\n", " sleep_duration_s = STEP_DURATION_MS / 1_000 * (n_step + 1) - (time.time() - t0)\n", " if sleep_duration_s > 0.01:\n", " time.sleep(sleep_duration_s)\n", " # get audio chunk to predict\n", " audio_end_idx = min(USER_AUDIO_ARRAY.shape[0], (n_step + 1) * step_n_array)\n", " audio_start_idx = max(0, audio_end_idx - max_context_n_array)\n", " # we are done if audio too small for prediction\n", " if audio_array_segment.shape[0] < int(MIN_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE):\n", " break\n", " # do prediction\n", " logits = predict_logits(model, [audio_array_segment])[0]\n", " tokens = get_tokens_from_logits(model, logits)\n", " # get best guess for token_start_idx based on what has already been decoded\n", " token_start_idx = user_prev_token_end_idx - int(round(\n", " (audio_start_idx - user_prev_audio_start_idx) / n_array_per_logit\n", " ))\n", " token_start_idx = _find_break_idx_from_guess(tokens, token_start_idx, decoded_words=user_words)\n", " # end-of-stream signal implicitly give through end of array\n", " if audio_end_idx == USER_AUDIO_ARRAY.shape[0]:\n", " user_words.extend(get_text_from_tokens(tokens[token_start_idx:]).split())\n", " break\n", " # find reliable token end index\n", " token_end_idx = token_start_idx + _find_break_idx(tokens[token_start_idx:])\n", " # add tokens to user stack\n", " user_words.extend(get_text_from_tokens(tokens[token_start_idx:token_end_idx]).split()) \n", " # display results for user\n", " output_text = \" \".join([\n", " w \n", " for w in user_words + get_text_from_tokens(tokens[token_end_idx:]).split() \n", " if len(w) > 0\n", " ])\n", " if len(output_text) == 0:\n", " output_text = \" \"\n", " print(output_text, end=\"\\r\")\n", " # prepare for next step\n", " user_prev_audio_start_idx = audio_start_idx\n", " user_prev_audio_end_idx = audio_end_idx\n", " user_prev_token_end_idx = token_end_idx\n", " n_step += 1\n", "# TODO: improve the above with 'partial' vs 'final' results" ] }, { "cell_type": "markdown", "id": "b4758f4a", "metadata": {}, "source": [ "## Sliding window inference (multi user)" ] }, { "cell_type": "code", "execution_count": null, "id": "ac4f1a48", "metadata": {}, "outputs": [], "source": [ "# we are pretending that user upload the audio in USER_AUDIO_ARRAYS\n", "# they do so continuously such that we can realtime process chunks of STEP_DURATION_MS\n", "# end of stream is known here but needs to be communicated separately in a real scenario\n", "STEP_DURATION_MS = 500\n", "MAX_CONTEXT_DURATION_S = 5\n", "RESPECT_REALTIME = True\n", "\n", "# caluclate some basics\n", "step_n_array = int(STEP_DURATION_MS / 1_000 * SAMPLE_RATE)\n", "n_array_per_logit = 8000 / 13 # TODO: is there a better estimate of this?\n", "max_context_n_array = int(MAX_CONTEXT_DURATION_S * SAMPLE_RATE)\n", "\n", "# store user info\n", "user_data = [\n", " {\n", " \"words\": [],\n", " \"prev_audio_start_idx\": 0,\n", " \"prev_audio_end_idx\": 0,\n", " \"prev_token_end_idx\": 0,\n", " \"is_done\": False\n", " } for _ in range(len(USER_AUDIO_ARRAYS))\n", "]\n", "\n", "display_boxes(len(USER_AUDIO_ARRAYS))\n", "\n", "n_step = 0\n", "t0 = time.time()\n", "while True:\n", " # sleep if necessary to simulate realtime\n", " if RESPECT_REALTIME:\n", " sleep_duration_s = STEP_DURATION_MS / 1_000 * (n_step + 1) - (time.time() - t0)\n", " if sleep_duration_s > 0.01:\n", " time.sleep(sleep_duration_s)\n", " # get audio chunks to predict in this batch\n", " # TODO: limit this to a certain batchsize and do triaging for which users to serve\n", " batch = []\n", " for user_idx, audio_array in enumerate(USER_AUDIO_ARRAYS):\n", " if user_data[user_idx][\"is_done\"]:\n", " continue\n", " audio_end_idx = min(audio_array.shape[0], (n_step + 1) * step_n_array)\n", " # end-of-stream signal implicitly give through end of array\n", " is_final_pred = audio_end_idx == audio_array.shape[0]\n", " audio_start_idx = min(audio_array.shape[0], max(0, n_step * step_n_array - max_context_n_array))\n", " audio_array_segment = audio_array[audio_start_idx:audio_end_idx]\n", " # we are done if audio too small for prediction\n", " if audio_array_segment.shape[0] < int(MIN_PREDICT_DURATION_MS / 1_000 * SAMPLE_RATE):\n", " continue\n", " batch.append({\n", " \"audio_array\": audio_array_segment,\n", " \"user_idx\": user_idx,\n", " \"audio_start_idx\": audio_start_idx,\n", " \"audio_end_idx\": audio_end_idx,\n", " \"is_final_pred\": is_final_pred,\n", " })\n", " if len(batch) == 0:\n", " break\n", " # do prediction\n", " logits_list = predict_logits(model, [m[\"audio_array\"] for m in batch])\n", " for m, logits in zip(batch, logits_list):\n", " user_idx = m[\"user_idx\"]\n", " audio_start_idx = m[\"audio_start_idx\"]\n", " audio_end_idx = m[\"audio_end_idx\"]\n", " is_final_pred = m[\"is_final_pred\"]\n", " prev_audio_start_idx = user_data[user_idx][\"prev_audio_start_idx\"]\n", " prev_token_end_idx = user_data[user_idx][\"prev_token_end_idx\"]\n", " # decode tokens\n", " tokens = get_tokens_from_logits(model, logits)\n", " # get best guess for token_start_idx based on what has already been decoded\n", " token_start_idx = prev_token_end_idx - int(round(\n", " (audio_start_idx - prev_audio_start_idx) / n_array_per_logit\n", " ))\n", " token_start_idx = _find_break_idx_from_guess(\n", " tokens, token_start_idx, \n", " decoded_words=user_data[user_idx][\"words\"],\n", " )\n", " # end-of-stream signal implicitly give through end of array\n", " if is_final_pred:\n", " user_data[user_idx][\"words\"].extend(get_text_from_tokens(tokens[token_start_idx:]).split())\n", " user_data[user_idx][\"is_done\"] = True\n", " continue\n", " # find reliable token end index\n", " token_end_idx = token_start_idx + _find_break_idx(tokens[token_start_idx:])\n", " # add tokens to user stack\n", " user_data[user_idx][\"words\"].extend(get_text_from_tokens(tokens[token_start_idx:token_end_idx]).split()) \n", " # prepare for next step\n", " user_data[user_idx][\"prev_audio_start_idx\"] = audio_start_idx\n", " user_data[user_idx][\"prev_audio_end_idx\"] = audio_end_idx\n", " user_data[user_idx][\"prev_token_end_idx\"] = token_end_idx\n", " # display results for user\n", " output_text = \" \".join([\n", " w \n", " for w in user_data[user_idx][\"words\"] + get_text_from_tokens(tokens[token_end_idx:]).split() \n", " if len(w) > 0\n", " ])\n", " display_text(user_idx, output_text)\n", " n_step += 1" ] }, { "cell_type": "code", "execution_count": null, "id": "b2c0b475", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "784da302", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9af7bc0a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3fa8f8c2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fb348fce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d35edee1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8862c3a4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4279a35c", "metadata": {}, "source": [ "## TODO:" ] }, { "cell_type": "code", "execution_count": null, "id": "cb1363ea", "metadata": {}, "outputs": [], "source": [ "# -- short term --\n", "# TODO: reduce context to 2-3s?\n", "# TODO: implement 'partial' and 'final' (probably final until one back and next one becomes partial)\n", "# TODO: add punctuation for final\n", "# TODO: add pyctcdecode for final (beam 5 and hotwords, no LM?)\n", "# TODO: hotwords format (warn if not normalized?)\n", "\n", "# -- medium term --\n", "# TODO: look at max gpu memory consumption\n", "# https://stackoverflow.com/questions/58216000/get-total-amount-of-free-gpu-memory-and-available-using-pytorch\n", "# TODO: add diarization for final\n", "# TODO: how do we deal with 2-channel, esp diarize\n", "# TODO: implement async batch transcript\n", "\n", "# -- long term --\n", "# TODO: use vad to avoid partial word errors at end?\n", "# TODO: consider what to do about normalization: model.cfg.preprocessor.normalize\n", "# TODO: ask nvidia for causal conformer, squeezeformer and LM\n", "# TODO: test transducer\n", "# TODO: implement cache aware for speed" ] }, { "cell_type": "code", "execution_count": null, "id": "33e6051e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b4c88258", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d6f6cc14", "metadata": {}, "source": [ "## test accuracy" ] }, { "cell_type": "code", "execution_count": 8, "id": "a3aeade8", "metadata": {}, "outputs": [], "source": [ "import re\n", "import json\n", "import funcy\n", "\n", "from suno_utils.audio import Tokens\n", "from suno_utils.utils.metrics import get_wer_bulk, get_cer_bulk\n", "\n", "data_dir = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/to_sanas/2022-08-12\"\n", "# data_dir = \"/mnt/data-ssd-1/data/private/customer/sanas/2022-08-04-fili-callcenter/to_sanas/2022-08-15\"\n", "\n", "dev_meta = []\n", "with open(os.path.join(data_dir, \"segment_metadata.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " # only allowed meta tag is [laughter] and hesitations need to be removed\n", " text = m[\"transcript\"][\"text\"]\n", " text = re.sub(r\"\\s*\\-\\-\\s*\", \" \", text)\n", " text = re.sub(r\"\\s*\\[laughter\\]\\s*\", \" \", text)\n", " text = normalize_whitespace(text)\n", " if \"[\" in text:\n", " continue\n", " dev_meta.append({\n", " \"duration_s\": m[\"duration_s\"],\n", " \"filepath\": os.path.join(data_dir, m[\"uri\"]),\n", " \"text\": text,\n", " \"text_norm\": Tokens.from_dict(m[\"transcript_normalized\"][\"tokens\"]).plaintext,\n", " })\n", "print(len(dev_meta), \"files loaded\")\n", "print(round(np.sum([m[\"duration_s\"] for m in dev_meta]) / 60 / 60, 1), \"hours\")" ] }, { "cell_type": "code", "execution_count": 10, "id": "ecb7eb53", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 942/942 [02:59<00:00, 5.24it/s]\n" ] } ], "source": [ "greedy_preds = []\n", "for chunk_meta in tqdm.tqdm(funcy.chunks(16, dev_meta), total=int(np.ceil(len(dev_meta) / 16))):\n", " audio_arr_list = [Audio.from_file(m[\"filepath\"]).array_float for m in chunk_meta]\n", " logits_list = predict_logits(model, audio_arr_list)\n", " greedy_preds.extend([decode_logits(model, logits) for logits in logits_list])" ] }, { "cell_type": "code", "execution_count": 11, "id": "2912dfb4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20.6% WER\n" ] } ], "source": [ "wer_val = get_wer_bulk(\n", " [m[\"text_norm\"] for m in dev_meta],\n", " greedy_preds,\n", ")\n", "print(\"{}% WER\".format(round(wer_val * 100, 1)))" ] }, { "cell_type": "code", "execution_count": 12, "id": "0ab940a8", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 942/942 [02:07<00:00, 7.40it/s]\n" ] } ], "source": [ "decoder_preds = []\n", "for chunk_meta in tqdm.tqdm(funcy.chunks(16, dev_meta), total=int(np.ceil(len(dev_meta) / 16))):\n", " audio_arr_list = [Audio.from_file(m[\"filepath\"]).array_float for m in chunk_meta]\n", " logits_list = predict_logits(model, audio_arr_list)\n", " decoder_preds.extend([\n", " decoder.decode(\n", " logits,\n", " beam_width=5,\n", " beam_prune_logp=-10.0,\n", " token_min_logp=-5.0,\n", " ) for logits in logits_list\n", " ])" ] }, { "cell_type": "code", "execution_count": 13, "id": "1a5aeb0d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "20.5% WER for decoder preds\n" ] } ], "source": [ "wer_val = get_wer_bulk(\n", " [m[\"text_norm\"] for m in dev_meta],\n", " decoder_preds,\n", ")\n", "print(\"{}% WER for decoder preds\".format(round(wer_val * 100, 1)))" ] }, { "cell_type": "code", "execution_count": null, "id": "741486ef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a7699038", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3c6f3495", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6b0bc9e4", "metadata": {}, "source": [ "## Decoders" ] }, { "cell_type": "code", "execution_count": null, "id": "e6ce11fe", "metadata": {}, "outputs": [], "source": [ "# https://pytorch.org/audio/main/tutorials/asr_inference_with_ctc_decoder_tutorial.html\n", "from torchaudio.models.decoder import ctc_decoder\n", "torch_decoder = ctc_decoder(\n", " lexicon=lexicon_file,\n", " tokens=tokens_file,\n", "# lm=kenlm_file,\n", " beam_size=beam_width,\n", " beam_threshold=10,\n", ")\n", "\n", "decoder = build_ctcdecoder(model.decoder.vocabulary)" ] }, { "cell_type": "code", "execution_count": null, "id": "05513faf", "metadata": {}, "outputs": [], "source": [ "\" \".join(torch_decoder(logits)[0][0].words)" ] }, { "cell_type": "code", "execution_count": null, "id": "7636a875", "metadata": {}, "outputs": [], "source": [ "# pyctcdecoe\n", "decoder.decode(\n", " logits,\n", " beam_width=5,\n", " beam_prune_logp=-10.0,\n", " token_min_logp=-5.0,\n", ") for logits in logits_list\n", "\n", "\n", "# maxtasksperchild=10 in Pool if we instantiate outside\n", "# with multiprocessing.get_context(\"fork\").Pool(10) as pool:\n", "# _ = decoder.decode_beams_batch(pool, logits_list[:128], beam_width=50)" ] }, { "cell_type": "code", "execution_count": null, "id": "00f1fff8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4a58a45b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c7c13780", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6c29be9e", "metadata": {}, "source": [ "## cap/punct" ] }, { "cell_type": "code", "execution_count": 1, "id": "df08134c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[NeMo W 2022-10-24 17:30:00 optimizers:67] Could not import distributed_fused_adam optimizer from Apex\n", "[NeMo W 2022-10-24 17:30:02 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", "[NeMo W 2022-10-24 17:30:03 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "100% [......................................................................] 245117658 / 245117658" ] } ], "source": [ "import nemo\n", "import nemo.collections.nlp as nemo_nlp\n", "from suno_utils.utils.display import suppress_logging\n", "from IPython.utils.io import capture_output\n", "\n", "with suppress_logging():\n", " punct_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n", " model_name=\"punctuation_en_bert\",\n", " )\n", " punct_2_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n", " model_name=\"punctuation_en_distilbert\",\n", " )" ] }, { "cell_type": "code", "execution_count": 35, "id": "806e91f7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 733 ms, sys: 0 ns, total: 733 ms\n", "Wall time: 732 ms\n" ] } ], "source": [ "%%time\n", "with suppress_logging():\n", " with capture_output():\n", " for _ in range(10):\n", " out = punct_model.add_punctuation_capitalization(\n", " ['how are you i recently came across this interesting place oh really very cool '*2]*16\n", " )\n", "# ~12 ms per pred, ~3 with batch 16, ~1.5 with batch 64" ] }, { "cell_type": "code", "execution_count": null, "id": "bd9129f4", "metadata": {}, "outputs": [], "source": [ "import nemo\n", "import nemo.collections.nlp as nemo_nlp\n", "from suno_utils.utils.display import suppress_logging\n", "from IPython.utils.io import capture_output\n", "\n", "with suppress_logging():\n", " punct_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n", " model_name=\"punctuation_en_bert\",\n", " )\n", " punct_2_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n", " model_name=\"punctuation_en_distilbert\",\n", " )\n", "\n", "%%time\n", "with suppress_logging():\n", " with capture_output():\n", " for _ in range(10):\n", " out = punct_model.add_punctuation_capitalization(\n", " ['how are you i recently came across this interesting place oh really very cool']*16\n", " )\n", "# ~12 ms per pred, ~3 with batch 16, ~1.5 with batch 64" ] }, { "cell_type": "code", "execution_count": null, "id": "29672670", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fdae0519", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "2735f6d0", "metadata": {}, "source": [ "## Denormalize" ] }, { "cell_type": "code", "execution_count": null, "id": "072ae551", "metadata": {}, "outputs": [], "source": [ "from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer\n", "\n", "with suppress_logging():\n", " itn_model = InverseNormalizer(lang=\"en\")\n", "# itn_2_model = nemo_nlp.models.ThutmoseTaggerModel.from_pretrained(model_name=\"itn_en_thutmose_bert\")" ] }, { "cell_type": "code", "execution_count": null, "id": "e4f7138b", "metadata": {}, "outputs": [], "source": [ "spoken = \"we paid fifteen dollars for this desk from a t and t i think cause i work at the f b i\"\n", "print(itn_model.inverse_normalize(spoken, verbose=False))\n", "# print(itn_2_model._infer([spoken])[0].split(\"\\t\")[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "1eae4639", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d1ea137f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4be9e432", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3bc27346", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "b28a55a8", "metadata": {}, "outputs": [], "source": [ "docker run --gpus all \\\n", " -it \\\n", " -v /home/georg/code/NeMo:/NeMo \\\n", " -v /home/georg:/home/georg \\\n", " -p 8339:8339 \\\n", " --shm-size=8g \\\n", " --ulimit memlock=-1 \\\n", " --ulimit stack=67108864 \\\n", " nvcr.io/nvidia/pytorch:22.09-py3\n", " \n", "# -v /mnt/data-ssd-1:/mnt/data-ssd-1 \\" ] }, { "cell_type": "code", "execution_count": null, "id": "79dd68db", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4f1fd993", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f94eaa3b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "04069c03", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }