{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df08134c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[NeMo W 2022-10-24 17:30:00 optimizers:67] Could not import distributed_fused_adam optimizer from Apex\n",
      "[NeMo W 2022-10-24 17:30:02 experimental:27] Module <class 'nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers.MegatronPretrainingRandomBatchSampler'> is experimental, not ready for production and is not fully supported. Use at your own risk.\n",
      "[NeMo W 2022-10-24 17:30:03 experimental:27] Module <class 'nemo.collections.nlp.models.text_normalization_as_tagging.thutmose_tagger.ThutmoseTaggerModel'> is experimental, not ready for production and is not fully supported. Use at your own risk.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100% [......................................................................] 245117658 / 245117658"
     ]
    }
   ],
   "source": [
    "import nemo\n",
    "import nemo.collections.nlp as nemo_nlp\n",
    "from suno_utils.utils.display import suppress_logging\n",
    "from IPython.utils.io import capture_output\n",
    "\n",
    "with suppress_logging():\n",
    "    punct_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n",
    "        model_name=\"punctuation_en_bert\",\n",
    "    )\n",
    "    punct_2_model = nemo_nlp.models.PunctuationCapitalizationModel.from_pretrained(\n",
    "        model_name=\"punctuation_en_distilbert\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "806e91f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 733 ms, sys: 0 ns, total: 733 ms\n",
      "Wall time: 732 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with suppress_logging():\n",
    "    with capture_output():\n",
    "        for _ in range(10):\n",
    "            out = punct_model.add_punctuation_capitalization(\n",
    "                ['how are you i recently came across this interesting place oh really very cool '*2]*16\n",
    "            )\n",
    "# ~12 ms per pred, ~3 with batch 16, ~1.5 with batch 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "889be973",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a664037a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33dd934d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f56e2512",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bfb9886",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e33c1d6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9de069bd",
   "metadata": {},
   "source": [
    "## denorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "8cb96a71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-10-24 18:24:16 tokenize_and_classify:64] Creating ClassifyFst grammars.\n"
     ]
    }
   ],
   "source": [
    "# create inverse text normalization instance\n",
    "from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer\n",
    "with suppress_logging():\n",
    "    itn_model = InverseNormalizer(lang=\"en\")\n",
    "    itn_2_model = nemo_nlp.models.ThutmoseTaggerModel.from_pretrained(model_name=\"itn_en_thutmose_bert\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "acac965a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we paid $15 for this desk from AT&T i think cause i work at the f b i\n",
      "we paid $ 15 for this desk from at & t i think cause i work at the fbi \n"
     ]
    }
   ],
   "source": [
    "spoken = \"we paid fifteen dollars for this desk from a t and t i think cause i work at the f b i\"\n",
    "print(itn_model.inverse_normalize(spoken, verbose=False))\n",
    "print(itn_2_model._infer([spoken])[0].split(\"\\t\")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04ca8760",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b89daaf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae8b4dfd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "471d20fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-09-22 15:29:08 tokenize_and_classify:87] Creating ClassifyFst grammars.\n"
     ]
    }
   ],
   "source": [
    "from nemo_text_processing.text_normalization.normalize import Normalizer\n",
    "normalizer = Normalizer(input_case='cased', lang='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bd4649d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created cache_dir/en_tn_post_processing.far\n",
      "[NeMo I 2022-09-22 15:22:18 tokenize_and_classify_with_audio:101] Creating ClassifyFst grammars. This might take some time...\n",
      "Created cache_dir/_cased_en_tn_False_deterministic.far\n",
      "[NeMo I 2022-09-22 15:29:03 tokenize_and_classify_with_audio:229] ClassifyFst grammars are saved to cache_dir/_cased_en_tn_False_deterministic.far.\n",
      "Created cache_dir/en_tn_False_deterministic_verbalizer.far\n",
      "[NeMo I 2022-09-22 15:29:05 verbalize_final:76] VerbalizeFinalFst grammars are saved to cache_dir/en_tn_False_deterministic_verbalizer.far.\n"
     ]
    }
   ],
   "source": [
    "from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio\n",
    "normalizer = NormalizerWithAudio(\n",
    "        lang=\"en\",\n",
    "        input_case=\"cased\",\n",
    "        overwrite_cache=False,\n",
    "        cache_dir=\"cache_dir\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4d5edf70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'we work for the FBI and others'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "written = \"we work for the FBI and others\"\n",
    "normalized = normalizer.normalize(written, verbose=False, punct_post_process=False)\n",
    "normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "460a93b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we work for the FBI and others\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'we work for the F B I and others', 'we work for the FBI and others'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "written = \"we work for the FBI and others\"\n",
    "normalized = normalizer_audio.normalize(written, n_tagged=10, punct_post_process=True)\n",
    "normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e2bc1ca6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-10-24 18:04:38 cloud:56] Found existing object /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_t5_tagger/53ce8e48e52f76d04a2d6de4bf14ae40/itn_en_t5_tagger.nemo.\n",
      "[NeMo I 2022-10-24 18:04:38 cloud:62] Re-using file from: /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_t5_tagger/53ce8e48e52f76d04a2d6de4bf14ae40/itn_en_t5_tagger.nemo\n",
      "[NeMo I 2022-10-24 18:04:38 common:911] Instantiating model from pre-trained checkpoint\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at albert-base-v2 were not used when initializing AlbertForTokenClassification: ['predictions.dense.bias', 'predictions.LayerNorm.weight', 'predictions.LayerNorm.bias', 'predictions.decoder.bias', 'predictions.decoder.weight', 'predictions.dense.weight', 'predictions.bias']\n",
      "- This IS expected if you are initializing AlbertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing AlbertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of AlbertForTokenClassification were not initialized from the model checkpoint at albert-base-v2 and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-10-24 18:04:40 save_restore_connector:243] Model DuplexTaggerModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_t5_tagger/53ce8e48e52f76d04a2d6de4bf14ae40/itn_en_t5_tagger.nemo.\n",
      "[NeMo I 2022-10-24 18:04:40 cloud:56] Found existing object /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_t5_decoder/b23f97d4a4781522b371b6dfbc14e4f2/itn_en_t5_decoder.nemo.\n",
      "[NeMo I 2022-10-24 18:04:40 cloud:62] Re-using file from: /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_t5_decoder/b23f97d4a4781522b371b6dfbc14e4f2/itn_en_t5_decoder.nemo\n",
      "[NeMo I 2022-10-24 18:04:40 common:911] Instantiating model from pre-trained checkpoint\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[NeMo W 2022-10-24 18:04:41 modelPT:149] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n",
      "    Validation config : \n",
      "    data_path: /home/aleksandraa/data/tn_data/en_data_split/dev.tsv\n",
      "    batch_size: 8\n",
      "    shuffle: false\n",
      "    max_insts: -1\n",
      "    use_cache: false\n",
      "    num_workers: 3\n",
      "    pin_memory: false\n",
      "    drop_last: false\n",
      "    \n",
      "[NeMo W 2022-10-24 18:04:43 modelPT:149] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \n",
      "    Validation config : \n",
      "    data_path: /home/aleksandraa/data/tn_data/en_data_split/dev.tsv\n",
      "    batch_size: 8\n",
      "    shuffle: false\n",
      "    max_insts: -1\n",
      "    use_cache: false\n",
      "    num_workers: 3\n",
      "    pin_memory: false\n",
      "    drop_last: false\n",
      "    \n",
      "[NeMo E 2022-10-24 18:04:44 common:504] Model instantiation failed!\n",
      "    Target class:\tnemo.collections.nlp.models.duplex_text_normalization.duplex_decoder.DuplexDecoderModel\n",
      "    Error(s):\tNo module named 'fused_layer_norm_cuda'\n",
      "    Traceback (most recent call last):\n",
      "      File \"/home/georg/code/NeMo/nemo/core/classes/common.py\", line 483, in from_config_dict\n",
      "        instance = imported_cls(cfg=config, trainer=trainer)\n",
      "      File \"/home/georg/code/NeMo/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py\", line 78, in __init__\n",
      "        self.model = AutoModelForSeq2SeqLM.from_pretrained(cfg.transformer)\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py\", line 446, in from_pretrained\n",
      "        return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/modeling_utils.py\", line 2113, in from_pretrained\n",
      "        model = cls(config, *model_args, **model_kwargs)\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\", line 1477, in __init__\n",
      "        self.encoder = T5Stack(encoder_config, self.shared)\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\", line 843, in __init__\n",
      "        [T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\", line 843, in <listcomp>\n",
      "        [T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\", line 627, in __init__\n",
      "        self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias))\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\", line 558, in __init__\n",
      "        self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)\n",
      "      File \"/home/georg/venvs/ml/lib/python3.8/site-packages/apex/normalization/fused_layer_norm.py\", line 364, in __init__\n",
      "        fused_layer_norm_cuda = importlib.import_module(\"fused_layer_norm_cuda\")\n",
      "      File \"/usr/lib/python3.8/importlib/__init__.py\", line 127, in import_module\n",
      "        return _bootstrap._gcd_import(name[level:], package, level)\n",
      "      File \"<frozen importlib._bootstrap>\", line 1014, in _gcd_import\n",
      "      File \"<frozen importlib._bootstrap>\", line 991, in _find_and_load\n",
      "      File \"<frozen importlib._bootstrap>\", line 973, in _find_and_load_unlocked\n",
      "    ModuleNotFoundError: No module named 'fused_layer_norm_cuda'\n",
      "    \n"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'fused_layer_norm_cuda'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_4080831/2840853918.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnemo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlp\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnemo_nlp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnemo_nlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mduplex_text_normalization\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDuplexTaggerModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"itn_en_t5\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mdecoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnemo_nlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mduplex_text_normalization\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDuplexDecoderModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"itn_en_t5\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;31m# normalizer = nemo_nlp.models.duplex_text_normalization.DuplexTextNormalizationModel(tagger, decoder, lang=\"en\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/classes/common.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, model_name, refresh_cache, override_config_path, map_location, strict, return_config, trainer, save_restore_connector)\u001b[0m\n\u001b[1;32m    848\u001b[0m             )\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m         instance = class_.restore_from(\n\u001b[0m\u001b[1;32m    851\u001b[0m             \u001b[0mrestore_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnemo_model_file_in_cache\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    852\u001b[0m             \u001b[0moverride_config_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverride_config_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/classes/modelPT.py\u001b[0m in \u001b[0;36mrestore_from\u001b[0;34m(cls, restore_path, override_config_path, map_location, strict, return_config, save_restore_connector, trainer)\u001b[0m\n\u001b[1;32m    314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m         \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_save_restore_connector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_restore_connector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m         instance = cls._save_restore_connector.restore_from(\n\u001b[0m\u001b[1;32m    317\u001b[0m             \u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrestore_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverride_config_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m         )\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/connectors/save_restore_connector.py\u001b[0m in \u001b[0;36mrestore_from\u001b[0;34m(self, calling_cls, restore_path, override_config_path, map_location, strict, return_config, trainer)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;31m# Get path where the command is executed - the artifacts will be \"retrieved\" there\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0;31m# (original .nemo behavior)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m         loaded_params = self.load_config_and_state_dict(\n\u001b[0m\u001b[1;32m    236\u001b[0m             \u001b[0mcalling_cls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrestore_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverride_config_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m         )\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/connectors/save_restore_connector.py\u001b[0m in \u001b[0;36mload_config_and_state_dict\u001b[0;34m(self, calling_cls, restore_path, override_config_path, map_location, strict, return_config, trainer)\u001b[0m\n\u001b[1;32m    156\u001b[0m                 \u001b[0;31m# get the class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m                 \u001b[0mcalling_cls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_model_restore_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_being_restored\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfolder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtmpdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m                 \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalling_cls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_config_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m                 \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap_location\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m                 \u001b[0;31m# add load_state_dict override\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/classes/common.py\u001b[0m in \u001b[0;36mfrom_config_dict\u001b[0;34m(cls, config, trainer)\u001b[0m\n\u001b[1;32m    503\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mprev_error\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    504\u001b[0m                         \u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprev_error\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 505\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    506\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    507\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_cfg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/NeMo/nemo/core/classes/common.py\u001b[0m in \u001b[0;36mfrom_config_dict\u001b[0;34m(cls, config, trainer)\u001b[0m\n\u001b[1;32m    495\u001b[0m                     \u001b[0maccepts_trainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSerialization\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_inspect_signature_for_trainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    496\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0maccepts_trainer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 497\u001b[0;31m                         \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    498\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    499\u001b[0m                         \u001b[0minstance\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/code/NeMo/nemo/collections/nlp/models/duplex_text_normalization/duplex_decoder.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, cfg, trainer)\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mno_lm_init\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoModelForSeq2SeqLM\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_sequence_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'max_sequence_len'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_max_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'mode'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'joint'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mmodel_class\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m         raise ValueError(\n\u001b[1;32m    448\u001b[0m             \u001b[0;34mf\"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   2111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2112\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mContextManagers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit_contexts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2113\u001b[0;31m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2115\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdevice_map\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"auto\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config)\u001b[0m\n\u001b[1;32m   1475\u001b[0m         \u001b[0mencoder_config\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_cache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m         \u001b[0mencoder_config\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_encoder_decoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1477\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT5Stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshared\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1479\u001b[0m         \u001b[0mdecoder_config\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, embed_tokens)\u001b[0m\n\u001b[1;32m    841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m         self.block = nn.ModuleList(\n\u001b[0;32m--> 843\u001b[0;31m             \u001b[0;34m[\u001b[0m\u001b[0mT5Block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    844\u001b[0m         )\n\u001b[1;32m    845\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinal_layer_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT5LayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0md_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    841\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m         self.block = nn.ModuleList(\n\u001b[0;32m--> 843\u001b[0;31m             \u001b[0;34m[\u001b[0m\u001b[0mT5Block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    844\u001b[0m         )\n\u001b[1;32m    845\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfinal_layer_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT5LayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0md_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, has_relative_attention_bias)\u001b[0m\n\u001b[1;32m    625\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_decoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_decoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    626\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModuleList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 627\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT5LayerSelfAttention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    628\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_decoder\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    629\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT5LayerCrossAttention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config, has_relative_attention_bias)\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    557\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSelfAttention\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT5Attention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhas_relative_attention_bias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 558\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT5LayerNorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0md_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm_epsilon\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    559\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout_rate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venvs/ml/lib/python3.8/site-packages/apex/normalization/fused_layer_norm.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, normalized_shape, eps, elementwise_affine)\u001b[0m\n\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    363\u001b[0m         \u001b[0;32mglobal\u001b[0m \u001b[0mfused_layer_norm_cuda\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 364\u001b[0;31m         \u001b[0mfused_layer_norm_cuda\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"fused_layer_norm_cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    366\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnormalized_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumbers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIntegral\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.8/importlib/__init__.py\u001b[0m in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m    125\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m             \u001b[0mlevel\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_bootstrap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gcd_import\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.8/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_gcd_import\u001b[0;34m(name, package, level)\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.8/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.8/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load_unlocked\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'fused_layer_norm_cuda'"
     ]
    }
   ],
   "source": [
    "import nemo\n",
    "import nemo.collections.nlp as nemo_nlp\n",
    "tagger = nemo_nlp.models.duplex_text_normalization.DuplexTaggerModel.from_pretrained(model_name=\"itn_en_t5\")\n",
    "decoder = nemo_nlp.models.duplex_text_normalization.DuplexDecoderModel.from_pretrained(model_name=\"itn_en_t5\")\n",
    "# normalizer = nemo_nlp.models.duplex_text_normalization.DuplexTextNormalizationModel(tagger, decoder, lang=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0ab5d5bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-10-24 17:49:03 cloud:66] Downloading from: https://api.ngc.nvidia.com/v2/models/nvidia/nemo/itn_en_thutmose_bert/versions/1.9.0/files/itn_en_thutmose_bert.nemo to /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_thutmose_bert/bf3a085f3b78525c3b3abe09bd0c62c8/itn_en_thutmose_bert.nemo\n",
      "100% [......................................................................] 445409280 / 445409280[NeMo I 2022-10-24 17:50:21 common:911] Instantiating model from pre-trained checkpoint\n",
      "[NeMo I 2022-10-24 17:50:21 tokenizer_utils:130] Getting HuggingFace AutoTokenizer with pretrained_model_name: bert-base-uncased, vocab_file: /tmp/tmp5lyrkj33/1742bd18a2874ba6b6cb1d643dabcb2b_vocab.txt, merges_files: None, special_tokens_dict: {}, and use_fast: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using eos_token, but it is not set yet.\n",
      "Using bos_token, but it is not set yet.\n",
      "[NeMo W 2022-10-24 17:50:22 modelPT:222] You tried to register an artifact under config key=tokenizer.vocab_file but an artifact for it has already been registered.\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertEncoder: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertEncoder from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertEncoder from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-10-24 17:50:24 save_restore_connector:243] Model ThutmoseTaggerModel was successfully restored from /home/georg/.cache/torch/NeMo/NeMo_1.13.0rc0/itn_en_thutmose_bert/bf3a085f3b78525c3b3abe09bd0c62c8/itn_en_thutmose_bert.nemo.\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b38b9e14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i spent $ 12.4 on candy\\ti spent twelve point four dollars on candy\\t<SELF> <SELF> _12 . 4_ _$<< <SELF> <SELF>\\t<SELF> <SELF> _$<< _12 . 4_ <SELF> <SELF>\\tPLAIN PLAIN MONEY MONEY MONEY MONEY PLAIN PLAIN']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e12b5256",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i work at the fbi and so does mikey\\ti work at the f b i and so does mikey\\t<SELF> <SELF> <SELF> <SELF> _f <SELF> i_ <SELF> <SELF> <SELF> <SELF>\\t<SELF> <SELF> <SELF> <SELF> _f <SELF> i_ <SELF> <SELF> <SELF> <SELF>\\tPLAIN PLAIN PLAIN PLAIN LETTERS LETTERS LETTERS PLAIN PLAIN PLAIN PLAIN']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp_model._infer([\"i work at the f b i and so does mikey\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01d40bf2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "177da3c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d95035b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ca58597",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72ea4a6a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ec4788f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}