Upload folder using huggingface_hub

config.json

@@ -0,0 +1,201 @@
+{
+    "output_path": "./run/training/",
+    "logger_uri": null,
+    "run_name": "Sherlock-Holmes-2-epochs",
+    "project_name": "XTTS-v2 Finetune",
+    "run_description": "\n        GPT XTTS training\n        ",
+    "print_step": 50,
+    "plot_step": 100,
+    "model_param_stats": false,
+    "wandb_entity": null,
+    "dashboard_logger": "wandb",
+    "save_on_interrupt": true,
+    "log_model_step": 1000,
+    "save_step": 1000,
+    "save_n_checkpoints": 1,
+    "save_checkpoints": true,
+    "save_all_best": false,
+    "save_best_after": 0,
+    "target_loss": null,
+    "print_eval": true,
+    "test_delay_epochs": 0,
+    "run_eval": true,
+    "run_eval_steps": null,
+    "distributed_backend": "nccl",
+    "distributed_url": "tcp://localhost:54321",
+    "mixed_precision": false,
+    "precision": "fp16",
+    "epochs": 2,
+    "batch_size": 3,
+    "eval_batch_size": 3,
+    "grad_clip": null,
+    "scheduler_after_epoch": true,
+    "lr": 5e-06,
+    "optimizer": "AdamW",
+    "optimizer_params": {
+        "betas": [
+            0.9,
+            0.96
+        ],
+        "eps": 1e-08,
+        "weight_decay": 0.01
+    },
+    "lr_scheduler": "MultiStepLR",
+    "lr_scheduler_params": {
+        "milestones": [
+            900000,
+            2700000,
+            5400000
+        ],
+        "gamma": 0.5,
+        "last_epoch": -1
+    },
+    "use_grad_scaler": false,
+    "allow_tf32": false,
+    "cudnn_enable": true,
+    "cudnn_deterministic": false,
+    "cudnn_benchmark": false,
+    "training_seed": 1,
+    "model": "xtts",
+    "num_loader_workers": 0,
+    "num_eval_loader_workers": 0,
+    "use_noise_augment": false,
+    "audio": {
+        "sample_rate": 16000,
+        "output_sample_rate": 24000,
+        "dvae_sample_rate": 16000
+    },
+    "use_phonemes": false,
+    "phonemizer": null,
+    "phoneme_language": null,
+    "compute_input_seq_cache": false,
+    "text_cleaner": null,
+    "enable_eos_bos_chars": false,
+    "test_sentences_file": "",
+    "phoneme_cache_path": null,
+    "characters": null,
+    "add_blank": false,
+    "batch_group_size": 48,
+    "loss_masking": null,
+    "min_audio_len": 1,
+    "max_audio_len": Infinity,
+    "min_text_len": 1,
+    "max_text_len": Infinity,
+    "compute_f0": false,
+    "compute_energy": false,
+    "compute_linear_spec": false,
+    "precompute_num_workers": 0,
+    "start_by_longest": false,
+    "shuffle": false,
+    "drop_last": false,
+    "datasets": [
+        {
+            "formatter": "",
+            "dataset_name": "",
+            "path": "",
+            "meta_file_train": "",
+            "ignored_speakers": null,
+            "language": "",
+            "phonemizer": "",
+            "meta_file_val": "",
+            "meta_file_attn_mask": ""
+        }
+    ],
+    "test_sentences": [
+        {
+            "text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
+            "speaker_wav": "datasets/Sherlock Holmes Stories  Read by Benedict Cumberbatch/wavs/chunk_0220.wav",
+            "language": "en"
+        },
+        {
+            "text": "This cake is great. It's so delicious and moist.",
+            "speaker_wav": "datasets/Sherlock Holmes Stories  Read by Benedict Cumberbatch/wavs/chunk_0220.wav",
+            "language": "en"
+        }
+    ],
+    "eval_split_max_size": 256,
+    "eval_split_size": 0.01,
+    "use_speaker_weighted_sampler": false,
+    "speaker_weighted_sampler_alpha": 1.0,
+    "use_language_weighted_sampler": false,
+    "language_weighted_sampler_alpha": 1.0,
+    "use_length_weighted_sampler": false,
+    "length_weighted_sampler_alpha": 1.0,
+    "model_args": {
+        "gpt_batch_size": 1,
+        "enable_redaction": false,
+        "kv_cache": true,
+        "gpt_checkpoint": "",
+        "clvp_checkpoint": null,
+        "decoder_checkpoint": null,
+        "num_chars": 255,
+        "tokenizer_file": "./XTTS-files/vocab.json",
+        "gpt_max_audio_tokens": 605,
+        "gpt_max_text_tokens": 402,
+        "gpt_max_prompt_tokens": 70,
+        "gpt_layers": 30,
+        "gpt_n_model_channels": 1024,
+        "gpt_n_heads": 16,
+        "gpt_number_text_tokens": 6681,
+        "gpt_start_text_token": 261,
+        "gpt_stop_text_token": 0,
+        "gpt_num_audio_tokens": 1026,
+        "gpt_start_audio_token": 1024,
+        "gpt_stop_audio_token": 1025,
+        "gpt_code_stride_len": 1024,
+        "gpt_use_masking_gt_prompt_approach": true,
+        "gpt_use_perceiver_resampler": true,
+        "input_sample_rate": 22050,
+        "output_sample_rate": 24000,
+        "output_hop_length": 256,
+        "decoder_input_dim": 1024,
+        "d_vector_dim": 512,
+        "cond_d_vector_in_each_upsampling_layer": true,
+        "duration_const": 102400,
+        "min_conditioning_length": 66150,
+        "max_conditioning_length": 143677,
+        "gpt_loss_text_ce_weight": 0.01,
+        "gpt_loss_mel_ce_weight": 1.0,
+        "debug_loading_failures": true,
+        "max_wav_length": 255995,
+        "max_text_length": 66150,
+        "mel_norm_file": "./XTTS-files/mel_stats.pth",
+        "dvae_checkpoint": "./XTTS-files/dvae.pth",
+        "xtts_checkpoint": "./XTTS-files/model.pth",
+        "vocoder": ""
+    },
+    "model_dir": null,
+    "languages": [
+        "en",
+        "es",
+        "fr",
+        "de",
+        "it",
+        "pt",
+        "pl",
+        "tr",
+        "ru",
+        "nl",
+        "cs",
+        "ar",
+        "zh-cn",
+        "hu",
+        "ko",
+        "ja",
+        "hi"
+    ],
+    "temperature": 0.85,
+    "length_penalty": 1.0,
+    "repetition_penalty": 2.0,
+    "top_k": 50,
+    "top_p": 0.85,
+    "num_gpt_outputs": 1,
+    "gpt_cond_len": 12,
+    "gpt_cond_chunk_len": 4,
+    "max_ref_len": 10,
+    "sound_norm_refs": false,
+    "optimizer_wd_only_on_weights": true,
+    "weighted_loss_attrs": null,
+    "weighted_loss_multipliers": null,
+    "github_branch": "unknown"
+}

ipykernel_launcher.py

@@ -0,0 +1,18 @@
+"""Entry point for launching an IPython kernel.
+
+This is separate from the ipykernel package so we can avoid doing imports until
+after removing the cwd from sys.path.
+"""
+
+import sys
+from pathlib import Path
+
+if __name__ == "__main__":
+    # Remove the CWD from sys.path while we load stuff.
+    # This is added back by InteractiveShellApp.init_path()
+    if sys.path[0] == "" or Path(sys.path[0]) == Path.cwd():
+        del sys.path[0]
+
+    from ipykernel import kernelapp as app
+
+    app.launch_new_instance()

model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15583983f6227c2c94dd690afb57bcd499b678865ba7ac2abc82ac18dc2b63ab
+size 5607926869

trainer_0_log.txt

@@ -0,0 +1,486 @@
+ > Training Environment:
+ | > Backend: Torch
+ | > Mixed precision: False
+ | > Precision: float32
+ | > Current device: 0
+ | > Num. of GPUs: 1
+ | > Num. of CPUs: 20
+ | > Num. of Torch Threads: 1
+ | > Torch seed: 1
+ | > Torch CUDNN: True
+ | > Torch CUDNN deterministic: False
+ | > Torch CUDNN benchmark: False
+ | > Torch TF32 MatMul: False
+
+ > Model has 518442047 parameters
+
+ > EPOCH: 0/2
+ --> run\training\Sherlock-Holmes-2-epochs-April-25-2025_03+08PM-0000000
+
+ > EVALUATION 
+
+   --> STEP: 0
+     | > loss_text_ce: 0.02065003104507923  (0.02065003104507923)
+     | > loss_mel_ce: 4.8800153732299805  (4.8800153732299805)
+     | > loss: 4.900665283203125  (4.900665283203125)
+
+   --> STEP: 1
+     | > loss_text_ce: 0.02369523048400879  (0.02369523048400879)
+     | > loss_mel_ce: 4.812253952026367  (4.812253952026367)
+     | > loss: 4.835948944091797  (4.835948944091797)
+
+   --> STEP: 2
+     | > loss_text_ce: 0.021101634949445724  (0.022398432716727257)
+     | > loss_mel_ce: 4.779322624206543  (4.795788288116455)
+     | > loss: 4.800424098968506  (4.818186521530151)
+
+   --> STEP: 3
+     | > loss_text_ce: 0.021626941859722137  (0.02214126909772555)
+     | > loss_mel_ce: 4.4567179679870605  (4.682764848073323)
+     | > loss: 4.478344917297363  (4.704905986785889)
+
+   --> STEP: 4
+     | > loss_text_ce: 0.021944627165794373  (0.022092108614742756)
+     | > loss_mel_ce: 4.730851650238037  (4.694786548614502)
+     | > loss: 4.752796173095703  (4.716878533363342)
+
+   --> STEP: 5
+     | > loss_text_ce: 0.021313846111297607  (0.021936456114053725)
+     | > loss_mel_ce: 4.908026218414307  (4.7374344825744625)
+     | > loss: 4.92933988571167  (4.759370803833008)
+
+   --> STEP: 6
+     | > loss_text_ce: 0.02120722085237503  (0.021814916903773945)
+     | > loss_mel_ce: 4.877439022064209  (4.760768572489421)
+     | > loss: 4.898646354675293  (4.782583395640056)
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.06062706311543783 (+0.0)
+     | > avg_loss_text_ce: 0.021814916903773945 (+0.0)
+     | > avg_loss_mel_ce: 4.760768572489421 (+0.0)
+     | > avg_loss: 4.782583395640056 (+0.0)
+
+
+ > EPOCH: 1/2
+ --> run\training\Sherlock-Holmes-2-epochs-April-25-2025_03+08PM-0000000
+
+ > TRAINING (2025-04-25 15:08:41) 
+
+   --> TIME: 2025-04-25 15:08:41 -- STEP: 0/370 -- GLOBAL_STEP: 0
+     | > loss_text_ce: 0.022243821993470192  (0.022243821993470192)
+     | > loss_mel_ce: 5.101950168609619  (5.101950168609619)
+     | > loss: 0.06100231409072876  (0.06100231409072876)
+     | > current_lr: 5e-06 
+     | > step_time: 0.1968  (0.1967782974243164)
+     | > loader_time: 0.054  (0.0540008544921875)
+
+
+   --> TIME: 2025-04-25 15:08:52 -- STEP: 50/370 -- GLOBAL_STEP: 50
+     | > loss_text_ce: 0.021904677152633667  (0.023353138603270054)
+     | > loss_mel_ce: 4.993193626403809  (4.857127528190612)
+     | > loss: 0.05970355123281479  (0.05810096099972725)
+     | > current_lr: 5e-06 
+     | > step_time: 0.1014  (0.09666483402252198)
+     | > loader_time: 0.053  (0.053735547065734864)
+
+
+   --> TIME: 2025-04-25 15:09:02 -- STEP: 100/370 -- GLOBAL_STEP: 100
+     | > loss_text_ce: 0.02186979539692402  (0.023663721550256013)
+     | > loss_mel_ce: 5.097776889801025  (4.844553368091585)
+     | > loss: 0.06094817444682121  (0.057954966314136984)
+     | > current_lr: 5e-06 
+     | > step_time: 0.1057  (0.09815988302230835)
+     | > loader_time: 0.0514  (0.05377385139465332)
+
+
+   --> TIME: 2025-04-25 15:09:19 -- STEP: 150/370 -- GLOBAL_STEP: 150
+     | > loss_text_ce: 0.026617947965860367  (0.023607976113756497)
+     | > loss_mel_ce: 4.41134786605835  (4.817071544329327)
+     | > loss: 0.05283292755484581  (0.0576271382222573)
+     | > current_lr: 5e-06 
+     | > step_time: 0.1661  (0.14195078055063884)
+     | > loader_time: 0.046  (0.053330375353495274)
+
+
+   --> TIME: 2025-04-25 15:09:37 -- STEP: 200/370 -- GLOBAL_STEP: 200
+     | > loss_text_ce: 0.021193694323301315  (0.02364832037128508)
+     | > loss_mel_ce: 5.093357086181641  (4.778294647932054)
+     | > loss: 0.06088750809431076  (0.057165988851338626)
+     | > current_lr: 5e-06 
+     | > step_time: 0.3385  (0.16869270443916323)
+     | > loader_time: 0.053  (0.052930202484130856)
+
+
+   --> TIME: 2025-04-25 15:09:57 -- STEP: 250/370 -- GLOBAL_STEP: 250
+     | > loss_text_ce: 0.025131795555353165  (0.02353109458088875)
+     | > loss_mel_ce: 4.591306686401367  (4.7446932544708265)
+     | > loss: 0.05495760217308998  (0.05676457673311233)
+     | > current_lr: 5e-06 
+     | > step_time: 0.4551  (0.19466437816619875)
+     | > loader_time: 0.0546  (0.05248429012298584)
+
+
+   --> TIME: 2025-04-25 15:10:23 -- STEP: 300/370 -- GLOBAL_STEP: 300
+     | > loss_text_ce: 0.02108745090663433  (0.023446181671073044)
+     | > loss_mel_ce: 4.584284782409668  (4.713035838603975)
+     | > loss: 0.05482586473226547  (0.05638669184098641)
+     | > current_lr: 5e-06 
+     | > step_time: 0.3054  (0.22047462224960337)
+     | > loader_time: 0.0541  (0.05223121722539266)
+
+
+   --> TIME: 2025-04-25 15:10:49 -- STEP: 350/370 -- GLOBAL_STEP: 350
+     | > loss_text_ce: 0.025105265900492668  (0.023427489916128773)
+     | > loss_mel_ce: 4.507345676422119  (4.689699961798533)
+     | > loss: 0.053957752883434296  (0.056108661315270834)
+     | > current_lr: 5e-06 
+     | > step_time: 0.2924  (0.24373621872493206)
+     | > loader_time: 0.0491  (0.05209174701145717)
+
+
+ > EVALUATION 
+
+   --> STEP: 0
+     | > loss_text_ce: 0.020510854199528694  (0.020510854199528694)
+     | > loss_mel_ce: 4.415481090545654  (4.415481090545654)
+     | > loss: 4.4359917640686035  (4.4359917640686035)
+
+   --> STEP: 1
+     | > loss_text_ce: 0.023326078429818153  (0.023326078429818153)
+     | > loss_mel_ce: 4.377121448516846  (4.377121448516846)
+     | > loss: 4.400447368621826  (4.400447368621826)
+
+   --> STEP: 2
+     | > loss_text_ce: 0.020727859809994698  (0.022026969119906425)
+     | > loss_mel_ce: 4.428055763244629  (4.402588605880737)
+     | > loss: 4.4487833976745605  (4.424615383148193)
+
+   --> STEP: 3
+     | > loss_text_ce: 0.021357735618948936  (0.02180389128625393)
+     | > loss_mel_ce: 4.123639106750488  (4.309605439503987)
+     | > loss: 4.144996643066406  (4.331409136454265)
+
+   --> STEP: 4
+     | > loss_text_ce: 0.02157936803996563  (0.021747760474681854)
+     | > loss_mel_ce: 4.35579252243042  (4.321152210235596)
+     | > loss: 4.377371788024902  (4.342899799346924)
+
+   --> STEP: 5
+     | > loss_text_ce: 0.021084001287817955  (0.021615008637309075)
+     | > loss_mel_ce: 4.472175121307373  (4.351356792449951)
+     | > loss: 4.493258953094482  (4.372971630096435)
+
+   --> STEP: 6
+     | > loss_text_ce: 0.02100624516606331  (0.021513548058768112)
+     | > loss_mel_ce: 4.452880382537842  (4.368277390797933)
+     | > loss: 4.473886489868164  (4.389790773391724)
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.054283698399861656 (-0.006343364715576172)
+     | > avg_loss_text_ce: 0.021513548058768112 (-0.00030136884500583244)
+     | > avg_loss_mel_ce: 4.368277390797933 (-0.3924911816914882)
+     | > avg_loss: 4.389790773391724 (-0.392792622248332)
+
+ > BEST MODEL : run\training\Sherlock-Holmes-2-epochs-April-25-2025_03+08PM-0000000\best_model_370.pth
+ > Training Environment:
+ | > Backend: Torch
+ | > Mixed precision: False
+ | > Precision: float32
+ | > Current device: 0
+ | > Num. of GPUs: 1
+ | > Num. of CPUs: 20
+ | > Num. of Torch Threads: 1
+ | > Torch seed: 1
+ | > Torch CUDNN: True
+ | > Torch CUDNN deterministic: False
+ | > Torch CUDNN benchmark: False
+ | > Torch TF32 MatMul: False
+
+ > Model has 518442047 parameters
+
+ > EPOCH: 0/3
+ --> run\training\Sherlock-Holmes-3-epochs-April-25-2025_03+11PM-0000000
+
+ > EVALUATION 
+
+   --> STEP: 0
+     | > loss_text_ce: 0.02065003104507923  (0.02065003104507923)
+     | > loss_mel_ce: 4.8800153732299805  (4.8800153732299805)
+     | > loss: 4.900665283203125  (4.900665283203125)
+
+   --> STEP: 1
+     | > loss_text_ce: 0.02369523048400879  (0.02369523048400879)
+     | > loss_mel_ce: 4.812253952026367  (4.812253952026367)
+     | > loss: 4.835948944091797  (4.835948944091797)
+
+   --> STEP: 2
+     | > loss_text_ce: 0.021101634949445724  (0.022398432716727257)
+     | > loss_mel_ce: 4.779322624206543  (4.795788288116455)
+     | > loss: 4.800424098968506  (4.818186521530151)
+
+   --> STEP: 3
+     | > loss_text_ce: 0.021626941859722137  (0.02214126909772555)
+     | > loss_mel_ce: 4.4567179679870605  (4.682764848073323)
+     | > loss: 4.478344917297363  (4.704905986785889)
+
+   --> STEP: 4
+     | > loss_text_ce: 0.021944627165794373  (0.022092108614742756)
+     | > loss_mel_ce: 4.730851650238037  (4.694786548614502)
+     | > loss: 4.752796173095703  (4.716878533363342)
+
+   --> STEP: 5
+     | > loss_text_ce: 0.021313846111297607  (0.021936456114053725)
+     | > loss_mel_ce: 4.908026218414307  (4.7374344825744625)
+     | > loss: 4.92933988571167  (4.759370803833008)
+
+   --> STEP: 6
+     | > loss_text_ce: 0.02120722085237503  (0.021814916903773945)
+     | > loss_mel_ce: 4.877439022064209  (4.760768572489421)
+     | > loss: 4.898646354675293  (4.782583395640056)
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.061905860900878906 (+0.0)
+     | > avg_loss_text_ce: 0.021814916903773945 (+0.0)
+     | > avg_loss_mel_ce: 4.760768572489421 (+0.0)
+     | > avg_loss: 4.782583395640056 (+0.0)
+
+
+ > EPOCH: 1/3
+ --> run\training\Sherlock-Holmes-3-epochs-April-25-2025_03+11PM-0000000
+
+ > TRAINING (2025-04-25 15:12:02) 
+
+   --> TIME: 2025-04-25 15:12:03 -- STEP: 0/370 -- GLOBAL_STEP: 0
+     | > loss_text_ce: 0.022243821993470192  (0.022243821993470192)
+     | > loss_mel_ce: 5.101950645446777  (5.101950645446777)
+     | > loss: 0.06100231781601906  (0.06100231781601906)
+     | > current_lr: 5e-06 
+     | > step_time: 0.0796  (0.07958102226257324)
+     | > loader_time: 0.0468  (0.04683685302734375)
+
+
+   --> TIME: 2025-04-25 15:12:13 -- STEP: 50/370 -- GLOBAL_STEP: 50
+     | > loss_text_ce: 0.021904677152633667  (0.023353138603270054)
+     | > loss_mel_ce: 4.993193626403809  (4.857127528190612)
+     | > loss: 0.05970355123281479  (0.05810096099972725)
+     | > current_lr: 5e-06 
+     | > step_time: 0.0971  (0.09639417648315429)
+     | > loader_time: 0.0485  (0.050910625457763675)
+
+
+   --> TIME: 2025-04-25 15:12:27 -- STEP: 100/370 -- GLOBAL_STEP: 100
+     | > loss_text_ce: 0.02186979539692402  (0.023663721568882464)
+     | > loss_mel_ce: 5.097776889801025  (4.844553372859956)
+     | > loss: 0.06094817444682121  (0.05795496646314859)
+     | > current_lr: 5e-06 
+     | > step_time: 0.2023  (0.12174204587936405)
+     | > loader_time: 0.049  (0.05165000915527344)
+
+
+   --> TIME: 2025-04-25 15:13:04 -- STEP: 150/370 -- GLOBAL_STEP: 150
+     | > loss_text_ce: 0.026617947965860367  (0.023607976151009402)
+     | > loss_mel_ce: 4.41134786605835  (4.817071537971498)
+     | > loss: 0.05283292755484581  (0.057627138197422025)
+     | > current_lr: 5e-06 
+     | > step_time: 0.2527  (0.25384192148844403)
+     | > loader_time: 0.0494  (0.051709008216857914)
+
+
+   --> TIME: 2025-04-25 15:13:41 -- STEP: 200/370 -- GLOBAL_STEP: 200
+     | > loss_text_ce: 0.021193763241171837  (0.023648322029039263)
+     | > loss_mel_ce: 5.093355655670166  (4.778295999765397)
+     | > loss: 0.060887496918439865  (0.057166005037724975)
+     | > current_lr: 5e-06 
+     | > step_time: 0.4276  (0.3186194431781769)
+     | > loader_time: 0.056  (0.051824249029159546)
+
+
+   --> TIME: 2025-04-25 15:14:17 -- STEP: 250/370 -- GLOBAL_STEP: 250
+     | > loss_text_ce: 0.025131838396191597  (0.023531100630760193)
+     | > loss_mel_ce: 4.591292381286621  (4.744693719863892)
+     | > loss: 0.05495743080973625  (0.05676458239555359)
+     | > current_lr: 5e-06 
+     | > step_time: 0.7552  (0.35306662845611575)
+     | > loader_time: 0.0536  (0.05188997745513916)
+
+
+   --> TIME: 2025-04-25 15:15:09 -- STEP: 300/370 -- GLOBAL_STEP: 300
+     | > loss_text_ce: 0.021087417379021645  (0.023446188066154727)
+     | > loss_mel_ce: 4.584291934967041  (4.713036131064096)
+     | > loss: 0.05482594668865204  (0.05638669546693564)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5866  (0.41808438301086415)
+     | > loader_time: 0.0495  (0.051725912888844816)
+
+
+   --> TIME: 2025-04-25 15:15:56 -- STEP: 350/370 -- GLOBAL_STEP: 350
+     | > loss_text_ce: 0.0251054335385561  (0.02342749488140857)
+     | > loss_mel_ce: 4.507317066192627  (4.689699707031248)
+     | > loss: 0.053957413882017136  (0.05610865838825703)
+     | > current_lr: 5e-06 
+     | > step_time: 0.4707  (0.45240074838910777)
+     | > loader_time: 0.0487  (0.05162254674094064)
+
+
+ > EVALUATION 
+
+   --> STEP: 0
+     | > loss_text_ce: 0.020510999485850334  (0.020510999485850334)
+     | > loss_mel_ce: 4.415628910064697  (4.415628910064697)
+     | > loss: 4.436140060424805  (4.436140060424805)
+
+   --> STEP: 1
+     | > loss_text_ce: 0.023326119408011436  (0.023326119408011436)
+     | > loss_mel_ce: 4.377127647399902  (4.377127647399902)
+     | > loss: 4.400453567504883  (4.400453567504883)
+
+   --> STEP: 2
+     | > loss_text_ce: 0.02072770707309246  (0.02202691324055195)
+     | > loss_mel_ce: 4.428041458129883  (4.402584552764893)
+     | > loss: 4.4487690925598145  (4.424611330032349)
+
+   --> STEP: 3
+     | > loss_text_ce: 0.02135772444307804  (0.021803850308060646)
+     | > loss_mel_ce: 4.123610973358154  (4.3095933596293134)
+     | > loss: 4.144968509674072  (4.33139705657959)
+
+   --> STEP: 4
+     | > loss_text_ce: 0.021579395979642868  (0.0217477367259562)
+     | > loss_mel_ce: 4.355801105499268  (4.321145296096802)
+     | > loss: 4.37738037109375  (4.34289288520813)
+
+   --> STEP: 5
+     | > loss_text_ce: 0.02108406275510788  (0.02161500193178654)
+     | > loss_mel_ce: 4.472189903259277  (4.3513542175292965)
+     | > loss: 4.493273735046387  (4.372969055175782)
+
+   --> STEP: 6
+     | > loss_text_ce: 0.02100609987974167  (0.021513518256445725)
+     | > loss_mel_ce: 4.452908515930176  (4.368279933929443)
+     | > loss: 4.473914623260498  (4.389793316523234)
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.0531382163365682 (-0.008767644564310707)
+     | > avg_loss_text_ce: 0.021513518256445725 (-0.00030139864732822014)
+     | > avg_loss_mel_ce: 4.368279933929443 (-0.3924886385599775)
+     | > avg_loss: 4.389793316523234 (-0.3927900791168213)
+
+ > BEST MODEL : run\training\Sherlock-Holmes-3-epochs-April-25-2025_03+11PM-0000000\best_model_370.pth
+
+ > EPOCH: 2/3
+ --> run\training\Sherlock-Holmes-3-epochs-April-25-2025_03+11PM-0000000
+
+ > TRAINING (2025-04-25 15:17:32) 
+
+   --> TIME: 2025-04-25 15:18:36 -- STEP: 30/370 -- GLOBAL_STEP: 400
+     | > loss_text_ce: 0.026931729167699814  (0.023552863424022994)
+     | > loss_mel_ce: 4.232264518737793  (4.534945948918659)
+     | > loss: 0.0507047176361084  (0.05426784406105677)
+     | > current_lr: 5e-06 
+     | > step_time: 1.5333  (1.5590810775756836)
+     | > loader_time: 0.0677  (0.06684258778889975)
+
+
+   --> TIME: 2025-04-25 15:20:26 -- STEP: 80/370 -- GLOBAL_STEP: 450
+     | > loss_text_ce: 0.020132526755332947  (0.023305393802002073)
+     | > loss_mel_ce: 4.565151214599609  (4.511084467172622)
+     | > loss: 0.05458671227097511  (0.05398083268664777)
+     | > current_lr: 5e-06 
+     | > step_time: 1.7984  (1.6084659487009048)
+     | > loader_time: 0.0573  (0.062030056118965136)
+
+
+   --> TIME: 2025-04-25 15:22:09 -- STEP: 130/370 -- GLOBAL_STEP: 500
+     | > loss_text_ce: 0.022751471027731895  (0.02329650246180021)
+     | > loss_mel_ce: 4.295829772949219  (4.466045319117033)
+     | > loss: 0.05141168087720871  (0.05344454657572966)
+     | > current_lr: 5e-06 
+     | > step_time: 1.6361  (1.58745659498068)
+     | > loader_time: 0.0562  (0.06053082759563739)
+
+
+   --> TIME: 2025-04-25 15:23:50 -- STEP: 180/370 -- GLOBAL_STEP: 550
+     | > loss_text_ce: 0.024590985849499702  (0.02325609718552894)
+     | > loss_mel_ce: 4.346665382385254  (4.431982457637788)
+     | > loss: 0.0520387664437294  (0.05303855517672168)
+     | > current_lr: 5e-06 
+     | > step_time: 1.4534  (1.5663768596119352)
+     | > loader_time: 0.053  (0.05971963670518663)
+
+
+   --> TIME: 2025-04-25 15:25:27 -- STEP: 230/370 -- GLOBAL_STEP: 600
+     | > loss_text_ce: 0.022246310487389565  (0.023372901101475177)
+     | > loss_mel_ce: 4.374294281005859  (4.418170050952747)
+     | > loss: 0.05233976989984512  (0.05287551223907782)
+     | > current_lr: 5e-06 
+     | > step_time: 1.1929  (1.5410345740940259)
+     | > loader_time: 0.0506  (0.05902964861496635)
+
+
+   --> TIME: 2025-04-25 15:27:11 -- STEP: 280/370 -- GLOBAL_STEP: 650
+     | > loss_text_ce: 0.022131072357296944  (0.023339813382231768)
+     | > loss_mel_ce: 4.388411521911621  (4.400581210851671)
+     | > loss: 0.0525064580142498  (0.05266572743920344)
+     | > current_lr: 5e-06 
+     | > step_time: 1.9263  (1.53961922611509)
+     | > loader_time: 0.0595  (0.05847194194793701)
+
+
+   --> TIME: 2025-04-25 15:29:05 -- STEP: 330/370 -- GLOBAL_STEP: 700
+     | > loss_text_ce: 0.023882780224084854  (0.023317814375640778)
+     | > loss_mel_ce: 4.366799831390381  (4.392894887924197)
+     | > loss: 0.052270032465457916  (0.052573961662975226)
+     | > current_lr: 5e-06 
+     | > step_time: 1.6675  (1.555066782055479)
+     | > loader_time: 0.0466  (0.05809157906156598)
+
+
+ > EVALUATION 
+
+   --> STEP: 0
+     | > loss_text_ce: 0.020527256652712822  (0.020527256652712822)
+     | > loss_mel_ce: 4.198864936828613  (4.198864936828613)
+     | > loss: 4.2193922996521  (4.2193922996521)
+
+   --> STEP: 1
+     | > loss_text_ce: 0.023331278935074806  (0.023331278935074806)
+     | > loss_mel_ce: 4.249141693115234  (4.249141693115234)
+     | > loss: 4.272472858428955  (4.272472858428955)
+
+   --> STEP: 2
+     | > loss_text_ce: 0.02069421485066414  (0.022012746892869473)
+     | > loss_mel_ce: 4.16215181350708  (4.205646753311157)
+     | > loss: 4.1828460693359375  (4.227659463882446)
+
+   --> STEP: 3
+     | > loss_text_ce: 0.021366318687796593  (0.021797270824511845)
+     | > loss_mel_ce: 3.9948670864105225  (4.135386864344279)
+     | > loss: 4.016233444213867  (4.15718412399292)
+
+   --> STEP: 4
+     | > loss_text_ce: 0.021582989022135735  (0.021743700373917818)
+     | > loss_mel_ce: 4.194777965545654  (4.150234639644623)
+     | > loss: 4.216361045837402  (4.1719783544540405)
+
+   --> STEP: 5
+     | > loss_text_ce: 0.021038610488176346  (0.021602682396769525)
+     | > loss_mel_ce: 4.166754245758057  (4.153538560867309)
+     | > loss: 4.187792778015137  (4.17514123916626)
+
+   --> STEP: 6
+     | > loss_text_ce: 0.021066777408123016  (0.021513364898661774)
+     | > loss_mel_ce: 4.266147136688232  (4.172306656837463)
+     | > loss: 4.2872138023376465  (4.193819999694824)
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.05410194396972656 (+0.0009637276331583636)
+     | > avg_loss_text_ce: 0.021513364898661774 (-1.5335778395103583e-07)
+     | > avg_loss_mel_ce: 4.172306656837463 (-0.19597327709197998)
+     | > avg_loss: 4.193819999694824 (-0.19597331682841013)
+
+ > BEST MODEL : run\training\Sherlock-Holmes-3-epochs-April-25-2025_03+11PM-0000000\best_model_740.pth