from datetime import timedelta import json from huggingface_hub import hf_hub_download import torch import torch.nn.functional as F import torchaudio import librosa from torch import nn from transformers import Wav2Vec2ConformerModel from torch_state_bridge import state_bridge from torch.nn.utils.rnn import pad_sequence from safetensors.torch import load_file import webrtcvad from torch.utils.data import Dataset , DataLoader import srt class ChunkedData(Dataset): def __init__(self, wav, sr): if sr != 16000: wav = torchaudio.functional.resample(wav, sr, 16000) self.wav = wav.mean(0, keepdim=True) self.sr = 16000 # Sirf timestamps store karo, actual chunk nahi self.ts = self.make_chunk_timestamps(self.wav) def __len__(self): return len(self.ts) def __getitem__(self, i): st, ed = self.ts[i] st_i = int(st * self.sr) ed_i = int(ed * self.sr) chunk = self.wav[:, st_i:ed_i].squeeze() return chunk, self.ts[i] def make_chunk_timestamps(self, wav, sr=16000, ag=2, min_s=10, max_s=15, ms=30): wav_int16 = (wav * 32768).clamp(-32768, 32767).short().squeeze(0) frame_len = int(sr * ms / 1000) num_frames = len(wav_int16) // frame_len wav_int16 = wav_int16[: num_frames * frame_len] frames = wav_int16.view(num_frames, frame_len) vad = webrtcvad.Vad(ag) speech = torch.tensor( [vad.is_speech(frame.numpy().tobytes(), sr) for frame in frames], dtype=torch.bool ) timestamps = [] total_samples = len(wav_int16) min_len = int(min_s * sr) max_len = int(max_s * sr) st = 0 while st < total_samples: ed = min(st + max_len, total_samples) if ed - st < min_len and ed < total_samples: ed = min(st + min_len, total_samples) timestamps.append(( round(st / sr, 2), round(ed / sr, 2) )) st = ed return timestamps def padding_audio(batch): audios, times = zip(*batch) lengths = torch.tensor([audio.numel() for audio in audios]) times = torch.tensor(times, dtype=torch.float32) padded = pad_sequence(audios, batch_first=True) return padded, lengths, times def calc_length(lengths, all_paddings=2, kernel_size=3, stride=2, repeat_num=1): add_pad = all_paddings - kernel_size for _ in range(repeat_num): lengths = torch.floor((lengths.float() + add_pad) / stride + 1) return lengths class Op(nn.Module): def __init__(self, func,allow_self=False): super().__init__() self.func = func self.allow_self = allow_self def forward(self, x): if self.allow_self: return self.func(self,x) return self.func(x) class Wav2Vec2ConformerRNNT(Wav2Vec2ConformerModel): def __init__(self, config): self.language = config.languages[0] if len(config.languages) > 1: config.hidden_size = 1024 config.num_hidden_layers = 24 config.conv_depthwise_kernel_size = 9 config.conv_stride = [2,2,2] config.conv_kernel = [3,3,3] config.conv_dim = [256,256,256] config.feat_extract_norm = "group" config.intermediate_size = 4096 config.num_feat_extract_layers = len(config.conv_dim) config.lstm_layer = 2 self.cache_length = None self.hop, self.preemph, self.eps, self.pad_to = 160, 0.97, 2**-24, 16 self.denorm = (2 ** config.num_feat_extract_layers) * self.hop / config.sampling_rate self.scaler = config.hidden_size ** (1/2) super().__init__(config) self.eval() def init_weights(self): del self.encoder.pos_conv_embed config = self.config self.enc = nn.Linear(config.hidden_size, config.joint_hidden) self.pred = nn.Linear(config.pred_hidden, config.joint_hidden) self.joint = nn.Linear(config.joint_hidden, config.vocab_size // 22 + 1) self.embed = nn.Embedding(config.vocab_size+1, config.pred_hidden, padding_idx=config.vocab_size) self.lstm = nn.LSTM(config.pred_hidden, config.pred_hidden, config.lstm_layer, batch_first=True) self.act = nn.ReLU(inplace=True) self.spec = torchaudio.transforms.Spectrogram(n_fft=512, hop_length=160, win_length=400, center=False) self.mask_layer = Op(lambda self_obj,x : x.masked_fill(self_obj.cache_pad_mask.unsqueeze(1), 0),True) self.register_buffer( "mel_fb", torch.tensor( librosa.filters.mel( sr=self.config.sampling_rate, n_fft=512, n_mels=80 ) ) ) for idx,l in enumerate(self.feature_extractor.conv_layers): if len(self.config.languages) == 1 or idx == 0: l.conv = nn.Conv2d(l.conv.in_channels,l.conv.out_channels,l.conv.kernel_size[0],l.conv.stride,1) l.layer_norm = nn.Identity() else: l.conv = nn.Sequential(nn.Conv2d(l.conv.in_channels,l.conv.out_channels,l.conv.kernel_size[0],l.conv.stride,1,groups=l.conv.out_channels),nn.Conv2d(l.conv.in_channels,l.conv.out_channels, 1)) self.feature_extractor.conv_layers.append(Op(lambda x : x.transpose(1, 2))) self.feature_projection.projection = nn.Linear(config.conv_dim[-1] * int(calc_length(torch.tensor(80.),repeat_num=self.config.num_feat_extract_layers)),config.hidden_size) self.feature_projection.layer_norm = Op(lambda x:x.permute(0, 2, 1, 3).flatten(2)) for l in self.encoder.layers: l.conv_module.glu = nn.Sequential(l.conv_module.glu,self.mask_layer) l.conv_module.pointwise_conv1.bias = nn.Parameter(torch.empty(l.conv_module.pointwise_conv1.out_channels)) l.conv_module.pointwise_conv2.bias = nn.Parameter(torch.empty(l.conv_module.pointwise_conv2.out_channels)) l.conv_module.depthwise_conv.bias = nn.Parameter(torch.empty(l.conv_module.depthwise_conv.out_channels)) self.encoder.layer_norm = nn.Identity() if len(self.config.languages) > 1: self.lang_joint_net = nn.ModuleDict({l: nn.Linear(config.joint_hidden, config.vocab_size // 22 + 1) for l in config.languages}) return super().init_weights() def _mask_hidden_states(self, hidden_states, mask_time_indices = None, attention_mask = None): hidden_states = hidden_states * self.scaler self.mask_layer.cache_pad_mask = (torch.arange(hidden_states.size(1), device=hidden_states.device).unsqueeze(0) >= self.cache_length.unsqueeze(1)) return super()._mask_hidden_states(hidden_states, mask_time_indices, attention_mask) def preprocessing(self, x): x, l = x l = (l // self.hop + 1).long() x = torch.cat((x[:, :1], x[:, 1:] - self.preemph * x[:, :-1]), 1) x = (self.mel_fb @ self.spec(x) + self.eps).log() T = x.size(-1) m = torch.arange(T, device=x.device)[None] >= l[:, None] x = x.masked_fill(m[:, None], 0) μ = x.sum(-1) / l[:, None] denom = torch.clamp(l[:, None] - 1, min=1) σ = (((x - μ[..., None])**2).sum(-1) / denom + 1e-5).sqrt() x = ((x - μ[..., None]) / σ[..., None]).masked_fill(m[:, None], 0) self.cache_length = calc_length(l, repeat_num=self.config.num_feat_extract_layers).long() return F.pad(x, (0, (-T) % self.pad_to)).transpose(1, 2) def forward(self, input_values): return self._greedy_decode(super().forward(self.preprocessing(input_values)).last_hidden_state) @torch.inference_mode() def transcribe(self,wav,sr,batch_size): device = next(self.parameters()).device subtitles = [] for batch, lengths, timestamp in DataLoader(ChunkedData(wav, sr),batch_size,collate_fn=padding_audio): batch = batch.to(device) lengths = lengths.to(device) timestamp = timestamp.to(device) subtitles.extend(self.make_srt(self.forward((batch, lengths)),timestamp)) yield srt.compose(subtitles) del batch del lengths def load_state_dict(self, state_dict, strict=True, assign=False): state_dict.pop('ctc_decoder.decoder_layers.0.bias', None) state_dict.pop('ctc_decoder.decoder_layers.0.weight', None) state_dict['preprocessor.featurizer.fb'] = state_dict['preprocessor.featurizer.fb'].squeeze(0) changes = """ preprocessor.featurizer.fb,mel_fb preprocessor.featurizer.window,spec.window norm_feed_forward1,ffn1_layer_norm norm_feed_forward2,ffn2_layer_norm feed_forward1.linear1,ffn1.intermediate_dense feed_forward1.linear2,ffn1.output_dense feed_forward2.linear1,ffn2.intermediate_dense feed_forward2.linear2,ffn2.output_dense norm_self_att,self_attn_layer_norm norm_out,final_layer_norm norm_conv,conv_module.layer_norm .conv.,.conv_module. decoder.prediction.dec_rnn.lstm,lstm decoder.prediction.embed,embed joint.enc,enc joint.pred,pred joint.joint_net.2,lang_joint_net encoder.pre_encode.conv_module.0,feature_extractor.conv_layers.0.conv encoder.pre_encode.out,feature_projection.projection """ if len(self.config.languages) == 1: changes += f"""lang_joint_net.{self.language},joint encoder.pre_encode.conv_module.{{n}},feature_extractor.conv_layers.{{(n/2)}}.conv""" else: state_dict["joint.weight"] = self.joint.weight.clone() state_dict["joint.bias"] = self.joint.bias.clone() changes += """encoder.pre_encode.conv_module.{n},encoder.pre_encode.conv_module.{(n-2)} encoder.pre_encode.conv_module.{n},feature_extractor.conv_layers.{(n//3+1)}.conv.{(n%3)} """ # replicate many changes for complex maths state_dict = state_bridge(state_dict, changes) if len(self.config.languages) == 1: state_dict = {k: v for k, v in state_dict.items() if "lang_joint_net" not in k} return super().load_state_dict(state_dict, strict, assign) @torch.jit.export def _greedy_decode(self, enc_out: torch.Tensor): B, T, _ = enc_out.size() device = enc_out.device enc_proj = self.enc(enc_out) max_symbols = self.config.max_symbols_per_step max_len = T * max_symbols token_buffer = torch.full( (B, max_len), -1, dtype=torch.long, device=device ) start_buffer = torch.zeros( (B, max_len), device=device ) lengths = torch.zeros(B, dtype=torch.long, device=device) last = torch.full( (B, 1), self.config.blank_id, dtype=torch.long, device=device ) h = None for t in range(T): e = enc_proj[:, t:t+1] for _ in range(max_symbols): p, h2 = self.lstm(self.embed(last), h) joint = self.joint(self.act(e + self.pred(p))).squeeze(1) n = joint.argmax(-1) blank = n.eq(self.config.blank_id) emit_mask = ~blank if not emit_mask.any(): break pos = lengths[emit_mask] token_buffer[emit_mask, pos] = n[emit_mask] start_buffer[emit_mask, pos] = t * self.denorm lengths[emit_mask] += 1 last = torch.where(emit_mask[:, None], n[:, None], last) if h is None: h = h2 else: keep_mask = blank.view(1, -1, 1) h = ( torch.where(keep_mask, h[0], h2[0]), torch.where(keep_mask, h[1], h2[1]), ) tokens = [] starts = [] for b in range(B): L = lengths[b] tokens.append(token_buffer[b, :L]) starts.append(start_buffer[b, :L]) return tokens, starts def make_srt(self, decoded, ts): tokens_list, starts_list = decoded start_token_segment = ( self.config.languages.index(self.language) * self.joint.out_features ) all_tokens = [] all_starts = [] all_ends = [] device = tokens_list[0].device for tokens, starts, (seg_start, seg_end) in zip( tokens_list, starts_list, ts): tokens = tokens + start_token_segment starts = starts + seg_start all_tokens.append(tokens) all_starts.append(starts) all_ends.append(torch.cat([starts[1:], seg_end[None]])) # newline marker all_tokens.append(torch.tensor([-1], device=device)) all_starts.append(torch.tensor([seg_end], device=device)) all_ends.append(torch.tensor([seg_end + 0.005], device=device)) return [ srt.Subtitle( i, timedelta(seconds=float(st)), timedelta(seconds=float(en)), "" if tok == -1 else self.config.vocab[int(tok)] ) for i, (tok, st, en) in enumerate( zip( torch.cat(all_tokens), torch.cat(all_starts), torch.cat(all_ends) ), 1 ) ] @classmethod def from_pretrained( cls, pretrained_model_name_or_path, config=None, language=None, use_jit=False, use_quantization=False): if config is None: raise ValueError("config must be provided") if language: config.languages = [language] vocab_file = hf_hub_download( pretrained_model_name_or_path, "vocab.json" ) vocab_json = json.load(open(vocab_file)) config.vocab = [''] + vocab_json['small'][language] model = cls(config) weight_file = hf_hub_download( pretrained_model_name_or_path, f"{language or 'all'}.safetensors" ) model.load_state_dict(load_file(weight_file)) if use_quantization: model = torch.quantization.quantize_dynamic(model) return model