Upload folder using huggingface_hub

ultravox_config.py

@@ -139,6 +139,3 @@ class UltravoxConfig(transformers.PretrainedConfig):
         self.initializer_range = self.text_config.initializer_range
 
         super().__init__(**kwargs)
-
-
-UltravoxConfig.register_for_auto_class()

ultravox_model.py

@@ -11,8 +11,8 @@ import transformers.modeling_outputs
 import transformers.models
 
 # We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
 from .ultravox_config import UltravoxConfig
-from .ultravox_processing import UltravoxProcessor
 from .whisper_model_modified import WhisperEncoder as ModifiedWhisperEncoder
 
 
@@ -188,10 +188,9 @@ class UltravoxModel(
         return model_input
 
     @classmethod
-    def _create_audio_tower(cls, config: UltravoxConfig) -> Union[
-        transformers.Wav2Vec2Model,
-        transformers.models.whisper.modeling_whisper.WhisperEncoder,
-    ]:
+    def _create_audio_tower(
+        cls, config: UltravoxConfig
+    ) -> Union[transformers.Wav2Vec2Model, ModifiedWhisperEncoder]:
         if config.audio_model_id is not None:
             if "whisper" in config.audio_model_id is not None:
                 audio_tower = ModifiedWhisperEncoder.from_pretrained(
@@ -398,14 +397,11 @@ class UltravoxProjector(nn.Sequential):
         return hidden_states
 
 
-transformers.AutoConfig.register("ultravox", UltravoxConfig)
-transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
-# transformers.AutoModelForCausalLM.register(
-#     UltravoxConfig, UltravoxModel
-# )
+UltravoxConfig.register_for_auto_class()
 UltravoxModel.register_for_auto_class()
-transformers.AutoProcessor.register(UltravoxConfig, UltravoxProcessor)
-# UltravoxModel.register_for_auto_class("AutoModelForCausalLM")
 
+transformers.AutoConfig.register("ultravox", UltravoxConfig)
+transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
+# transformers.AutoProcessor.register(UltravoxConfig, UltravoxProcessor)  # TODO: make processo work standalone
 
 transformers.activations.ACT2FN["swiglu"] = SwiGLU

ultravox_pipeline.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional
 import transformers
 
 # We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
 from .ultravox_model import UltravoxModel
 from .ultravox_processing import UltravoxProcessor
 
@@ -104,5 +105,6 @@ class UltravoxPipeline(transformers.Pipeline):
 transformers.pipelines.PIPELINE_REGISTRY.register_pipeline(
     "ultravox-pipeline",
     pipeline_class=UltravoxPipeline,
+    pt_model=transformers.AutoModel,
     type="multimodal",
 )

ultravox_processing.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Union
+from typing import Optional, Union
 
 import numpy as np
 import torch