Fix precomputed_loader to make embeddings optional

backend/utils/precomputed_loader.py

@@ -54,12 +54,11 @@ class PrecomputedDataLoader:
         """Check if pre-computed data is available."""
         metadata_file = self.data_dir / f"metadata_{self.version}.json"
         models_file = self.data_dir / f"models_{self.version}.parquet"
-        embeddings_file = self.data_dir / f"embeddings_{self.version}.parquet"
         
+        # Embeddings file is optional - coordinates are in models file
         return (
             metadata_file.exists() and
-            models_file.exists() and
-            embeddings_file.exists()
+            models_file.exists()
         )
     
     def load_models(self) -> pd.DataFrame:
@@ -115,16 +114,23 @@ class PrecomputedDataLoader:
         
         return embeddings, model_ids
     
-    def load_all(self) -> Tuple[pd.DataFrame, np.ndarray, Dict]:
+    def load_all(self) -> Tuple[pd.DataFrame, Optional[np.ndarray], Dict]:
         """
         Load all pre-computed data.
         
         Returns:
-            Tuple of (models_df, embeddings_array, metadata_dict)
+            Tuple of (models_df, embeddings_array_or_None, metadata_dict)
         """
         metadata = self.load_metadata()
         df = self.load_models()
-        embeddings, _ = self.load_embeddings()
+        
+        # Try to load embeddings, but they're optional
+        embeddings_file = self.data_dir / f"embeddings_{self.version}.parquet"
+        if embeddings_file.exists():
+            embeddings, _ = self.load_embeddings()
+        else:
+            logger.info("Embeddings file not found, skipping...")
+            embeddings = None
         
         return df, embeddings, metadata
 

netlify.toml

@@ -1,20 +1,18 @@
 [build]
   base = "frontend"
-  publish = "frontend/build"
+  publish = "build"
   command = "npm install --legacy-peer-deps && npm run build"
 
 [build.environment]
   NODE_VERSION = "18"
-  # Set this to your backend URL (Railway, Render, etc.)
-  # REACT_APP_API_URL = "https://your-backend-url.railway.app"
+  # Set your Railway backend URL in Netlify dashboard:
+  # Site settings → Environment variables → REACT_APP_API_URL
 
-# Redirect all routes to index.html for React Router (SPA routing)
 [[redirects]]
   from = "/*"
   to = "/index.html"
   status = 200
 
-# Security headers
 [[headers]]
   for = "/*"
   [headers.values]
@@ -23,3 +21,9 @@
     X-Content-Type-Options = "nosniff"
     Referrer-Policy = "strict-origin-when-cross-origin"
 
+# Cache static assets aggressively
+[[headers]]
+  for = "/static/*"
+  [headers.values]
+    Cache-Control = "public, max-age=31536000, immutable"
+

precompute_150k.log

@@ -0,0 +1,185 @@
+2025-11-29 22:40:15,455 - INFO - ============================================================
+2025-11-29 22:40:15,455 - INFO - FAST PRE-COMPUTATION STARTED
+2025-11-29 22:40:15,455 - INFO - ============================================================
+2025-11-29 22:40:15,455 - INFO - Sample size: 150,000
+2025-11-29 22:40:15,455 - INFO - Output directory: ../precomputed_data
+2025-11-29 22:40:15,455 - INFO - Version: v1
+2025-11-29 22:40:15,455 - INFO - PCA pre-reduction: True (50 dims)
+2025-11-29 22:40:15,455 - INFO - ============================================================
+2025-11-29 22:40:15,455 - INFO - Step 1/5: Loading model data (prioritizing base models)...
+Repo card metadata block was not found. Setting CardData to empty.
+2025-11-29 22:40:15,682 - WARNING - Repo card metadata block was not found. Setting CardData to empty.
+2025-11-29 22:40:19,390 - INFO - Loaded 150,000 models in 3.9 seconds
+2025-11-29 22:40:19,390 - INFO - Step 2/5: Generating embeddings...
+2025-11-29 22:40:19,390 - INFO - Building combined text from model fields...
+2025-11-29 22:40:19,488 - INFO - Use pytorch device_name: mps
+2025-11-29 22:40:19,488 - INFO - Load pretrained SentenceTransformer: all-MiniLM-L6-v2
+
+2025-11-29 22:45:49,794 - INFO - Generated embeddings: (150000, 384) in 5.5 minutes
+2025-11-29 22:45:49,794 - INFO - Step 2.5/5: PCA reduction (384 -> 50 dims)...
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_pca.py:604: RuntimeWarning: divide by zero encountered in matmul
+  C = X.T @ X
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_pca.py:604: RuntimeWarning: overflow encountered in matmul
+  C = X.T @ X
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_pca.py:604: RuntimeWarning: invalid value encountered in matmul
+  C = X.T @ X
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:148: RuntimeWarning: divide by zero encountered in matmul
+  X_transformed = X @ self.components_.T
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:148: RuntimeWarning: overflow encountered in matmul
+  X_transformed = X @ self.components_.T
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:148: RuntimeWarning: invalid value encountered in matmul
+  X_transformed = X @ self.components_.T
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:155: RuntimeWarning: divide by zero encountered in matmul
+  X_transformed -= xp.reshape(self.mean_, (1, -1)) @ self.components_.T
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:155: RuntimeWarning: overflow encountered in matmul
+  X_transformed -= xp.reshape(self.mean_, (1, -1)) @ self.components_.T
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/sklearn/decomposition/_base.py:155: RuntimeWarning: invalid value encountered in matmul
+  X_transformed -= xp.reshape(self.mean_, (1, -1)) @ self.components_.T
+2025-11-29 22:45:49,880 - INFO - PCA complete in 0.1s (preserved 92.5% variance)
+2025-11-29 22:45:49,880 - INFO - Reduced embeddings: (150000, 50)
+2025-11-29 22:45:49,880 - INFO - Step 3/5: Running OPTIMIZED UMAP for 3D coordinates...
+UMAP(low_memory=False, n_components=3, spread=1.5, verbose=True)
+Sat Nov 29 22:45:50 2025 Construct fuzzy simplicial set
+Sat Nov 29 22:45:50 2025 Finding Nearest Neighbors
+Sat Nov 29 22:45:50 2025 Building RP forest with 24 trees
+Sat Nov 29 22:45:52 2025 NN descent for 17 iterations
+	 1  /  17
+	 2  /  17
+	 3  /  17
+	 4  /  17
+	Stopping threshold met -- exiting after 4 iterations
+Sat Nov 29 22:46:02 2025 Finished Nearest Neighbor Search
+Sat Nov 29 22:46:04 2025 Construct embedding
+
+2025-11-29 22:56:30,884 - INFO - Generated 3D coordinates: (150000, 3) in 10.7 minutes
+2025-11-29 22:56:30,885 - INFO - Step 4/5: Running OPTIMIZED UMAP for 2D coordinates...
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/umap/spectral.py:548: UserWarning: Spectral initialisation failed! The eigenvector solver
+failed. This is likely due to too small an eigengap. Consider
+adding some noise or jitter to your data.
+
+Falling back to random initialisation!
+  warn(
+	completed  0  /  200 epochs
+	completed  20  /  200 epochs
+	completed  40  /  200 epochs
+	completed  60  /  200 epochs
+	completed  80  /  200 epochs
+	completed  100  /  200 epochs
+	completed  120  /  200 epochs
+	completed  140  /  200 epochs
+	completed  160  /  200 epochs
+	completed  180  /  200 epochs
+Sat Nov 29 22:56:30 2025 Finished embedding
+UMAP(low_memory=False, spread=1.5, verbose=True)
+Sat Nov 29 22:56:30 2025 Construct fuzzy simplicial set
+Sat Nov 29 22:56:30 2025 Finding Nearest Neighbors
+Sat Nov 29 22:56:30 2025 Building RP forest with 24 trees
+Sat Nov 29 22:56:31 2025 NN descent for 17 iterations
+	 1  /  17
+	 2  /  17
+	 3  /  17
+	 4  /  17
+	Stopping threshold met -- exiting after 4 iterations
+Sat Nov 29 22:56:35 2025 Finished Nearest Neighbor Search
+Sat Nov 29 22:56:35 2025 Construct embedding
+
+2025-11-29 23:16:50,380 - INFO - Generated 2D coordinates: (150000, 2) in 20.3 minutes
+2025-11-29 23:16:50,381 - INFO - Step 5/5: Saving to Parquet files...
+2025-11-29 23:16:51,521 - INFO - Saved models data: ../precomputed_data/models_v1.parquet (19.9 MB)
+2025-11-29 23:16:51,522 - ERROR - Pre-computation failed: 'modelId'
+Traceback (most recent call last):
+  File "/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/pandas/core/indexes/base.py", line 3812, in get_loc
+    return self._engine.get_loc(casted_key)
+           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
+  File "pandas/_libs/index.pyx", line 167, in pandas._libs.index.IndexEngine.get_loc
+  File "pandas/_libs/index.pyx", line 196, in pandas._libs.index.IndexEngine.get_loc
+  File "pandas/_libs/hashtable_class_helper.pxi", line 7088, in pandas._libs.hashtable.PyObjectHashTable.get_item
+  File "pandas/_libs/hashtable_class_helper.pxi", line 7096, in pandas._libs.hashtable.PyObjectHashTable.get_item
+KeyError: 'modelId'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/Users/hamidaho/hf_viz/backend/scripts/precompute_fast.py", line 266, in <module>
+    precompute_fast(
+    ~~~~~~~~~~~~~~~^
+        sample_size=args.sample_size,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    ...<3 lines>...
+        use_pca=not args.no_pca
+        ^^^^^^^^^^^^^^^^^^^^^^^
+    )
+    ^
+  File "/Users/hamidaho/hf_viz/backend/scripts/precompute_fast.py", line 187, in precompute_fast
+    'model_id': df['modelId'].values,
+                ~~^^^^^^^^^^^
+  File "/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/pandas/core/frame.py", line 4113, in __getitem__
+    indexer = self.columns.get_loc(key)
+  File "/Users/hamidaho/hf_viz/venv/lib/python3.13/site-packages/pandas/core/indexes/base.py", line 3819, in get_loc
+    raise KeyError(key) from err
+KeyError: 'modelId'
+	completed  0  /  200 epochs
+	completed  20  /  200 epochs
+	completed  40  /  200 epochs
+	completed  60  /  200 epochs
+	completed  80  /  200 epochs
+	completed  100  /  200 epochs
+	completed  120  /  200 epochs
+	completed  140  /  200 epochs
+	completed  160  /  200 epochs
+	completed  180  /  200 epochs
+Sat Nov 29 23:16:50 2025 Finished embedding

precomputed_data/metadata_v1.json

@@ -0,0 +1,19 @@
+{
+  "version": "v1",
+  "created_at": "2025-11-29T23:17:45.066035",
+  "total_models": 150000,
+  "embedding_dim": 384,
+  "umap_3d_shape": [
+    150000,
+    3
+  ],
+  "umap_2d_shape": [
+    150000,
+    2
+  ],
+  "unique_libraries": 364,
+  "unique_pipelines": 54,
+  "processing_time_minutes": 35,
+  "pca_dims": 50,
+  "pca_variance_preserved": 0.925
+}