Fix: add utils/data files and update gitignore

.dockerignore

@@ -32,10 +32,9 @@ frontend/.env.development.local
 frontend/.env.test.local
 frontend/.env.production.local
 
-# Cache (will be created at runtime)
-cache/*.pkl
-cache/*.npy
-!cache/.gitkeep
+# Keep cache files for fast startup (precomputed UMAP)
+# cache/*.pkl  # INCLUDED for HF Spaces deployment
+# cache/*.npy  # INCLUDED for HF Spaces deployment
 
 # Documentation
 *.md

.gitignore

@@ -34,5 +34,6 @@ flagged/
 # Data
 *.parquet
 *.csv
-data/
+/data/
+!frontend/src/utils/data/
 

Dockerfile

@@ -1,37 +1,33 @@
-# Use Python 3.11 slim image for smaller size
+# Hugging Face Spaces Docker deployment
 FROM python:3.11-slim
 
-# Set working directory
+# Create non-root user (required by HF Spaces)
+RUN useradd -m -u 1000 user
 WORKDIR /app
 
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    && rm -rf /var/lib/apt/lists/*
+# Install dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc g++ && \
+    rm -rf /var/lib/apt/lists/*
 
-# Copy requirements first for better caching
 COPY requirements.txt .
-COPY backend/requirements.txt backend/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
-# Install Python dependencies
-RUN pip install --no-cache-dir -r requirements.txt && \
-    pip install --no-cache-dir -r backend/requirements.txt
+# Copy application
+COPY --chown=user backend/ /app/backend/
 
-# Copy application code
-COPY . .
+# Bundle precomputed data for instant startup
+COPY --chown=user precomputed_data/ /app/precomputed_data/
+COPY --chown=user cache/ /app/cache/
 
-# Create cache directory
-RUN mkdir -p cache
+# Switch to non-root user
+USER user
 
-# Expose port (will be overridden by PORT env var in cloud platforms)
-EXPOSE 8000
-
-# Set environment variables
 ENV PYTHONUNBUFFERED=1
-ENV PORT=8000
+ENV PORT=7860
+ENV ALLOW_ALL_ORIGINS=true
 
-# Run the application
 WORKDIR /app/backend
-CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]
+EXPOSE 7860
 
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"]

frontend/src/utils/data/binaryFilter.ts

@@ -0,0 +1,152 @@
+/**
+ * Client-side filtering engine for binary dataset.
+ * Works with TypedArrays for fast vectorized operations.
+ */
+
+export interface ModelData {
+  x: Float32Array;
+  y: Float32Array;
+  z: Float32Array;
+  domainId: Uint8Array;
+  licenseId: Uint8Array;
+  familyId: Uint16Array;
+  flags: Uint8Array;
+  modelIds: string[];
+  domains: string[];
+  licenses: string[];
+}
+
+export interface FilterCriteria {
+  domains?: string[];
+  licenses?: string[];
+  isBaseModel?: boolean | null;
+  families?: number[];
+}
+
+/**
+ * Create a boolean mask based on filter criteria.
+ * Returns Uint8Array where 1 = included, 0 = excluded.
+ */
+export function createFilterMask(
+  data: ModelData,
+  criteria: FilterCriteria
+): Uint8Array {
+  const numModels = data.x.length;
+  const mask = new Uint8Array(numModels);
+
+  // Initialize all to 1 (included)
+  mask.fill(1);
+
+  // Filter by domain
+  if (criteria.domains && criteria.domains.length > 0) {
+    const domainSet = new Set(criteria.domains);
+    for (let i = 0; i < numModels; i++) {
+      const domain = data.domains[data.domainId[i]];
+      if (!domainSet.has(domain)) {
+        mask[i] = 0;
+      }
+    }
+  }
+
+  // Filter by license
+  if (criteria.licenses && criteria.licenses.length > 0) {
+    const licenseSet = new Set(criteria.licenses);
+    for (let i = 0; i < numModels; i++) {
+      const license = data.licenses[data.licenseId[i]];
+      if (!licenseSet.has(license)) {
+        mask[i] = 0;
+      }
+    }
+  }
+
+  // Filter by base model flag
+  if (criteria.isBaseModel !== undefined && criteria.isBaseModel !== null) {
+    for (let i = 0; i < numModels; i++) {
+      const isBase = (data.flags[i] & 0x01) !== 0;
+      if (isBase !== criteria.isBaseModel) {
+        mask[i] = 0;
+      }
+    }
+  }
+
+  // Filter by family
+  if (criteria.families && criteria.families.length > 0) {
+    const familySet = new Set(criteria.families);
+    for (let i = 0; i < numModels; i++) {
+      if (!familySet.has(data.familyId[i])) {
+        mask[i] = 0;
+      }
+    }
+  }
+
+  return mask;
+}
+
+/**
+ * Apply filter mask to data and return filtered arrays.
+ */
+export function applyMask(
+  data: ModelData,
+  mask: Uint8Array
+): {
+  x: Float32Array;
+  y: Float32Array;
+  z: Float32Array;
+  indices: number[];
+  count: number;
+} {
+  const count = mask.reduce((sum, val) => sum + val, 0);
+  const x = new Float32Array(count);
+  const y = new Float32Array(count);
+  const z = new Float32Array(count);
+  const indices: number[] = [];
+
+  let j = 0;
+  for (let i = 0; i < mask.length; i++) {
+    if (mask[i] === 1) {
+      x[j] = data.x[i];
+      y[j] = data.y[i];
+      z[j] = data.z[i];
+      indices.push(i);
+      j++;
+    }
+  }
+
+  return { x, y, z, indices, count };
+}
+
+/**
+ * Get model ID for a given index.
+ */
+export function getModelId(data: ModelData, index: number): string {
+  return data.modelIds[index] || '';
+}
+
+/**
+ * Get domain name for a given index.
+ */
+export function getDomain(data: ModelData, index: number): string {
+  return data.domains[data.domainId[index]] || '';
+}
+
+/**
+ * Get license name for a given index.
+ */
+export function getLicense(data: ModelData, index: number): string {
+  return data.licenses[data.licenseId[index]] || '';
+}
+
+/**
+ * Check if model is a base model (no parent).
+ */
+export function isBaseModel(data: ModelData, index: number): boolean {
+  return (data.flags[index] & 0x01) !== 0;
+}
+
+/**
+ * Check if model has children.
+ */
+export function hasChildren(data: ModelData, index: number): boolean {
+  return (data.flags[index] & 0x04) !== 0;
+}
+

frontend/src/utils/data/enhancedCache.ts

@@ -0,0 +1,220 @@
+/**
+ * Enhanced IndexedDB cache with TTL and automatic cleanup.
+ * Uses idb library for better reliability and performance.
+ */
+import { openDB, DBSchema, IDBPDatabase } from 'idb';
+
+interface CacheEntry {
+  key: string;
+  data: any;
+  timestamp: number;
+  ttl: number;
+  size?: number;
+}
+
+interface CacheDB extends DBSchema {
+  cache: {
+    key: string;
+    value: CacheEntry;
+    indexes: { 'by-timestamp': number };
+  };
+}
+
+class EnhancedIndexedDBCache {
+  private db: IDBPDatabase<CacheDB> | null = null;
+  private dbName = 'hfviz-cache-v2';
+  private readonly storeName: 'cache' = 'cache';
+  private maxSize = 100 * 1024 * 1024; // 100MB
+  private defaultTTL = 5 * 60 * 1000; // 5 minutes
+
+  async init(): Promise<void> {
+    try {
+      this.db = await openDB<CacheDB>(this.dbName, 2, {
+        upgrade(db, oldVersion) {
+          if (!db.objectStoreNames.contains('cache')) {
+            const store = db.createObjectStore('cache', { keyPath: 'key' });
+            store.createIndex('by-timestamp', 'timestamp');
+          }
+        },
+      });
+
+      // Cleanup on init
+      await this.cleanup();
+    } catch (err) {
+      console.error('Failed to initialize IndexedDB:', err);
+    }
+  }
+
+  async get<T = any>(key: string): Promise<T | null> {
+    if (!this.db) await this.init();
+    if (!this.db) return null;
+
+    try {
+      const entry = await this.db.get(this.storeName, key);
+      
+      if (!entry) return null;
+
+      // Check if expired
+      if (Date.now() - entry.timestamp > entry.ttl) {
+        await this.delete(key);
+        return null;
+      }
+
+      return entry.data as T;
+    } catch (err) {
+      console.error('IndexedDB get error:', err);
+      return null;
+    }
+  }
+
+  async set(key: string, data: any, ttl: number = this.defaultTTL): Promise<boolean> {
+    if (!this.db) await this.init();
+    if (!this.db) return false;
+
+    try {
+      // Estimate size
+      const size = new Blob([JSON.stringify(data)]).size;
+
+      // Check if we need to make space
+      await this.ensureSpace(size);
+
+      const entry: CacheEntry = {
+        key,
+        data,
+        timestamp: Date.now(),
+        ttl,
+        size,
+      };
+
+      await this.db.put(this.storeName, entry);
+      return true;
+    } catch (err) {
+      console.error('IndexedDB set error:', err);
+      return false;
+    }
+  }
+
+  async delete(key: string): Promise<boolean> {
+    if (!this.db) await this.init();
+    if (!this.db) return false;
+
+    try {
+      await this.db.delete(this.storeName, key);
+      return true;
+    } catch (err) {
+      console.error('IndexedDB delete error:', err);
+      return false;
+    }
+  }
+
+  async clear(): Promise<boolean> {
+    if (!this.db) await this.init();
+    if (!this.db) return false;
+
+    try {
+      await this.db.clear(this.storeName);
+      return true;
+    } catch (err) {
+      console.error('IndexedDB clear error:', err);
+      return false;
+    }
+  }
+
+  async cleanup(): Promise<void> {
+    if (!this.db) return;
+
+    try {
+      const tx = this.db.transaction(this.storeName, 'readwrite');
+      const store = tx.objectStore(this.storeName);
+      const now = Date.now();
+
+      let cursor = await store.openCursor();
+      while (cursor) {
+        const entry = cursor.value;
+        if (now - entry.timestamp > entry.ttl) {
+          await cursor.delete();
+        }
+        cursor = await cursor.continue();
+      }
+
+      await tx.done;
+    } catch (err) {
+      console.error('IndexedDB cleanup error:', err);
+    }
+  }
+
+  async ensureSpace(requiredSize: number): Promise<void> {
+    if (!this.db) return;
+
+    try {
+      // Get all entries sorted by timestamp
+      const tx = this.db.transaction(this.storeName, 'readwrite');
+      const index = tx.objectStore(this.storeName).index('by-timestamp');
+      const entries: CacheEntry[] = [];
+
+      let cursor = await index.openCursor();
+      while (cursor) {
+        entries.push(cursor.value);
+        cursor = await cursor.continue();
+      }
+
+      await tx.done;
+
+      // Calculate total size
+      let totalSize = entries.reduce((sum, e) => sum + (e.size || 0), 0);
+
+      // Remove oldest entries if needed
+      if (totalSize + requiredSize > this.maxSize) {
+        const sorted = entries.sort((a, b) => a.timestamp - b.timestamp);
+        
+        for (const entry of sorted) {
+          if (totalSize + requiredSize <= this.maxSize) break;
+          await this.delete(entry.key);
+          totalSize -= (entry.size || 0);
+        }
+      }
+    } catch (err) {
+      console.error('IndexedDB ensureSpace error:', err);
+    }
+  }
+
+  async stats(): Promise<{ count: number; totalSize: number }> {
+    if (!this.db) await this.init();
+    if (!this.db) return { count: 0, totalSize: 0 };
+
+    try {
+      const tx = this.db.transaction(this.storeName, 'readonly');
+      const store = tx.objectStore(this.storeName);
+      
+      let count = 0;
+      let totalSize = 0;
+      
+      let cursor = await store.openCursor();
+      while (cursor) {
+        count++;
+        totalSize += cursor.value.size || 0;
+        cursor = await cursor.continue();
+      }
+
+      await tx.done;
+      return { count, totalSize };
+    } catch (err) {
+      console.error('IndexedDB stats error:', err);
+      return { count: 0, totalSize: 0 };
+    }
+  }
+}
+
+// Export singleton
+export const enhancedCache = new EnhancedIndexedDBCache();
+
+// Auto-cleanup every 5 minutes
+if (typeof window !== 'undefined') {
+  setInterval(() => {
+    enhancedCache.cleanup();
+  }, 5 * 60 * 1000);
+}
+
+export default enhancedCache;
+
+

frontend/src/utils/data/indexedDB.ts

@@ -0,0 +1,262 @@
+/**
+ * IndexedDB utility for client-side caching of embeddings and model data.
+ * Enables offline access and faster subsequent loads.
+ */
+
+const DB_NAME = 'hf_viz_cache';
+const DB_VERSION = 1;
+
+interface CacheEntry<T> {
+  key: string;
+  data: T;
+  timestamp: number;
+  version: string;
+}
+
+const CACHE_VERSION = '1.0.0'; // Increment to invalidate old cache
+const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+const MAX_CACHE_SIZE = 100; // Maximum number of entries per store
+
+class IndexedDBCache {
+  private db: IDBDatabase | null = null;
+  private initPromise: Promise<void> | null = null;
+
+  private async init(): Promise<void> {
+    if (this.db) return;
+    if (this.initPromise) return this.initPromise;
+
+    this.initPromise = new Promise((resolve, reject) => {
+      const request = indexedDB.open(DB_NAME, DB_VERSION);
+
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        this.db = request.result;
+        resolve();
+      };
+
+      request.onupgradeneeded = (event) => {
+        const db = (event.target as IDBOpenDBRequest).result;
+
+        // Create object stores
+        if (!db.objectStoreNames.contains('embeddings')) {
+          db.createObjectStore('embeddings', { keyPath: 'key' });
+        }
+        if (!db.objectStoreNames.contains('reduced_embeddings')) {
+          db.createObjectStore('reduced_embeddings', { keyPath: 'key' });
+        }
+        if (!db.objectStoreNames.contains('models')) {
+          db.createObjectStore('models', { keyPath: 'key' });
+        }
+        if (!db.objectStoreNames.contains('stats')) {
+          db.createObjectStore('stats', { keyPath: 'key' });
+        }
+      };
+    });
+
+    return this.initPromise;
+  }
+
+  private async getStore(storeName: string, mode: 'readonly' | 'readwrite' = 'readonly'): Promise<IDBObjectStore> {
+    await this.init();
+    if (!this.db) throw new Error('Database not initialized');
+
+    const transaction = this.db.transaction([storeName], mode);
+    return transaction.objectStore(storeName);
+  }
+
+  async set<T>(storeName: string, key: string, data: T): Promise<void> {
+    await this.init();
+    if (!this.db) throw new Error('Database not initialized');
+
+    // Prepare entry data
+    const entry: CacheEntry<T> = {
+      key,
+      data,
+      timestamp: Date.now(),
+      version: CACHE_VERSION,
+    };
+
+    // Create a single transaction for all operations
+    const transaction = this.db.transaction([storeName], 'readwrite');
+    const store = transaction.objectStore(storeName);
+    
+    // Queue the put operation FIRST to ensure transaction stays alive
+    // This is critical - by queueing it immediately, we ensure the transaction
+    // has a pending request throughout all other operations
+    const putPromise = new Promise<void>((resolve, reject) => {
+      const request = store.put(entry);
+      request.onsuccess = () => resolve();
+      request.onerror = () => reject(request.error);
+    });
+    
+    // Enforce cache size limit - delete oldest entries if at limit
+    const count = await new Promise<number>((resolve, reject) => {
+      const countRequest = store.count();
+      countRequest.onsuccess = () => resolve(countRequest.result);
+      countRequest.onerror = () => reject(countRequest.error);
+    });
+
+    // Queue delete operations if needed
+    const deletePromises: Promise<void>[] = [];
+    
+    if (count >= MAX_CACHE_SIZE) {
+      // Get all entries, sort by timestamp, delete oldest
+      const allEntries: Array<{ key: string; timestamp: number }> = [];
+      const getAllRequest = store.openCursor();
+      
+      await new Promise<void>((resolve, reject) => {
+        getAllRequest.onsuccess = (event: any) => {
+          const cursor = event.target.result;
+          if (cursor) {
+            const entry = cursor.value as CacheEntry<T>;
+            allEntries.push({ key: entry.key, timestamp: entry.timestamp });
+            cursor.continue();
+          } else {
+            resolve();
+          }
+        };
+        getAllRequest.onerror = () => reject(getAllRequest.error);
+      });
+      
+      // Sort by timestamp (oldest first) and delete excess
+      allEntries.sort((a, b) => a.timestamp - b.timestamp);
+      const toDelete = allEntries.slice(0, count - MAX_CACHE_SIZE + 1);
+      
+      // Queue all delete operations immediately
+      for (const entryToDelete of toDelete) {
+        deletePromises.push(
+          new Promise<void>((resolve, reject) => {
+            const deleteRequest = store.delete(entryToDelete.key);
+            deleteRequest.onsuccess = () => resolve();
+            deleteRequest.onerror = () => reject(deleteRequest.error);
+          })
+        );
+      }
+    }
+    
+    // Wait for both deletes (if any) and put to complete
+    // The put request was queued first, so transaction stays alive throughout
+    await Promise.all([...deletePromises, putPromise]);
+  }
+
+  async get<T>(storeName: string, key: string): Promise<T | null> {
+    const store = await this.getStore(storeName);
+
+    return new Promise((resolve, reject) => {
+      const request = store.get(key);
+      request.onsuccess = async () => {
+        const entry = request.result as CacheEntry<T> | undefined;
+        if (!entry) {
+          resolve(null);
+          return;
+        }
+
+        // Check version
+        if (entry.version !== CACHE_VERSION) {
+          // Version mismatch, delete old entry using readwrite transaction
+          try {
+            await this.delete(storeName, key);
+          } catch (err) {
+            // Ignore delete errors
+          }
+          resolve(null);
+          return;
+        }
+
+        // Check TTL
+        const age = Date.now() - entry.timestamp;
+        if (age > CACHE_TTL_MS) {
+          // Cache expired, delete old entry using readwrite transaction
+          try {
+            await this.delete(storeName, key);
+          } catch (err) {
+            // Ignore delete errors
+          }
+          resolve(null);
+          return;
+        }
+
+        resolve(entry.data);
+      };
+      request.onerror = () => reject(request.error);
+    });
+  }
+
+  async has(storeName: string, key: string): Promise<boolean> {
+    const store = await this.getStore(storeName);
+
+    return new Promise((resolve, reject) => {
+      const request = store.getKey(key);
+      request.onsuccess = () => resolve(request.result !== undefined);
+      request.onerror = () => reject(request.error);
+    });
+  }
+
+  async delete(storeName: string, key: string): Promise<void> {
+    const store = await this.getStore(storeName, 'readwrite');
+
+    return new Promise((resolve, reject) => {
+      const request = store.delete(key);
+      request.onsuccess = () => resolve();
+      request.onerror = () => reject(request.error);
+    });
+  }
+
+  async clear(storeName: string): Promise<void> {
+    const store = await this.getStore(storeName, 'readwrite');
+
+    return new Promise((resolve, reject) => {
+      const request = store.clear();
+      request.onsuccess = () => resolve();
+      request.onerror = () => reject(request.error);
+    });
+  }
+
+  async getCacheSize(storeName: string): Promise<number> {
+    const store = await this.getStore(storeName);
+
+    return new Promise((resolve, reject) => {
+      const request = store.count();
+      request.onsuccess = () => resolve(request.result);
+      request.onerror = () => reject(request.error);
+    });
+  }
+
+  // Helper methods for specific data types
+  async cacheModels(key: string, models: any[]): Promise<void> {
+    return this.set('models', key, models);
+  }
+
+  async getCachedModels(key: string): Promise<any[] | null> {
+    return this.get<any[]>('models', key);
+  }
+
+  async cacheStats(key: string, stats: any): Promise<void> {
+    return this.set('stats', key, stats);
+  }
+
+  async getCachedStats(key: string): Promise<any | null> {
+    return this.get<any>('stats', key);
+  }
+
+  // Generate cache key from filter parameters
+  static generateCacheKey(params: {
+    minDownloads: number;
+    minLikes: number;
+    searchQuery?: string;
+    projectionMethod: string;
+  }): string {
+    return JSON.stringify({
+      minDownloads: params.minDownloads,
+      minLikes: params.minLikes,
+      searchQuery: params.searchQuery || '',
+      projectionMethod: params.projectionMethod,
+    });
+  }
+}
+
+// Export singleton instance
+export const cache = new IndexedDBCache();
+export { IndexedDBCache };
+export default cache;
+

netlify.toml

@@ -5,8 +5,6 @@
 
 [build.environment]
   NODE_VERSION = "18"
-  # Set your Railway backend URL in Netlify dashboard:
-  # Site settings → Environment variables → REACT_APP_API_URL
 
 [[redirects]]
   from = "/*"
@@ -21,9 +19,7 @@
     X-Content-Type-Options = "nosniff"
     Referrer-Policy = "strict-origin-when-cross-origin"
 
-# Cache static assets aggressively
 [[headers]]
   for = "/static/*"
   [headers.values]
     Cache-Control = "public, max-age=31536000, immutable"
-