old_contents: import climate
  import logging
  import numpy as np
  import os
  import pickle
  import theanets
  
  
  def compress(source, k, key):
      logging.info('%s: looking for %s', source, key)
  
      filenames = sorted(
          fn for fn in os.listdir(source)
          if fn.endswith('{}.npy'.format(key)))
      arrays = [
          np.clip(np.load(os.path.join(source, fn), mmap_mode='r'), -1, 1)
          for fn in filenames]
      B, N = 128, arrays[0].shape[1]
  
      logging.info('loaded %s rows of %d-D data from %d files',
                   sum(len(a) for a in arrays), N, len(arrays))
  
      def batch():
          batch = np.zeros((B, N), 'f')
          for b in range(B):
              a = np.random.randint(len(arrays))
              j = np.random.randint(len(arrays[a]))
              batch[b] = arrays[a][j]
          return [batch]
  
      pca = theanets.Autoencoder([N, (k, 'linear'), (N, 'tied')])
      pca.train(batch, momentum=0.9)
  
      for arr, fn in zip(arrays, filenames):
          out = os.path.join(source, fn.replace(key, '{}_{}'.format(key, k)))
          karr = pca.encode(arr.astype('f'))
          logging.info('%s: saving %s', out, karr.shape)
          np.save(out, karr)
  
      out = os.path.join(source, 'pca_{}_{}.pkl'.format(key, k))
      pickle.dump(pca, open(out, 'wb'))
  
  
  @climate.annotate(
      root='load data files from subject directories in this path',
      k=('compress to this many dimensions', 'option', None, int),
  )
  def main(root, k=400):
      for subject in sorted(os.listdir(root)):
          compress(os.path.join(root, subject), k, 'fjac')
          compress(os.path.join(root, subject), k, 'ijac')
  
  
  if __name__ == '__main__':
      climate.call(main)
  
new_contents: import climate
  import glob
  import gzip
  import io
  import lmj.cubes
  import logging
  import numpy as np
  import os
  import pandas as pd
  import pickle
  import theanets
  
  
  def compress(source, k, activation, **kwargs):
      fns = sorted(glob.glob(os.path.join(source, '*', '*_jac.csv.gz')))
      logging.info('%s: found %d jacobians', source, len(fns))
  
      # the clipping operation affects about 2% of jacobian values.
      dfs = [np.clip(pd.read_csv(fn, index_col='time').dropna(), -10, 10)
             for fn in fns]
  
      B, N = 128, dfs[0].shape[1]
  
      logging.info('loaded %s rows of %d-D data from %d files',
                   sum(len(df) for df in dfs), N, len(dfs))
  
      def batch():
          batch = np.zeros((B, N), 'f')
          for b in range(B):
              a = np.random.randint(len(dfs))
              batch[b] = dfs[a].iloc[np.random.randint(len(dfs[a])), :]
          return [batch]
  
      pca = theanets.Autoencoder([N, (k, activation), (N, 'tied')])
      pca.train(batch, **kwargs)
  
      key = '{}_k{}'.format(activation, k)
      if 'hidden_l1' in kwargs:
          key += '_s' + kwargs['hidden_l1']
  
      for df, fn in zip(dfs, fns):
          df = pd.DataFrame(pca.encode(df), index=df.index)
          s = io.StringIO()
          df.to_csv(s, index_label='time')
          out = fn.replace('_jac', '_jac_' + key)
          with gzip.open(out, 'wb') as handle:
              handle.write(s.getvalue().encode('utf-8'))
          logging.info('%s: saved %s', out, df.shape)
  
      out = os.path.join(source, 'pca_{}.pkl'.format(key))
      pickle.dump(pca, open(out, 'wb'))
  
  
  @climate.annotate(
      root='load data files from subject directories in this path',
      k=('compress to this many dimensions', 'option', None, int),
      activation=('use this activation function', 'option'),
  )
  def main(root, k=1000, activation='relu'):
      for subject in lmj.cubes.Experiment(root).subjects:
          compress(subject.root, k, activation,
                   momentum=0.9,
                   hidden_l1=0.9,
                   monitors={'hid1:out': (0.1, 0.5, 0.9)})
  
  
  if __name__ == '__main__':
      climate.call(main)
  
subject: Change compression to load data from CSVs.