old_contents: # -*- coding: utf-8 -*-
  
  from __future__ import division
  import sys
  from math import ceil
  
  from collections import defaultdict
  
  import codecs
  import json
  
  from multiprocessing import Pool as ThreadPool
  
  
  def stats(page, source, target):
  
    if target in ignore_langs:
      return
  
    with codecs.open("data/{1}.{0}/{1}.json".format(page, source, target), "r", "utf-8-sig") as f:
      left_links = json.load(f)
  
    with codecs.open("data/{1}.{0}/{1}.{2}.json".format(page, source, target), "r", "utf-8-sig") as f:
      left_links_translated = json.load(f)
  
    with codecs.open("data/{1}.{0}/{2}.json".format(page, source, target), "r", "utf-8-sig") as f:
      right_links = json.load(f)
  
    with codecs.open("data/{1}.{0}/{2}.{1}.json".format(page, source, target), "r", "utf-8-sig") as f:
      right_links_translated = json.load(f)
  
    # print set(left_links) - { x[0] for x in left_links_translated.items() }
  
    stats = {
      "left": len(left_links),
      "right": len(right_links),
      "left_untranslated": len(set(left_links) - { x[0] for x in left_links_translated.items() }),
      "left_absent": len( { x[1] for x in left_links_translated.items() } - set(right_links)),
      "intersection": len({ x[0] for x in left_links_translated.items()  } & { x[1] for x in right_links_translated.items() }),
      "right_absent": len(  { x[1] for x in right_links_translated.items() } - set(left_links)),
      "right_untranslated": len(set(right_links) - { x[0] for x in right_links_translated.items() })
    }
  
    return stats
  
  def compute_stats(source):
    lang = "en"
  
    if "#" in source:
      lang = source.split("#")[1]
      source = source.split("#")[0]
  
    langlinks = json.load(codecs.open("data/{1}.{0}.json".format(source, lang), "r", "utf-8-sig"))
  
    result = { l: stats(source, lang, l) for l in langlinks.keys() }
  
    with codecs.open("data/{1}.{0}.stats.json".format(source, lang), "w", "utf-8-sig") as f:
      json.dump(result, f, ensure_ascii=False, indent=2, separators=(',', ': '))
  
  ignore_langs = [ "th" ]
  ignore_langs = []
  
  if __name__ == "__main__":
    if len(sys.argv) < 2:
      sources = ["Love", "Revolution", "Wisdom", "Ethics", "Morality", "Surveillance"]
      sources = [ "Russia", "Crimea", "Ukraine"]
    else:
      sources = sys.argv[1:]
      print sources
  
    map(compute_stats, sources)
  
new_contents: # -*- coding: utf-8 -*-
  
  from __future__ import division
  import sys
  from math import ceil
  
  from collections import defaultdict
  
  import codecs
  import json
  
  from multiprocessing import Pool as ThreadPool
  
  
  def stats(page, source, target):
  
    if target in ignore_langs:
      return
  
    with codecs.open(u"data/{1}.{0}/{1}.json".format(page, source, target), "r", "utf-8-sig") as f:
      left_links = json.load(f)
  
    with codecs.open(u"data/{1}.{0}/{1}.{2}.json".format(page, source, target), "r", "utf-8-sig") as f:
      left_links_translated = json.load(f)
  
    with codecs.open(u"data/{1}.{0}/{2}.json".format(page, source, target), "r", "utf-8-sig") as f:
      right_links = json.load(f)
  
    with codecs.open(u"data/{1}.{0}/{2}.{1}.json".format(page, source, target), "r", "utf-8-sig") as f:
      right_links_translated = json.load(f)
  
    # print set(left_links) - { x[0] for x in left_links_translated.items() }
  
    stats = {
      "left": len(left_links),
      "right": len(right_links),
      "left_untranslated": len(set(left_links) - { x[0] for x in left_links_translated.items() }),
      "left_absent": len( { x[1] for x in left_links_translated.items() } - set(right_links)),
      "intersection": len({ x[0] for x in left_links_translated.items()  } & { x[1] for x in right_links_translated.items() }),
      "right_absent": len(  { x[1] for x in right_links_translated.items() } - set(left_links)),
      "right_untranslated": len(set(right_links) - { x[0] for x in right_links_translated.items() })
    }
  
    return stats
  
  def compute_stats(source):
    lang = "en"
  
    if "#" in source:
      lang = source.split("#")[1]
      source = source.split("#")[0]
  
    langlinks = json.load(codecs.open(u"data/{1}.{0}.json".format(source, lang), "r", "utf-8-sig"))
  
    result = { l: stats(source, lang, l) for l in langlinks.keys() }
  
    with codecs.open(u"data/{1}.{0}.stats.json".format(source, lang), "w", "utf-8-sig") as f:
      json.dump(result, f, ensure_ascii=False, indent=2, separators=(',', ': '))
  
  ignore_langs = [ "th" ]
  ignore_langs = []
  
  if __name__ == "__main__":
    if len(sys.argv) < 2:
      sources = ["Love", "Revolution", "Wisdom", "Ethics", "Morality", "Surveillance"]
      sources = [ "Russia", "Crimea", "Ukraine"]
    else:
      sources = sys.argv[1:]
      print sources
  
    map(compute_stats, sources)
  
subject: add dataset about Europe in 5 languages + update macro scripts