#!/usr/bin/env python
# coding: utf-8
testcode/
    spam.py
    fib.py
    grok/
    __init__.py
    blah.py

# spam.py
print("I'm spam")

def hello(name):
    print('Hello %s' % name)

# fib.py
print("I'm fib")

def fib(n):
    if n < 2:
        return 1
    else:
        return fib(n-1) + fib(n-2)

# grok/__init__.py
print("I'm grok.__init__")

# grok/blah.py
print("I'm grok.blah")

from urllib.request import urlopen
u = urlopen('http://localhost:15000/fib.py')
data = u.read().decode('utf-8')
print(data)

import imp
import urllib.request
import sys

def load_module(url):
    u = urllib.request.urlopen(url)
    source = u.read().decode('utf-8')
    mod = sys.modules.setdefault(url, imp.new_module(url))
    code = compile(source, url, 'exec')
    mod.__file__ = url
    mod.__package__ = ''
    exec(code, mod.__dict__)
    return mod

fib = load_module('http://localhost:15000/fib.py')

fib.fib(10)

spam = load_module('http://localhost:15000/spam.py')

spam.hello('Guido')

fib

spam

# urlimport.py
import sys
import importlib.abc
import imp
from urllib.request import urlopen
from urllib.error import HTTPError, URLError
from html.parser import HTMLParser

# Debugging
import logging
log = logging.getLogger(__name__)

# Get links from a given URL
def _get_links(url):
    class LinkParser(HTMLParser):
        def handle_starttag(self, tag, attrs):
            if tag == 'a':
                attrs = dict(attrs)
                links.add(attrs.get('href').rstrip('/'))

    links = set()
    try:
        log.debug('Getting links from %s' % url)
        u = urlopen(url)
        parser = LinkParser()
        parser.feed(u.read().decode('utf-8'))
    except Exception as e:
        log.debug('Could not get links. %s', e)
    log.debug('links: %r', links)
    return links

class UrlMetaFinder(importlib.abc.MetaPathFinder):
    def __init__(self, baseurl):
        self._baseurl = baseurl
        self._links = { }
        self._loaders = { baseurl : UrlModuleLoader(baseurl) }
 
    def find_module(self, fullname, path=None):
        log.debug('find_module: fullname=%r, path=%r', fullname, path)
        if path is None:
            baseurl = self._baseurl
        else:
            if not path[0].startswith(self._baseurl):
                return None
            baseurl = path[0]

        parts = fullname.split('.')
        basename = parts[-1]
        log.debug('find_module: baseurl=%r, basename=%r', baseurl, basename)
 
    # Check link cache
    if basename not in self._links:
        self._links[baseurl] = _get_links(baseurl)
 
    # Check if it's a package
    if basename in self._links[baseurl]:
        log.debug('find_module: trying package %r', fullname)
        fullurl = self._baseurl + '/' + basename
        # Attempt to load the package (which accesses __init__.py)
        loader = UrlPackageLoader(fullurl)
        try:
            loader.load_module(fullname)
            self._links[fullurl] = _get_links(fullurl)
            self._loaders[fullurl] = UrlModuleLoader(fullurl)
            log.debug('find_module: package %r loaded', fullname)
        except ImportError as e:
            log.debug('find_module: package failed. %s', e)
            loader = None
        return loader
 
    # A normal module
    filename = basename + '.py'
    if filename in self._links[baseurl]:
        log.debug('find_module: module %r found', fullname)
        return self._loaders[baseurl]
    else:
        log.debug('find_module: module %r not found', fullname)
        return None

    
    def invalidate_caches(self):
        log.debug('invalidating link cache')
        self._links.clear()

# Module Loader for a URL
class UrlModuleLoader(importlib.abc.SourceLoader):
    def __init__(self, baseurl):
        self._baseurl = baseurl
        self._source_cache = {}
 
    def module_repr(self, module):
        return '<urlmodule %r from %r>' % (module.__name__, module.__file__)
 
    # Required method
    def load_module(self, fullname):
        code = self.get_code(fullname)
        mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
        mod.__file__ = self.get_filename(fullname)
        mod.__loader__ = self
        mod.__package__ = fullname.rpartition('.')[0]
        exec(code, mod.__dict__)
        return mod
 
    # Optional extensions
    def get_code(self, fullname):
        src = self.get_source(fullname)
        return compile(src, self.get_filename(fullname), 'exec')
 
    def get_data(self, path):
        pass
 
    def get_filename(self, fullname):
        return self._baseurl + '/' + fullname.split('.')[-1] + '.py'
 
    def get_source(self, fullname):
        filename = self.get_filename(fullname)
        log.debug('loader: reading %r', filename)
        if filename in self._source_cache:
            log.debug('loader: cached %r', filename)
            return self._source_cache[filename]
        try:
            u = urlopen(filename)
            source = u.read().decode('utf-8')
            log.debug('loader: %r loaded', filename)
            self._source_cache[filename] = source
            return source
        except (HTTPError, URLError) as e:
            log.debug('loader: %r failed. %s', filename, e)
            raise ImportError("Can't load %s" % filename)
        
    def is_package(self, fullname):
        return False

# Package loader for a URL
class UrlPackageLoader(UrlModuleLoader):
    def load_module(self, fullname):
        mod = super().load_module(fullname)
        mod.__path__ = [ self._baseurl ]
        mod.__package__ = fullname
    def get_filename(self, fullname):
        return self._baseurl + '/' + '__init__.py'
 
    def is_package(self, fullname):
        return True

# Utility functions for installing/uninstalling the loader
_installed_meta_cache = { }
def install_meta(address):
    if address not in _installed_meta_cache:
        finder = UrlMetaFinder(address)
        _installed_meta_cache[address] = finder
        sys.meta_path.append(finder)
        log.debug('%r installed on sys.meta_path', finder)

def remove_meta(address):
    if address in _installed_meta_cache:
        finder = _installed_meta_cache.pop(address)
        sys.meta_path.remove(finder)
        log.debug('%r removed from sys.meta_path', finder)

# importing currently fails
import fib

# Load the importer and retry (it works)
import urlimport
urlimport.install_meta('http://localhost:15000')
import fib

import spam

import grok.blah

# urlimport.py

# ... include previous code above ...

# Path finder class for a URL
class UrlPathFinder(importlib.abc.PathEntryFinder):
    def __init__(self, baseurl):
        self._links = None
        self._loader = UrlModuleLoader(baseurl)
        self._baseurl = baseurl
 
    def find_loader(self, fullname):
        log.debug('find_loader: %r', fullname)
        parts = fullname.split('.')
        basename = parts[-1]
        # Check link cache
        if self._links is None:
            self._links = [] # See discussion
            self._links = _get_links(self._baseurl)
 
        # Check if it's a package
        if basename in self._links:
            log.debug('find_loader: trying package %r', fullname)
            fullurl = self._baseurl + '/' + basename
            # Attempt to load the package (which accesses __init__.py)
            loader = UrlPackageLoader(fullurl)
            try:
                loader.load_module(fullname)
                log.debug('find_loader: package %r loaded', fullname)
            except ImportError as e:
                log.debug('find_loader: %r is a namespace package', fullname)
                loader = None
            return (loader, [fullurl])
 

        # A normal module
        filename = basename + '.py'
        if filename in self._links:
            log.debug('find_loader: module %r found', fullname)
            return (self._loader, [])
        else:
            log.debug('find_loader: module %r not found', fullname)
            return (None, [])
 
    def invalidate_caches(self):
        log.debug('invalidating link cache')
        self._links = None
        
# Check path to see if it looks like a URL
_url_path_cache = {}
def handle_url(path):
    if path.startswith(('http://', 'https://')):
        log.debug('Handle path? %s. [Yes]', path)
        if path in _url_path_cache:
            finder = _url_path_cache[path]
        else:
            finder = UrlPathFinder(path)
            _url_path_cache[path] = finder
        return finder
    else:
        log.debug('Handle path? %s. [No]', path)

def install_path_hook():
    sys.path_hooks.append(handle_url)
    sys.path_importer_cache.clear()
    log.debug('Installing handle_url')

def remove_path_hook():
    sys.path_hooks.remove(handle_url)
    sys.path_importer_cache.clear()
    log.debug('Removing handle_url')

import fib

import urlimport

urlimport.install_path_hook()

# Add an entry to sys.path and watch it work
import sys
sys.path.append('http://localhost:15000')
import fib

import grok.blah

grok.blah.__file__

import imp
m = imp.new_module('spam')
m

m.__name__

import sys
import imp
m = sys.modules.setdefault('spam', imp.new_module('spam'))
m

import math
m = sys.modules.setdefault('math', imp.new_module('math'))
m


# m.sin(2)
m.cos(2)

from pprint import pprint
pprint(sys.meta_path)

class Finder:
    def find_module(self, fullname, path):
        print('Looking for', fullname, path)
        return None

import sys
sys.meta_path.insert(0, Finder()) # Insert as first entry
import math

import types

import threading

import xml.etree.ElementTree

del sys.meta_path[0]
sys.meta_path.append(Finder())
import urllib.request
import datetime

from pprint import pprint
import sys
pprint(sys.path)

pprint(sys.path_importer_cache)

class Finder:
    def find_loader(self, name):
        print('Looking for', name)
        return (None, [])

import sys
# Add a "debug" entry to the importer cache
sys.path_importer_cache['debug'] = Finder()
# Add a "debug" directory to sys.path
sys.path.insert(0, 'debug')
import threading

sys.path_importer_cache.clear()
def check_path(path):
    print('Checking', path)
    raise ImportError()

sys.path_hooks.insert(0, check_path)
import fib

def check_url(path):
    if path.startswith('http://'):
        return Finder()
    else:
        raise ImportError()

sys.path.append('http://localhost:15000')
sys.path_hooks[0] = check_url
import fib

import xml.etree.ElementTree
xml.__path__
xml.etree.__path__

# Check link cache
if self._links is None:
    self._links = [] # See discussion
    self._links = _get_links(self._baseurl)

import logging
logging.basicConfig(level=logging.DEBUG)
import urlimport
urlimport.install_path_hook()

import fib

import sys

import fib

