# Broken link finder/fixer
# 1. Starting with a given page, scan for all references
#    <* href="..."> mainly <a> but also <link>
#    <* src="..."> eg img, script, style
#    <* background="..."> for inline background image styles?
# 2. If ref is external (http:// or https:// and not gsarchive.net), record
#    the domain name in case there's a different sort of error. A separate
#    pass to validate external links will be of value, but should be cached
#    aggressively.
# 3. If ref is unnecessarily-external (pointing to gsarchive.net), autofix
#    to relative.
# 4. If ref is internal and the file exists, queue the target for scanning,
#    if not already queued or scanned.
# 5. If ref is internal and not found:
#    a. <nothing here yet - cf "is it an elephant">
#    b. Log the failure
# 6. Progressively go through the log and add autofixers
from urllib.parse import urlparse, urljoin, ParseResult
from bs4 import BeautifulSoup
root = "/home/rosuav/gsarchive/live"

scanned = { }
awaiting = []

def report(*msg):
	# TODO: Log to file?
	print(*msg)

def fix(oldurl, newurl, context):
	report("In", context, "replace", oldurl, "with", newurl)

def link(context, url, *, base="https://gsarchive.net/"):
	# Make the URL absolute
	print("Got link", context, url)
	uri = urljoin(urljoin(base, context), url)
	fn = None
	match urlparse(uri):
		case ParseResult(scheme="http", netloc="gsarchive.net"):
			report("Non-encrypted link within site", context, url)
		case ParseResult(scheme="http"):
			report("Non-encrypted link outside site", context, url)
		case ParseResult(scheme="https", netloc="gsarchive.net") as p:
			if url.startswith("https:"):
				# The URL was stored absolute, which is inefficient and vulnerable to error
				fix(url, p.path)
			fn = p.path
		case ParseResult(scheme="https"):
			report("External link", context, url)
		case ParseResult():
			report("Non-HTTP link", context, url)
		case _:
			report("Unparseable link", context, url)
	if not fn: return
	if fn in scanned: return
	scanned[fn] = 1
	awaiting.append(fn)

def find_links(fn):
	path = root + fn
	if path.endswith("/"):
		# Directory names get handled by index files
		path += "index.html" # Is this the only name available? If multiple, what priority order?
	with open(path, "rb") as f:
		soup = BeautifulSoup(f.read(), "html.parser")
	print("Got soup")
	for attr in "src", "href", "background":
		for elem in soup.find_all(attrs={attr: True}):
			link(fn, elem.get(attr))

link("/", "/")
while awaiting:
	fn = awaiting.pop()
	print(fn, "...")
	find_links(fn)

# Broken link finder/fixer
# 1. Starting with a given page, scan for all references
#    <* href="..."> mainly <a> but also <link>
#    <* src="..."> eg img, script, style
#    <* background="..."> for inline background image styles?
# 2. If ref is external (http:// or https:// and not gsarchive.net), record
#    the domain name in case there's a different sort of error. A separate
#    pass to validate external links will be of value, but should be cached
#    aggressively.
# 3. If ref is unnecessarily-external (pointing to gsarchive.net), autofix
#    to relative.
# 4. If ref is internal and the file exists, queue the target for scanning,
#    if not already queued or scanned.
# 5. If ref is internal and not found:
#    a. <nothing here yet - cf "is it an elephant">
#    b. Log the failure
# 6. Progressively go through the log and add autofixers
import os
from urllib.parse import urlparse, urljoin, ParseResult
from bs4 import BeautifulSoup
root = "/home/rosuav/gsarchive/live"

scanned = { }
awaiting = []

def report(*msg):
	# TODO: Log to file?
	print(*msg)

def fix(oldurl, newurl, context):
	report("In", context, "replace", oldurl, "with", newurl)

def path_from_fn(fn):
	path = root + fn
	if path.endswith("/"):
		# Directory names get handled by index files
		path += "index.html" # Is this the only name available? If multiple, what priority order?
	return path

def link(context, url, *, base="https://gsarchive.net/"):
	# Make the URL absolute
	print("Got link", context, url)
	uri = urljoin(urljoin(base, context), url)
	fn = None
	match urlparse(uri):
		case ParseResult(scheme="http", netloc="gsarchive.net"):
			report("Non-encrypted link within site", context, url)
		case ParseResult(scheme="http"):
			report("Non-encrypted link outside site", context, url)
		case ParseResult(scheme="https", netloc="gsarchive.net") as p:
			if url.startswith("https:"):
				# The URL was stored absolute, which is inefficient and vulnerable to error
				fix(url, p.path)
			fn = p.path
		case ParseResult(scheme="https"):
			report("External link", context, url)
		case ParseResult():
			report("Non-HTTP link", context, url)
		case _:
			report("Unparseable link", context, url)
	if not fn: return
	if fn in scanned: return
	scanned[fn] = 1
	if not os.path.exists(path_from_fn(fn)):
		report("Internal link not found", context, url)
	base, dot, ext = fn.rpartition(".")
	if not dot or ext in ("html", "htm"):
		awaiting.append(fn)
	# Anything else we should be checking? Scan CSS files for references, maybe?

def find_links(fn):
	with open(path_from_fn(fn), "rb") as f:
		soup = BeautifulSoup(f.read(), "html.parser")
	print("Got soup")
	for attr in "src", "href", "background":
		for elem in soup.find_all(attrs={attr: True}):
			link(fn, elem.get(attr))

link("/", "/")
while awaiting:
	fn = awaiting.pop()
	print(fn, "...")
	find_links(fn)

Check presence of images without recursing into them