# Beautiful Soup Documentation
# https://www.crummy.com/software/BeautifulSoup/bs4/doc/

print("Making the soup\n")
print("    To parse a document, pass it into the BeautifulSoup constructor. You can pass in a")
print("    string or an open filehandle:\n")

print("""        from bs4 import BeautifulSoup
        from bs4 import BeautifulSoup

        with open("index.html") as fp:
            soup = BeautifulSoup(fp)

        soup = BeautifulSoup("<html>data</html>")
        """)
print()
print("    First, the document is converted to Unicode, and HTML entities are converted to")
print("    Unicode characters:\n")

print("""        BeautifulSoup("Sacr&eacute; bleu!")
        <html><head></head><body>Sacré bleu!</body></html>
    """)
print()
print("    Beautiful Soup then parses the document using the best available parser. It will")
print("    use an HTML parser unless you specifically tell it to use an XML parser.")
print("    (See Parsing XML.)\n")

# Import the BeautifulSoup library
from bs4 import BeautifulSoup

print("Kinds of objects\n")

print("1.  Tag")
print()
print("    A Tag object corresponds to an XML or HTML tag in the original document:\n")
print("""        soup = BeautifulSoup('<b class="boldest">Extremely bold</b>', "html.parser" """)
soup = BeautifulSoup('<b class="boldest">Extremely bold</b>', "html.parser")
tag = soup.b
print(f"        type(tag): {type(tag)}")		# <class 'bsr.element.Tag'>
print()

print("    a.  Name\n")
print("        * Every tag has a name, accessible as .name:")
print(f"        * tag.name: {tag.name}\n")
print("        * If you change a tag’s name, the change will be reflected in any HTML markup")
print("          generated by Beautiful Soup:\n")

print("          tag.name = 'blockquote'")
tag.name = "blockquote"
print(f"          tag: {tag}\n")				# <blockquote class="boldest">Extremely bold</blockquote>

print("    b.  Attributes\n")
print('        A tag may have any number of attributes. The tag <b id="boldest"> has an')
print('        "attribute “id” whose value is “boldest”.')
print("        You can access a tag’s attributes by treating the tag like a dictionary:\n")

print("""            soup = BeautifulSoup('<b id="boldest">Extremely bold</b>', "html.parser"
            tag = soup.b
""")
soup = BeautifulSoup('<b id="boldest">Extremely bold</b>', "html.parser")
tag = soup.b
print(f"            tag['id']: {tag['id']}\n")		# 'boldest'

print("        You can access that dictionary directly as .attrs:\n")

print(f"            tag.attrs: {tag.attrs}\n")		# {'id': 'boldest'}

print("        You can add, remove, and modify a tag’s attributes. Again, this is done by")
print("        treating the tag as a dictionary:\n")

print("            tag['id'] = 'verybold'")
tag['id'] = 'verybold'

print("            tag['another-attribute'] = 1")
tag['another-attribute'] = 1

print(f"            tag: {tag}\n")				# <b another-attribute="1" id="verybold"></b>

print("            del tag['id']")
print("            del tag['another-attribute']")
del tag['id']
del tag['another-attribute']

print(f"            tag: {tag}\n")						# <b></b>

print("          Since we deleted the 'id' above, the following will produce a KeyError:\n")

print("              print(tag['id']): KeyError 'id'")
print(f"              tag.get('id'): {tag.get('id')}\n\n\n")	#None


print("        - Multi-valued attributes\n")

print("           - HTML 4 defines a few attributes that can have multiple values. HTML 5 removes a couple of them,")
print("             but defines a few more. The most common multi-valued attribute is class (that is, a tag can have")
print("             more than one CSS class). Others include rel, rev, accept-charset, headers, and accesskey.")
print("             Beautiful Soup presents the value(s) of a multi-valued attribute as a list:\n")

print("""                 css_soup = BeautifulSoup('<p class="body"></p>', 'html.parser')""")
css_soup = BeautifulSoup('<p class="body"></p>', 'html.parser')
print(f"                 css_soup.p['class']: {css_soup.p['class']}\n")			# ["body"]

print("""                 css_soup = BeautifulSoup('<p class="body strikeout"></p>', 'html.parser')""")
css_soup = BeautifulSoup('<p class="body strikeout"></p>', 'html.parser')
print(f"                     css_soup.p['class']: {css_soup.p['class']}\n")			# ["body", "strikeout"]

print("           - If an attribute looks like it has more than one value, but it’s not a multi-valued")
print("             attribute as defined by any version of the HTML standard, Beautiful Soup will leave")
print("             the attribute alone:\n")

print("""                 id_soup = BeautifulSoup('<p id="my id"></p>', 'html.parser')""")
id_soup = BeautifulSoup('<p id="my id"></p>', 'html.parser')
print(f"                 id_soup.p['id']: {id_soup.p['id']}\n")				# 'my id'

print("           - When you turn a tag back into a string, multiple attribute values are consolidated:\n")
print("""                 rel_soup = BeautifulSoup('<p>Back to the <a rel="index">homepage</a></p>','html.parser')""")
rel_soup = BeautifulSoup('<p>Back to the <a rel="index">homepage</a></p>','html.parser')
print(f"                 rel_soup.a['rel']:   {rel_soup.a['rel']}")			# ['index']
print("                 rel_soup.a['rel'] = ['index', 'contents']")
rel_soup.a['rel'] = ['index', 'contents']
print(f"                 rel_soup.p: {rel_soup.p}\n")	# <p>Back to the <a rel="index contents">homepage</a></p>

print("           - You can use `get_attribute_list to get a value that’s always a list, string,")
print("             whether or not it’s a multi-valued atribute\n")

print(f"                 id_soup.p.get_attribute_list('id'): {id_soup.p.get_attribute_list('id')}\n")
	                       # [“my id”]
print("           - If you parse a document as XML, there are no multi-valued attributes:\n")
print("""             xml_soup = BeautifulSoup('<p class="body strikeout"></p>','xml')""")
# xml_soup = BeautifulSoup('<p class="body strikeout"></p>','xml')
# # # print(f"   {xml_soup.p['class']}")			# u'body strikeout'
print()

print("2.  NavigableString\n")

print("    A string corresponds to a bit of text within a tag. Beautiful Soup uses the")
print("    NavigableString class to contain these bits of text:\n")

print(f"        tag.string: {tag.string}")			        # u'Extremely bold'
print(f"        type(tag.strinng){type(tag.string)}\n\n")   # <class 'bs4.element.NavigableString'>

print("    A NavigableString is just like a Python Unicode string, except that it also")
print("    supports some of the features described in Navigating the tree and Searching")
print("    the tree. You can convert a NavigableString to a Unicode string with unicode():")
print("    Python 3 renamed the unicode type to str, the old str type has been replaced by bytes.\n")

print("        unicode_string = str(tag.string)")
unicode_string = str(tag.string)
print(f"        unicode_string: {unicode_string}")				# 'Extremely bold'
print(f"        type(unicode_string): {type(unicode_string)}\n")			# <class 'str'>

print("    You can’t edit a string in place, but you can replace one string with another,")
print("    using replace_with():\n")

print('        tag.string.replace_with("No longer bold")')
tag.string.replace_with("No longer bold")
print(f"        tag: {tag}\n\n")			# <blockquote>No longer bold</blockquote>


print("3.  BeautifulSoup\n")
print("    The BeautifulSoup object itself represents the document as a whole. For most")
print("    purposes, you can treat it as a Tag object. This means it supports most of")
print("    the methods described in Navigating the tree and Searching the tree.\n")

print("    Since the BeautifulSoup object doesn’t correspond to an actual HTML or XML")
print("    tag, it has no name and no attributes. But sometimes it’s useful to look at")
print('    its .name, so it’s been given the special .name “[document]”\n')

print(f"        soup.name: {soup.name}\n\n")	# [document]


print("4.  Comments and other special strings\n")

print("    Tag, NavigableString, and BeautifulSoup cover almost everything you’ll see")
print("    in an HTML or XML file, but there are a few leftover bits. The only one")
print("    you’ll probably ever need to worry about is the comment:\n")

print("""        markup = "<b><!--Hey, buddy. Want to buy a used parser?--></b>"
        soup = BeautifulSoup(markup, 'html.parser')
        comment = soup.b.string""")
markup = "<b><!--Hey, buddy. Want to buy a used parser?--></b>"
soup = BeautifulSoup(markup, 'html.parser')
comment = soup.b.string

print(f"        type(comment): {type(comment)}\n")	# <class 'bs4.element.Comment'>

print("    The Comment object is just a special type of NavigableString:")
print(f"        comment: {comment}\n")	# u'Hey, buddy. Want to buy a used parser'

print("    But when it appears as part of an HTML document")
print("    a Comment is displayed with special formatting:")
print(f"\nsoup.b.prettify(): {soup.b.prettify()}\n")
                # <b>
                #  <!--Hey, buddy. Want to buy a used parser?-->
                # </b>

print("    Beautiful Soup defines classes for anything else that might show up in an")
print("    XML document: CData, ProcessingInstruction, Declaration, and Doctype. Just")
print("    like Comment, these classes are subclasses of NavigableString that add")
print("    something extra to the string. Here’s an example that replaces the comment")
print("    with a CDATA block:")

print("""
        from bs4 import CData
        cdata = CData("A CDATA block")
        comment.replace_with(cdata)
        print(soup.b.prettify())
""")
from bs4 import CData
cdata = CData("A CDATA block")
comment.replace_with(cdata)
print(soup.b.prettify())    # <b>
                            #  <![CDATA[A CDATA block]]>
                            # </b>
print("\n\n\n")


print("Navigating the tree\n")

print("""    Here’s the “Three sisters” HTML document again:\n

        <html><head><title>The Dormouse's story</title></head>
        <body>
        <p class="title"><b>The Dormouse's story</b></p>
        <p class="story">Once upon a time there were three little sisters; and their names were
        <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
        <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
        <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
        and they lived at the bottom of a well.</p>
        <p class="story">...</p>
    """)

html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>

<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""

print("        soup = BeautifulSoup(html_doc, 'html.parser')")
soup = BeautifulSoup(html_doc, 'html.parser')

print("\n    I’ll use this as an example to show you how to move from one part of a document to another.")

print("1. Going down\n")

print("    Tags may contain strings and other tags. These elements are the tag’s children.")
print("    Beautiful Soup provides a lot of different attributes for navigating and")
print("    iterating over a tag’s children.\n")

print("    Note that Beautiful Soup strings don’t support any of these attributes, because a")
print("    string can’t have children.\n")

print("    a. Navigating using tag names\n")
print("       The simplest way to navigate the parse tree is to say the name of the tag you")
print("       want. If you want the <head> tag, just say soup.head:")
print(f"           soup.head: {soup.head}")		# <head><title>The Dormouse's story</title></head>
print(f"           soup.title: {soup.title}\n")	# <title>The Dormouse's story</title>

print("       You can do use this trick again and again to zoom in on a certain part of the parse")
print("       tree. This code gets the first <b> tag beneath the <body> tag:")
print(f"            soup.body: {soup.body.b}\n")	# <b>The Dormouse's story</b>

print("       Using a tag name as an attribute will give you only the first tag by that name:")
print(f"       soup.a: {soup.a}\n")		# <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>

print("       If you need to get all the <a> tags, or anything more complicated than the first")
print("       tag with a certain name, you’ll need to use one of the methods described in")
print("       Searching the tree, such as find_all():\n")

print(f"           soup.find_all('a'): {soup.find_all('a')}\n\n")
                            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
                            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
                            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("           - .contents and .children\n")

print("               A tag’s children are available in a list called .contents:\n")

print("                   head_tag = soup.head")
head_tag = soup.head
print(f"                  soup.head: {soup.head}")
print(f"                  head_tag: {head_tag}\n")	# <head><title>The Dormouse's story</title></head>

print(f"                  soup.head.contents: {soup.head.contents}")
print(f"                  head_tag.contents: {head_tag.contents}")	# [<title>The Dormouse's story</title>]

print("                  title_tag = head_tag.contents[0]")
title_tag = head_tag.contents[0]
print(f"                  title_tag: {title_tag}\n")	# <title>The Dormouse's story</title>

print(f"                  title_tag.contents: {title_tag.contents}\n")

print("               The BeautifulSoup object itself has children. In this case, the <html> tag")
print("               is the child of the BeautifulSoup object.")
print(f"                   Len(soup.contents) = {len(soup.contents)}")	# 1
print(f"                   soup.contents[0] = {soup.contents[0].name}")	# None
print(f"                   soup.contents[1] = {soup.contents[1].name}\n")	# 'html'
print("               A string does not have .contents, because it can’t contain anything:")
print(f"                   text = title_tag.contents[0]")
text = title_tag.contents[0]
print(f"                   text: {text}")
print(f"                   text.contents: # AttributeError: 'NavigableString' object has no attribute 'contents'\n")

print("               Instead of getting them as a list, you can iterate over a tag’s children")
print("               using the .children generator:\n")

print("               Iterating over a tag's children with .children")
print("                   for child in title_tag.children:")
print("                       print(child)\n")

for child in title_tag.children:
    print(f"                   child: {child}")		# The Dormouse's story
print('\n\n')

print("           - .descendants\n")

print("             The .contents and .children attributes only consider a tag’s direct children.")
print("             For instance, the <head> tag has a single direct child– the <title> tag:\n")

print(f"                 head_tag.contents: {head_tag.contents}\n")	# [<title>The Dormouse's story</title>]

print('             But the <title> tag itself has a child: the string “The Dormouse’s story”.')
print("             There’s a sense in which that string is also a child of the <head> tag.")
print("             The .descendants attribute lets you iterate over all of a tag’s children,")
print("             recursively: its direct children, the children of its direct children, and so on:\n")

print("                 for child in head_tag.descendants:")
print("                     print(child)")	                 # <title>The Dormouse's story</title>
print()
for child in head_tag.descendants:
    print(f"                 child: {child}")	# <title>The Dormouse's story</title>
 									# The Dormouse's story
print()
print("             The <head> tag has only one child, but it has two descendants: the <title> tag")
print("             and the <title> tag’s child. The BeautifulSoup object only has one direct")
print("             child (the <html> tag), but it has a whole lot of descendants:\n")

print(f"                 len(list(soup.children)): {len(list(soup.children))}")         # 2
print(f"                 len(list(soup.descendants)): {len(list(soup.descendants))}\n")   # 25

print("           - .string\n")

print("             If a tag has only one child, and that child is a NavigableString, the child")
print("             is made available as .string:\n")

print(f"                 title_tag.string: {title_tag.string}\n")    # u'The Dormouse's story'

print("             If a tag’s only child is another tag, and that tag has a .string, then the")
print("             parent tag is considered to have the same .string as its child:\n")

print(f"                 head_tag.contents: {head_tag.contents}")	# [<title>The Dormouse's story</title>]
print(f"                 head_tag.string: {head_tag.string}\n")      # u'The Dormouse's story'

print("             If a tag contains more than one thing, then it’s not clear what .string")
print("             should refer to, so .string is defined to be None:\n")

print(f"                 print(soup.html.string): {print(soup.html.string)}\n")  # None

print("           - .strings and stripped_strings\n")

print("                 If there’s more than one thing inside a tag, you can still look at just the")
print("                 strings. Use the .strings generator:\n")

print("                     for string in soup.strings:")
print("                         print(repr(string))\n")

for string in soup.strings:
    print(f"                         {repr(string)}")
#                     # u"The Dormouse's story"
#                     # u'\n\n'
#                     # u"The Dormouse's story"
#                     # u'\n\n'
#                     # u'Once upon a time there were three little sisters; and their names were\n'
#                     # u'Elsie'
#                     # u',\n'
#                     # u'Lacie'
#                     # u' and\n'
#                     # u'Tillie'
#                     # u';\nand they lived at the bottom of a well.'
#                     # u'\n\n'
#                     # u'...'
#                     # u'\n'
print()
print("                 These strings tend to have a lot of extra whitespace, which you can remove by")
print("                 using the .stripped_strings generator instead:")
print()
print("                 for string in soup.stripped_strings:")
print("                     print(repr(string))\n")

for string in soup.stripped_strings:
    print(f"                     repr(string): {repr(string)}")
#                     # u"The Dormouse's story"
#                     # u"The Dormouse's story"
#                     # u'Once upon a time there were three little sisters; and their names were'
#                     # u'Elsie'
#                     # u','
#                     # u'Lacie'
#                     # u'and'
#                     # u'Tillie'
#                     # u';\nand they lived at the bottom of a well.'
#                     # u'...'
print()
print("                 Here, strings consisting entirely of whitespace are ignored, and whitespace at")
print("                 the beginning and end of strings is removed.\n\n\n")

print("2.  Going up\n")

print('   Continuing the “family tree” analogy, every tag and every string has a parent:')
print("   the tag that contains it.\n")

print("    a.  .parent\n")

print("        You can access an element’s parent with the .parent attribute. In the example")
print('        “three sisters” document, the <head> tag is the parent of the <title> tag:')
print("            title_tag = soup.title")
title_tag = soup.title
print(f"            title_tag: {title_tag}")   # <title>The Dormouse's story</title>
print(f"            title_tag.parent: {title_tag.parent}\n") # <head><title>The Dormouse's story</title></head>

print("             The title string itself has a parent: the <title> tag that contains it:")
print(f"                 title_tag.string.parent: {title_tag.string.parent}\n")   # <title>The Dormouse's story</title>

print("             The parent of a top-level tag like <html> is the BeautifulSoup object itself:")
print("                 html_tag = soup.html")
html_tag = soup.html
print(f"                 type(html_tag.parent): {type(html_tag.parent)}\n")   # <class 'bs4.BeautifulSoup'>

print("             And the .parent of a BeautifulSoup object is defined as None:")
print(f"                 soup.parent: {soup.parent}\n\n") # None

print("    b.  .parents\n")

print("        You can iterate over all of an element’s parents with .parents. This example")
print("        uses .parents to travel from an <a> tag buried deep within the document, to")
print("        the very top of the document:")
print("            link = soup.a")
link = soup.a
print(f"            link: {link}\n")    # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>

print("""        for parent in link.parents:
            if parent is None:
                print(parent)
            else:
                print(parent.name)
""")
for parent in link.parents:
    if parent is None:
        print(f"              {parent}")
    else:
        print(f"            {parent.name}")
                # p
                # body
                # html
                # [document]
                # None
print('\n\n')
print("3.  Going sideways\n")

print("    Consider a simple document like this:")

print('        sibling_soup = BeautifulSoup("<a><b>text1</b><c>text2</c></b></a>"\n')
sibling_soup = BeautifulSoup("<a><b>text1</b><c>text2</c></b></a>", 'html.parser')

print("        sibling_soup.prettify()")
print(sibling_soup.prettify())
            # <html>
            #  <body>
            #   <a>
            #    <b>
            #     text1
            #    </b>
            #    <c>
            #     text2
            #    </c>
            #   </a>
            #  </body>
            # </html>
print()

print("    The <b> tag and the <c> tag are at the same level: they’re both direct children of the")
print("    same tag. We call them siblings. When a document is pretty-printed, siblings show up at")
print("    the same indentation level. You can also use this relationship in the code you write.\n")

print("    a.  .next_sibling and .previous_sibling\n")

print("        You can use .next_sibling and .previous_sibling to navigate between page elements")
print("        that are on the same level of the parse tree:")
print("            sibling_soup.b.next_sibling: {sibling_soup.b.next_sibling}")   # <c>text2</c>
print("            sibling_soup.c.previous_sibling: {sibling_soup.c.previous_sibling}\n")   # <b>text1</b>

print("        The <b> tag has a .next_sibling, but no .previous_sibling, because there’s nothing")
print("        before the <b> tag on the same level of the tree. For the same reason, the <c> tag")
print("        has a .previous_sibling but no .next_sibling:")

print(f"            sibling_soup.b.previous_sibling: {sibling_soup.b.previous_sibling}")    # None
print(f"            sibling_soup.c.next_sibling: {sibling_soup.c.next_sibling}\n")    # None

print("        The strings “text1” and “text2” are not siblings, because they don’t have the same")
print("        parent:")
print(f"            sibling_soup.b.string: {sibling_soup.b.string}")    # u'text1'
print(f"            sibling_soup.b.string.next_sibling: {sibling_soup.b.string.next_sibling}\n")    # None

print("        In real documents, the .next_sibling or .previous_sibling of a tag will usually be")
print("        a string containing whitespace. Going back to the “three sisters” document:")
print("""            <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>
            <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>
            <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>
        """)
print("        You might think that the .next_sibling of the first <a> tag would be the second")
print("        <a> tag. But actually, it’s a string: the comma and newline that separate the")
print("        first <a> tag from the second:")
print("            link = soup.a")
link = soup.a
print(f"            link: {link}")    # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
print(f"            link.next_sibling: {link.next_sibling}\n")  # u',\n'

print("        The second <a> tag is actually the .next_sibling of the comma:")
print(f"            link.next_sibling.next_sibling: {link.next_sibling.next_sibling}\n")
                # <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>

print("    b.  .next_siblings and .previous_siblings\n")
print("    You can iterate over a tag’s siblings with .next_siblings or .previous_siblings:\n")

print("        for sibling in soup.a.next_siblings:")
print("            print(repr(sibling))")
for sibling in soup.a.next_siblings:
    print(f"                {repr(sibling)}")
        # u',\n'
        # <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
        # u' and\n'
        # <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
        # u'; and they lived at the bottom of a well.'
        # None

print('\n        for sibling in soup.find(id="link3").previous_siblings:')
print("            print(repr(sibling))")
for sibling in soup.find(id="link3").previous_siblings:
    print(f"                {repr(sibling)}")
        # ' and\n'
        # <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>
        # u',\n'
        # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
        # u'Once upon a time there were three little sisters; and their names were\n'
        # None

print("4.  Going back and forth\n")
print('    Take a look at the beginning of the “three sisters” document:')
print("""        <html><head><title>The Dormouse's story</title></head>
        <p class="title"><b>The Dormouse's story</b></p>
    """)
print("    An HTML parser takes this string of characters and turns it into a series of")
print('    events: “open an <html> tag”, “open a <head> tag”, “open a <title> tag”, “add')
print('    a string”, “close the <title> tag”, “open a <p> tag”, and so on. Beautiful Soup')
print('    offers tools for reconstructing the initial parse of the document.\n')

print("    a.  .next_element and .previous_element\n")
print("        The .next_element attribute of a string or tag points to whatever was parsed")
print("        immediately afterwards. It might be the same as .next_sibling, but it’s usually")
print("        drastically different.\n")

print("        Here’s the final <a> tag in the “three sisters” document. Its .next_sibling is a")
print("        string: the conclusion of the sentence that was interrupted by the start of the")
print("        <a> tag.:\n")

print('            last_a_tag = soup.find("a", id="link3")')
last_a_tag = soup.find("a", id="link3")

print(f"            last_a_tag: {last_a_tag}")
                # <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
print(f"            last_a_tag.next_sibling: {last_a_tag.next_sibling}\n")
                # '; and they lived at the bottom of a well.'

print("        But the .next_element of that <a> tag, the thing that was parsed immediately")
print("""        after the <a> tag, is not the rest of that sentence: it’s the word “Tillie”:""")
print(f"            last_a_tag.next_element: {last_a_tag.next_element}\n")    # u'Tillie'

print("""        That’s because in the original markup, the word “Tillie” appeared before that
        semicolon. The parser encountered an <a> tag, then the word “Tillie”, then
        the closing </a> tag, then the semicolon and rest of the sentence. The semicolon
        is on the same level as the <a> tag, but the word “Tillie” was encountered first.
    """)

print("        The .previous_element attribute is the exact opposite of .next_element.")
print("        It points to whatever element was parsed immediately before this one:")
print(f"            last_a_tag.previous_element: {last_a_tag.previous_element}")    # u' and\n'
print(f"            last_a_tag.previous_element.next_element: {last_a_tag.previous_element.next_element}")
                # <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>

print("    b.  .next_elements and .previous_elements\n")
print("        You should get the idea by now. You can use these iterators to move forward")
print("        or backward in the document as it was parsed:")
print("            for element in last_a_tag.next_elements:")
print("                print(repr(element))")
for element in last_a_tag.next_elements:
    print(f"                    {repr(element)}")
            # u'Tillie'
            # u';\nand they lived at the bottom of a well.'
            # u'\n\n'
            # <p class="story">...</p>
            # u'...'
            # u'\n'
            # None

print("Searching the tree\n")
print("    Beautiful Soup defines a lot of methods for searching the parse tree, but they’re all very")
print("    similar. I’m going to spend a lot of time explaining the two most popular methods: find()")
print("    and find_all(). The other methods take almost exactly the same arguments, so I’ll just")
print("    cover them briefly.\n")
print("    Once again, I’ll be using the “three sisters” document as an example:\n")
print("""
        <html><head><title>The Dormouse's story</title></head>
        <body>
        <p class="title"><b>The Dormouse's story</b></p>

        <p class="story">Once upon a time there were three little sisters; and their names were
        <a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
        <a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
        <a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
        and they lived at the bottom of a well.</p>

        <p class="story">...</p>
""")

html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>

<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>

<p class="story">...</p>
"""
print("        soup = BeautifulSoup(html_doc, 'html.parser')\n")
soup = BeautifulSoup(html_doc, 'html.parser')

print("    By passing in a filter to an argument like find_all(), you can zoom in on the parts")
print("    of the document you’re interested in.\n")

print("1.  Kinds of filters\n")
print("""    Before talking in detail about find_all() and similar methods, I want to show
    examples of different filters you can pass into these methods. These filters show
    up again and again, throughout the search API. You can use them to filter based on
    a tag’s name, on its attributes, on the text of a string, or on some combination
    of these.
    """)
print("    a.  A string\n")
print("        The simplest filter is a string. Pass a string to a search method and Beautiful")
print("        Soup will perform a match against that exact string. This code finds all the")
print("        <b> tags in the document:")
print(f"            soup.find_all('b'): {soup.find_all('b')}\n")  # [<b>The Dormouse's story</b>]

print("        If you pass in a byte string, Beautiful Soup will assume the string is encoded as")
print("        UTF-8. You can avoid this by passing in a Unicode string instead.\n")

print("    b.  A regular expression\n")
print("""        If you pass in a regular expression object, Beautiful Soup will filter against
        that regular expression using its search() method. This code finds all the tags whose names
        start with the letter “b”; in this case, the <body> tag and the <b> tag:
    """)
print("""            import re
            for tag in soup.find_all(re.compile("^b")):
                print(tag.name)
    """)
import re
for tag in soup.find_all(re.compile("^b")):
    print(f"                    {tag.name}")
                # body
                # b
print("\n            This code finds all the tags whose names contain the letter ‘t’:")

for tag in soup.find_all(re.compile("t")):
    print(f"                    {tag.name}")
                # html
                # title

print("\n    c.  A list\n")
print("        If you pass in a list, Beautiful Soup will allow a string match against any item")
print("        in that list. This code finds all the <a> tags and all the <b> tags:\n")

print(f'            soup.find_all(["a","b"]): {soup.find_all(["a","b"])}\n')
            # [<b>The Dormouse's story</b>,
            #  <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("        True\n")
print("        The value True matches everything it can. This code finds all the tags in the")
print("        document, but none of the text strings:")
print("            for tag in soup.find_all(True):")
print("                print(tag.name)")
for tag in soup.find_all(True):
    print(f"                {tag.name}")
                # html
                # head
                # title
                # body
                # p
                # b
                # p
                # a
                # a
                # a
                # p

print("\n    d.  A function")
print("        If none of the other matches work for you, define a function that takes an")
print("        element as its only argument. The function should return True if the")
print("        argument matches, and False otherwise.\n")

print("        Here’s a function that returns True if a tag defines the “class” attribute")
print("        but doesn’t define the “id” attribute:\n")

print("""            def has_class_but_no_id(tag):
                return tag.has_attr('class') and not tag.has_attr('id')
    """)
def has_class_but_no_id(tag):
    return tag.has_attr('class') and not tag.has_attr('id')

print("        Pass this function into find_all() and you’ll pick up all the <p> tags:")
print("            soup.find_all(has_class_but_no_id)")
print(soup.find_all(has_class_but_no_id))
            # [<p class="title"><b>The Dormouse's story</b></p>,
            #  <p class="story">Once upon a time there were...</p>,
            #  <p class="story">...</p>]
print("\n        This function only picks up the <p> tags. It doesn’t pick up the <a> tags,")
print("        because those tags define both “class” and “id”. It doesn’t pick up tags like")
print('        <html> and <title>, because those tags don’t define “class”.\n')

print("""        If you pass in a function to filter on a specific attribute like href, the
        argument passed into the function will be the attribute value, not the whole tag. Here’s
        a function that finds all a tags whose href attribute does not match a regular expression:
            def not_lacie(href):
                return href and not re.compile("lacie").search(href)
            soup.find_all(href=not_lacie)
    """)

def not_lacie(href):
    return href and not re.compile("lacie").search(href)
print(f"                {soup.find_all(href=not_lacie)}")
                # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
                #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("""
        The function can be as complicated as you need it to be. Here’s a function")
        that returns True if a tag is surrounded by string objects:

            from bs4 import NavigableString
            def surrounded_by_strings(tag):
                return (isinstance(tag.next_element, NavigableString)
                        and isinstance(tag.previous_element, NavigableString))

            for tag in soup.find_all(surrounded_by_strings):
            print tag.name
        """)
from bs4 import NavigableString
def surrounded_by_strings(tag):
    return (isinstance(tag.next_element, NavigableString)
            and isinstance(tag.previous_element, NavigableString))

for tag in soup.find_all(surrounded_by_strings):
    print(f"                {tag.name}")
            # p
            # a
            # a
            # a
            # p

print("\n        Now we’re ready to look at the search methods in detail.\n")

print("2.  find_all()\n")
print("    Signature: find_all(name, attrs, recursive, string, limit, **kwargs)\n")

print("    The find_all() method looks through a tag’s descendants and retrieves all descendants")
print("    that match your filters. I gave several examples in Kinds of filters, but here are a")
print("    few more:")
print(f'        soup.find_all("title"): {soup.find_all("title")}')
            # [<title>The Dormouse's story</title>]
print(f'        soup.find_all("p", "title"): {soup.find_all("p", "title")}')
            # [<p class="title"><b>The Dormouse's story</b></p>]
print(f'        soup.find_all("a"): {soup.find_all("a")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
print(f'        soup.find_all(id="link2"): {soup.find_all(id="link2")}')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("        import re")
print(f'            soup.find(string=re.compile("sisters")): {soup.find(string=re.compile("sisters"))}')
            # u'Once upon a time there were three little sisters; and their names were\n'

print("""    Some of these should look familiar, but others are new. What does it mean to pass
    in a value for string, or id? Why does find_all("p", "title") find a <p> tag with the CSS
    class “title”? Let’s look at the arguments to find_all().
    """)

print("    a.  The name argument\n")
print("        Pass in a value for name and you’ll tell Beautiful Soup to only consider tags")
print("        with certain names. Text strings will be ignored, as will tags whose names")
print("        that don’t match.\n")

print("        This is the simplest usage:")
print(f'            soup.find_all("title"): {soup.find_all("title")}\n')
            # [<title>The Dormouse's story</title>]

print("        Recall from Kinds of filters that the value to name can be a string, a regular")
print("        expression, a list, a function, or the value True.\n")

print("    b.  The keyword arguments\n")
print("""        Any argument that’s not recognized will be turned into a filter on one of a tag’s
        attributes. If you pass in a value for an argument called id, Beautiful Soup will filter
        against each tag’s ‘id’ attribute:""")
print(f"            soup.find_all(id='link2'): {soup.find_all(id='link2')}")
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("\n        If you pass in a value for href, Beautiful Soup will filter against each")
print("        tag’s ‘href’ attribute:")
print(f'            soup.find_all(href=re.compile("elsie")): {soup.find_all(href=re.compile("elsie"))}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

print("\n        You can filter an attribute based on a string, a regular expression, a list,")
print("        a function, or the value True.\n")
print("        This code finds all tags whose id attribute has a value, regardless of what the")
print("        value is:")
print(f"            soup.find_all(id=True): {soup.find_all(id=True)}")
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("\n        You can filter multiple attributes at once by passing in more than one")
print("        keyword argument:")
print(f"            soup.find_all(href=re.compile('elsie'), id='link1'): \
                    {soup.find_all(href=re.compile('elsie'), id='link1')}")
                    # [<a class="sister" href="http://example.com/elsie" id="link1">three</a>]

print("\n        Some attributes, like the data-* attributes in HTML 5, have names that can’t")
print("        be used as the names of keyword arguments:\n")
print("""            data_soup = BeautifulSoup('<div data-foo="value">foo!</div>')""")
data_soup = BeautifulSoup('<div data-foo="value">foo!</div>',"html.parser")
print(f'                data_soup.find_all(data-foo="value")')
print("                      # SyntaxError: keyword can't be an expression")
print('\n\n')
print("    c.  Searching by CSS class\n")
print("""        It’s very useful to search for a tag that has a certain CSS class, but the name of the CSS attribute,
        “class”, is a reserved word in Python. Using class as a keyword argument will give you a syntax error. As of
        Beautiful Soup 4.1.2, you can search by CSS class using the keyword argument class_:""")
print(f'            soup.find_all("a", class_="sister"): {soup.find_all("a", class_="sister")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("\n        As with any keyword argument, you can pass class_ a string, a regular expression,")
print("        a function, or True:")
print(f'            soup.find_all(class_=re.compile("itl")): {soup.find_all(class_=re.compile("itl"))}')
            # [<p class="title"><b>The Dormouse's story</b></p>]

print("            def has_six_characters(css_class):")
print("                return css_class is not None and len(css_class) == 6")
def has_six_characters(css_class):
    return css_class is not None and len(css_class) == 6

print(f"                    {soup.find_all(class_=has_six_characters)}")
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("\n        Remember that a single tag can have multiple values for its “class” attribute. When you search for a")
print("        tag that matches a certain CSS class, you’re matching against any of its CSS classes:")
print("""            css_soup = BeautifulSoup('<p class="body strikeout"></p>')""")
css_soup = BeautifulSoup('<p class="body strikeout"></p>',"html.parser")
print(f'                css_soup.find_all("p", class_="strikeout"): {css_soup.find_all("p", class_="strikeout")}')
            # [<p class="body strikeout"></p>]
print(f'                css_soup.find_all("p", class_="body"): {css_soup.find_all("p", class_="body")}')
            # [<p class="body strikeout"></p>]

print("\n        You can also search for the exact string value of the class attribute:")
print(f'            css_soup.find_all("p", class_="body strikeout"): {css_soup.find_all("p", class_="body strikeout")}')
            # [<p class="body strikeout"></p>]

print("\n        But searching for variants of the string value won’t work:")
print(f'            css_soup.find_all("p", class_="strikeout body"): {css_soup.find_all("p", class_="strikeout body")}')
            # []

print("        If you want to search for tags that match two or more CSS classes, you should use a CSS selector:")
print(f'            css_soup.select("p.strikeout.body"): {css_soup.select("p.strikeout.body")}')
            # [<p class="body strikeout"></p>]

print("""\n        In older versions of Beautiful Soup, which don’t have the class_ shortcut, you can use the
        attrs trick mentioned above. Create a dictionary whose value for “class” is the string (or regular expression,
        or whatever) you want to search for:""")

print('            soup.find_all("a", attrs={"class": "sister"}', end=': ')
answer = soup.find_all("a", attrs={"class": "sister"})
print(answer)
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("""\n\n    d.  The string argument\n
        With string you can search for strings instead of tags. As with name and the keyword arguments, you")
        can pass in a string, a regular expression, a list, a function, or the value True.")
        Here are some examples:
        """)

print(f'            soup.find_all(string="Elsie"): {soup.find_all(string="Elsie")}')    # [u'Elsie']
print(f'            soup.find_all(string=["Tillie", "Elsie", "Lacie"]): \
                    {soup.find_all(string=["Tillie", "Elsie", "Lacie"])}')
            # [u'Elsie', u'Lacie', u'Tillie']
print(f'            soup.find_all(string=re.compile("Dormouse")): {soup.find_all(string=re.compile("Dormouse"))}')
            # [u"The Dormouse's story", u"The Dormouse's story"]

def is_the_only_string_within_a_tag(s):
    """Return True if this string is the only child of its parent tag."""
    return (s == s.parent.string)

print(f'            soup.find_all(string=is_the_only_string_within_a_tag): \
                    {soup.find_all(string=is_the_only_string_within_a_tag)}')
            # [u"The Dormouse's story", u"The Dormouse's story", u'Elsie', u'Lacie', u'Tillie', u'...']

print("""\n        Although string is for finding strings, you can combine it with arguments that find tags: Beautiful
        Soup will print("find all tags whose .string matches your value for string. This code finds the <a> tags whose
        .string is “Elsie”:""")

print(f'            soup.find_all("a", string="Elsie"): {soup.find_all("a", string="Elsie")}')
            # [<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>]

print("\n        The string argument is new in Beautiful Soup 4.4.0. In earlier versions it was called text:")
print(f'            soup.find_all("a", text="Elsie"): {soup.find_all("a", text="Elsie")}')
            # [<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>]

print("\n\n    e.  The limit argument\n")
print("""        find_all() returns all the tags and strings that match your filters. This can take a while if the
        document is large. If you don’t need all the results, you can pass in a number for limit. This works just like
        the LIMIT keyword in SQL. It tells Beautiful Soup to stop gathering results after it’s found a certain number.
    """)
print('        There are three links in the “three sisters” document, but this code only finds the first two:')
print(f'            soup.find_all("a", limit=2): {soup.find_all("a", limit=2)}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("\n\n    f.  The recursive argument\n")
print("""        If you call mytag.find_all(), Beautiful Soup will examine all the descendants of mytag: its children,
        its children’s children, and so on. If you only want Beautiful Soup to consider direct children, you can pass
        in recursive=False. See the difference here:""")
print(f'            soup.html.find_all("title"): {soup.html.find_all("title")}')
            # [<title>The Dormouse's story</title>]
print(f'            soup.html.find_all("title", recursive=False): {soup.html.find_all("title", recursive=False)}')
            # []
print("""        Here’s that part of the document:

        <html>
         <head>
          <title>
           The Dormouse's story
          </title>
         </head>
        ...

        The <title> tag is beneath the <html> tag, but it’s not directly beneath the <html> tag: the <head> tag is
        in the way. Beautiful Soup finds the <title> tag when it’s allowed to look at all descendants of the <html>
        tag, but when recursive=False restricts it to the <html> tag’s immediate children, it finds nothing.

        Beautiful Soup offers a lot of tree-searching methods (covered below), and they mostly take the same
        arguments as find_all(): name, attrs, string, limit, and the keyword arguments. But the recursive argument
        is different: find_all() and find() are the only methods that support it. Passing recursive=False into a
        method like find_parents() wouldn’t be very useful.
    """)

print("3.  Calling a tag is like calling find_all()\n")
print("""    Because find_all() is the most popular method in the Beautiful Soup search API, you
    can use a shortcut for it. If you treat the BeautifulSoup object or a Tag object as though it
    were a function, then it’s the same as calling find_all() on that object. These two lines of
    code are equivalent:""")
print(f'        soup.find_all("a"): {soup.find_all("a")}')
print(f'        soup("a"): {soup("a")}\n')

print("    These two lines are also equivalent:")
print(f"        soup.title.find_all(string=True): {soup.title.find_all(string=True)}")
print(f"        soup.title(string=True): {soup.title(string=True)}\n\n")

print("4.  find()\n")
print("    Signature: find(name, attrs, recursive, string, **kwargs)\n")

print("""    The find_all() method scans the entire document looking for results, but sometimes
    you only want to find one result. If you know a document only has one <body> tag, it’s a
    waste of time to scan the entire document looking for more. Rather than passing in limit=1
    every time you call find_all, you can use the find() method. These two lines of code are
    nearly equivalent:""")
print(f"        soup.find_all('title', limit=1 : {soup.find_all('title', limit=1)}")
            # [<title>The Dormouse's story</title>]
print(f"        soup.find('title') : {soup.find('title')}\n")
            # <title>The Dormouse's story</title>

print("    The only difference is that find_all() returns a list containing the single result,")
print("    and find() just returns the result.\n")

print("    If find_all() can’t find anything, it returns an empty list. If find() can’t")
print("    find anything, it returns None:")
print(f'        soup.find("nosuchtag") : {soup.find("nosuchtag")}\n')
            # None

print("    Remember the soup.head.title trick from Navigating using tag names? That trick")
print("    works by repeatedly calling find():")
print(f"        soup.head.title : {soup.head.title}")
            # <title>The Dormouse's story</title>
print(f'        soup.find("head").find("title") : {soup.find("head").find("title")}\n\n')
            # <title>The Dormouse's story</title>

print("5.  find_parents() and find_parent()\n")
print("    Signature: find_parents(name, attrs, string, limit, **kwargs)")
print("    Signature: find_parent(name, attrs, string, **kwargs)")

print("""    I spent a lot of time above covering find_all() and find(). The Beautiful Soup API
    defines ten other methods for searching the tree, but don’t be afraid. Five of these methods
    are basically the same as find_all(), and the other five are basically the same as find().
    The only differences are in what parts of the tree they search.

    First let’s consider find_parents() and find_parent(). Remember that find_all() and find()
    work their way down the tree, looking at tag’s descendants. These methods do the opposite:
    they work their way up the tree, looking at a tag’s (or a string’s) parents. Let’s try them
    out, starting from a string buried deep in the “three daughters” document:""")
print('        a_string = soup.find(string="Lacie")')
a_string = soup.find(string="Lacie")
print(f'            a_string : {a_string}')
            # u'Lacie'
print(f'        a_string.find_parents("a") : {a_string.find_parents("a")}')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]
print(f'        a_string.find_parent("p") : {a_string.find_parent("p")}')
            # <p class="story">Once upon a time there were three little sisters; and their names were
            #  <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a> and
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>;
            #  and they lived at the bottom of a well.</p>

print(f'            a_string.find_parents("p", class_="title") : {a_string.find_parents("p", class_="title")}')
            # []

print("""    One of the three <a> tags is the direct parent of the string in question, so our
    search finds it. One of the three <p> tags is an indirect parent of the string, and our
    search finds that as well. There’s a <p> tag with the CSS class “title” somewhere in the
    document, but it’s not one of this string’s parents, so we can’t find it with find_parents().

    You may have made the connection between find_parent() and find_parents(), and the
    .parent and .parents attributes mentioned earlier. The connection is very strong. These
    search methods actually use .parents to iterate over all the parents, and check each one
    against the provided filter to see if it matches.\n\n""")

print("""6.  find_next_siblings() and find_next_sibling()
    Signature: find_next_siblings(name, attrs, string, limit, **kwargs)
    Signature: find_next_sibling(name, attrs, string, **kwargs)

    These methods use .next_siblings to iterate over the rest of an element’s siblings in the tree. The
    find_next_siblings() method returns all the siblings that match, and find_next_sibling() only returns
    the first one:

        first_link = soup.a""")
first_link = soup.a
print(f"        first_link : {first_link}")
            # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>

print(f'        first_link.find_next_siblings("a") : {first_link.find_next_siblings("a")}\n')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
print('        first_story_paragraph = soup.find("p", "story")')

first_story_paragraph = soup.find("p", "story")
print(f'        first_story_paragraph.find_next_sibling("p") : {first_story_paragraph.find_next_sibling("p")}')
            # <p class="story">...</p>


print("7.  find_previous_siblings() and find_previous_sibling()\n")
print("    Signature: find_previous_siblings(name, attrs, string, limit, **kwargs)")
print("    Signature: find_previous_sibling(name, attrs, string, **kwargs)\n")

print("""    These methods use .previous_siblings to iterate over an element’s siblings that precede it in the tree.")
    The find_previous_siblings() method returns all the siblings that match, and find_previous_sibling() only returns
    the first one:
    """)

print('        last_link = soup.find("a", id="link3")')
last_link = soup.find("a", id="link3")
print(f"        last_link : {last_link}")
            # <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
print(f'        last_link.find_previous_siblings("a") : \
                {last_link.find_previous_siblings("a")}\n')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

print('        first_story_paragraph = soup.find("p", "story")')
first_story_paragraph = soup.find("p", "story")
print(f'        first_story_paragraph.find_previous_sibling("p") : {first_story_paragraph.find_previous_sibling("p")}\n\n')
            # <p class="title"><b>The Dormouse's story</b></p>

print("""8.  find_all_next() and find_next()
    Signature: find_all_next(name, attrs, string, limit, **kwargs)
    Signature: find_next(name, attrs, string, **kwargs)

    These methods use .next_elements to iterate over whatever tags and strings that come after it in the document.
    The find_all_next() method returns all matches, and find_next() only returns the first match:

        first_link = soup.a""")
first_link = soup.a
print(f"        first_link : {first_link}")
            # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
print("        first_link.find_all_next(string=True) : ", end='')
print(first_link.find_all_next(string=True))
            # [u'Elsie', u',\n', u'Lacie', u' and\n', u'Tillie',
            #  u';\nand they lived at the bottom of a well.', u'\n\n', u'...', u'\n']
print(f'        first_link.find_next("p") : {first_link.find_next("p")}\n')
            # <p class="story">...</p>

print("""    In the first example, the string “Elsie” showed up, even though it was contained within the <a> tag we
    started from. In the second example, the last <p> tag in the document showed up, even though it’s not in the same
    part of the tree as the <a> tag we started from. For these methods, all that matters is that an element match the
    filter, and show up later in the document than the starting element.


    """)

print("""9.  find_all_previous() and find_previous()

    Signature: find_all_previous(name, attrs, string, limit, **kwargs)
    Signature: find_previous(name, attrs, string, **kwargs)

    These methods use .previous_elements to iterate over the tags and strings that came before it in the document.
    The find_all_previous() method returns all matches, and find_previous() only returns the first match:

        first_link = soup.a""")
first_link = soup.a
print(f"        first_link : {first_link}")
            # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>
print('        first_link.find_all_previous("p")')
print(first_link.find_all_previous("p"))
            # [<p class="story">Once upon a time there were three little sisters; ...</p>,
            #  <p class="title"><b>The Dormouse's story</b></p>]
print(f'\n        first_link.find_previous("title") : {first_link.find_previous("title")}\n')
            # <title>The Dormouse's story</title>

print("""    The call to find_all_previous("p") found the first paragraph in the document (the one with class=”title”),
    but it also finds the second paragraph, the <p> tag that contains the <a> tag we started with. This shouldn’t be
    too surprising: we’re looking at all the tags that show up earlier in the document than the one we started with.
    A <p> tag that contains an <a> tag must have shown up before the <a> tag it contains.

    """)

print("""10. CSS selectors

    As of version 4.7.0, Beautiful Soup supports most CSS4 selectors via the SoupSieve project. If you installed
    Beautiful Soup through pip, SoupSieve was installed at the same time, so you don’t have to do anything extra.

    BeautifulSoup has a .select() method which uses SoupSieve to run a CSS selector against a parsed document and
    return all the matching elements. Tag has a similar method which runs a CSS selector against the contents of
    a single tag.

    (Earlier versions of Beautiful Soup also have the .select() method, but only the most commonly-used CSS selectors
    are supported.)

    The SoupSieve documentation lists all the currently supported CSS selectors, but here are some of the basics:

    You can find tags:
    """)
print(f'        soup.select("title") : {soup.select("title")}\n')
            # [<title>The Dormouse's story</title>]

print('        soup.select("p:nth-of-type(3)") : {soup.select("p:nth-of-type(3)")}\n')
            # [<p class="story">...</p>]

print("    Find tags beneath other tags:\n")
print(f'        soup.select("body a") : {soup.select("body a")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie"  id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print(f'        soup.select("html head title") : {soup.select("html head title")}\n')
            # [<title>The Dormouse's story</title>]

print("    Find tags directly beneath other tags:\n")
print(f'        soup.select("head > title") : {soup.select("head > title")}')
            # [<title>The Dormouse's story</title>]
print('        soup.select("p > a") : {soup.select("p > a")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie"  id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
print(f'        soup.select("p > a:nth-of-type(2)") : {soup.select("p > a:nth-of-type(2)")}')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]
print(f'        soup.select("p > #link1") : {soup.select("p > #link1")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]
print('        soup.select("body > a") : {soup.select("body > a")}\n\n')
            # []

print("    Find the siblings of tags:")
print(f'        soup.select("#link1 ~ .sister") : {soup.select("#link1 ~ .sister")}')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie"  id="link3">Tillie</a>]
print(f'        soup.select("#link1 + .sister") : {soup.select("#link1 + .sister")}\n')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("    Find tags by CSS class:")
print(f'        soup.select(".sister") : {soup.select(".sister")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
print(f'        soup.select("[class~=sister]") : {soup.select("[class~=sister]")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("\n    Find tags by ID:\n")
print(f'        soup.select("#link1") : {soup.select("#link1")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]
print(f'        soup.select("a#link2") : {soup.select("a#link2")}')
            # [<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("\n    Find tags that match any selector from a list of selectors:")
print(f'        soup.select("#link1,#link2") : {soup.select("#link1,#link2")}')
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]

print("\n    Test for the existence of an attribute:")
print(f"        soup.select('a[href]') : {soup.select('a[href]')}")
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

print("\n    Find tags by attribute value:")
print(f"""        soup.select('a[href="http://example.com/elsie"]')""", end=' : ')
print(soup.select('a[href="http://example.com/elsie"]'))
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]
print("""        soup.select('a[href^="http://example.com/"]')""")

example = soup.select('a[href^="http://example.com/"]')
print(f"""        soup.select('a[href^="http://example.com/"]')) : {example}""")
            # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
            #  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
            #  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]

example = soup.select('a[href$="tillie"]')
print(f"""        soup.select('a[href$="tillie"]') : {example}""")
            # [<a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]
example = soup.select('a[href*=".com/el"]')
print(f"""        soup.select('a[href*=".com/el"]') : {example}""")
            # # [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>]

print("\n    There’s also a method called select_one(), which finds only the first tag that matches a selector:")
print(f'        soup.select_one(".sister") : {soup.select_one(".sister")}')
            # <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>

print("""\n\n    When handling a CSS selector that uses namespaces, Beautiful Soup uses the namespace abbreviations it found
    when parsing the document. You can override this by passing in your own dictionary of abbreviations:

        namespaces = dict(first="http://namespace1/", second="http://namespace2/")
    """)
namespaces = dict(first="http://namespace1/", second="http://namespace2/")
example = soup.select("second|child", namespaces=namespaces)
print(f'        soup.select("second|child", namespaces=namespaces) : {example}')
            # # [<ns1:child>I'm in namespace 2</ns1:child>]

print("""
    All this CSS selector stuff is a convenience for people who already know the CSS selector syntax. You can do all of
    this with the Beautiful Soup API. And if CSS selectors are all you need, you should parse the document with lxml:
    it’s a lot faster. But this lets you combine CSS selectors with the Beautiful Soup API.


    """)

print("""Modifying the tree

    Beautiful Soup’s main strength is in searching the parse tree, but you can also modify the tree and write your
    changes as a new HTML or XML document.

1.  Changing tag names and attributes

    I covered this earlier, in Attributes, but it bears repeating. You can rename a tag, change the values of its
    attributes, add new attributes, and delete attributes:

        soup = BeautifulSoup('<b class="boldest">Extremely bold</b>')
        tag = soup.b
        tag.name = "blockquote"
        tag['class'] = 'verybold'
        tag['id'] = 1
    """)
soup = BeautifulSoup('<b class="boldest">Extremely bold</b>', 'html.parser')
tag = soup.b
tag.name = "blockquote"
tag['class'] = 'verybold'
tag['id'] = 1
print(f"        tag : {tag}")
            # <blockquote class="verybold" id="1">Extremely bold</blockquote>
print("""
        del tag['class']
        del tag['id']
    """)
del tag['class']
del tag['id']
print(f"        tag : {tag}")
            # <blockquote>Extremely bold</blockquote>

print("""
2.  Modifying .string

    If you set a tag’s .string attribute, the tag’s contents are replaced with the string you give:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        tag = soup.a
        tag.string = "New link text."
""")
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
tag = soup.a
tag.string = "New link text."

print(f"        tag : {tag}")
            # <a href="http://example.com/">New link text.</a>

print("\n    Be careful: if the tag contained other tags, they and all their contents will be destroyed.\n")
print("""    a.  append()
        You can add to a tag’s contents with Tag.append(). It works just like calling .append() on a Python list:

            soup = BeautifulSoup("<a>Foo</a>")""")
soup = BeautifulSoup("<a>Foo</a>", 'html.parser')
print(f'            soup.a.append("Bar") : {soup.a.append("Bar")}')
print(f'            soup : {soup}')
            # <html><head></head><body><a>FooBar</a></body></html>
print(f'            soup.a.contents : {soup.a.contents}')
            # [u'Foo', u'Bar']

print("""    b.  extend()

        Starting in Beautiful Soup 4.7.0, Tag also supports a method called .extend(), which works just like calling
        .extend() on a Python list:

            soup = BeautifulSoup("<a>Soup</a>")
    """)
soup = BeautifulSoup("<a>Soup</a>", 'html.parser')
print(f"""            soup.a.extend(["'s", " ", "on"]) : {soup.a.extend(["'s", " ", "on"])}""")
print(f'            soup : {soup}')
            # <html><head></head><body><a>Soup's on</a></body></html>
print(f'            soup.a.contents : {soup.a.contents}')
            # [u'Soup', u''s', u' ', u'on']

print("""    c.  NavigableString() and .new_tag()
        If you need to add a string to a document, no problem–you can pass a Python string in to append(), or you can
        call the NavigableString constructor:

            soup = BeautifulSoup("<b></b>")
            tag = soup.b""")
soup = BeautifulSoup("<b></b>", 'html.parser')
tag = soup.b
print(f'            tag.append("Hello") : {tag.append("Hello")}\n')
print('            new_string = NavigableString(" there")')
new_string = NavigableString(" there")
print(f'            tag.append(new_string) : {tag.append(new_string)}')
print(f'            tag : {tag}')
            # <b>Hello there.</b>
print(f'            tag.contents : {tag.contents}\n')
            # [u'Hello', u' there']

print("""    If you want to create a comment or some other subclass of NavigableString, just call the constructor:

            from bs4 import Comment
            new_comment = Comment("Nice to see you.")
            tag.append(new_comment)
    """)

from bs4 import Comment
new_comment = Comment("Nice to see you.")
tag.append(new_comment)
print(f'            tag : {tag}')
            # <b>Hello there<!--Nice to see you.--></b>
print(f'            tag.contents : {tag.contents}')
            # [u'Hello', u' there', u'Nice to see you.']

print("\n        Only the first argument, the tag name, is required.\n")

print("""    d.  insert()

        Tag.insert() is just like Tag.append(), except the new element doesn’t necessarily go at the end of its
        parent’s .contents. It’ll be inserted at whatever numeric position you say. It works just like .insert()
        on a Python list:

            markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
            soup = BeautifulSoup(markup)
            tag = soup.a
    """)
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
tag = soup.a
print(f'            tag.insert(1, "but did not endorse ") : {tag.insert(1, "but did not endorse ")}')
print(f'            tag : {tag}')
            # <a href="http://example.com/">I linked to but did not endorse <i>example.com</i></a>
print(f'            tag.contents : {tag.contents}')
            # [u'I linked to ', u'but did not endorse', <i>example.com</i>]

print("""
    e.  insert_before() and insert_after()
        The insert_before() method inserts tags or strings immediately before something else in the parse tree:

            soup = BeautifulSoup("<b>stop</b>")
            tag = soup.new_tag("i")
            tag.string = "Don't"
    """)
soup = BeautifulSoup("<b>stop</b>", 'html.parser')
tag = soup.new_tag("i")
tag.string = "Don't"
print(f'            soup.b.string.insert_before(tag) : {soup.b.string.insert_before(tag)}')
print(f'            soup.b : {soup.b}')
            # <b><i>Don't</i>stop</b>

print("""
        The insert_after() method inserts tags or strings immediately following something else in the parse tree:

            div = soup.new_tag('div')
            div.string = 'ever'
    """)
div = soup.new_tag('div')
div.string = 'ever'

print(f'            soup.b.i.insert_after(" you ", div) : {soup.b.i.insert_after(" you ", div)}')
print(f'            soup.b : {soup.b}')
            # <b><i>Don't</i> you <div>ever</div> stop</b>
print(f'            soup.b.contents : {soup.b.contents}')
            # [<i>Don't</i>, u' you', <div>ever</div>, u'stop']

print("""
    f.  clear()

    Tag.clear() removes the contents of a tag:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        tag = soup.a

        tag.clear()
    """)
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
tag = soup.a
tag.clear()
print(f'            tag : {tag}')
            # <a href="http://example.com/"></a>

print("""
    g. extract()

    PageElement.extract() removes a tag or string from the tree. It returns the tag or string that was extracted:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        a_tag = soup.a
        i_tag = soup.i.extract()
    """)
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
a_tag = soup.a
i_tag = soup.i.extract()

print(f'            a_tag : {a_tag}')
            # <a href="http://example.com/">I linked to</a>
print(f'            i_tag : {i_tag}')
            # <i>example.com</i>
print(f'            print(i_tag.parent) : {print(i_tag.parent)}')
            # None

print("""
    h.  decompose()

        Tag.decompose() removes a tag from the tree, then completely destroys it and its contents:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        a_tag = soup.a
    """)
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
a_tag = soup.a

print(f'            soup.i.decompose() : {soup.i.decompose()}')
print(f'            a_tag : {a_tag}')
                # <a href="http://example.com/">I linked to</a>

print("""
    i.  replace_with()

        PageElement.replace_with() removes a tag or string from the tree,
        and replaces it with the tag or string of your choice:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        a_tag = soup.a

        new_tag = soup.new_tag("b")
        new_tag.string = "example.net"
        a_tag.i.replace_with(new_tag)
    """)
markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
a_tag = soup.a
new_tag = soup.new_tag("b")
new_tag.string = "example.net"
a_tag.i.replace_with(new_tag)

print(f'            a_tag : {a_tag}')
            # <a href="http://example.com/">I linked to <b>example.net</b></a>

print("""
        replace_with() returns the tag or string that was replaced, so that you can examine
        it or add it back to another part of the tree.

    """)

print("""
    j.  wrap()

        PageElement.wrap() wraps an element in the tag you specify. It returns the new wrapper:

            soup = BeautifulSoup("<p>I wish I was bold.</p>")""")

soup = BeautifulSoup("<p>I wish I was bold.</p>", 'html.parser')
print(f'            soup.p.string.wrap(soup.new_tag("b")) : {soup.p.string.wrap(soup.new_tag("b"))}')
            # <b>I wish I was bold.</b>
print(f'            soup.p.wrap(soup.new_tag("div")) : {soup.p.wrap(soup.new_tag("div"))}')
            # <div><p><b>I wish I was bold.</b></p></div>

print("\n        This method is new in Beautiful Soup 4.0.5.\n\n")

print("""
    k.  unwrap()

    Tag.unwrap() is the opposite of wrap(). It replaces a tag with whatever’s inside that tag.
    It’s good for stripping out markup:

        markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
        soup = BeautifulSoup(markup)
        a_tag = soup.a
        a_tag.i.unwrap()""")

markup = '<a href="http://example.com/">I linked to <i>example.com</i></a>'
soup = BeautifulSoup(markup, 'html.parser')
a_tag = soup.a
a_tag.i.unwrap()

print(f'            a_tag : {a_tag}')
            # <a href="http://example.com/">I linked to example.com</a>

print("\n        Like replace_with(), unwrap() returns the tag that was replaced.")


print("""
Output

1.  Pretty-printing

    The prettify() method will turn a Beautiful Soup parse tree into a nicely formatted Unicode string, with a
    separate line for each HTML/XML tag and string:

        markup = ‘<a href=”http://example.com/“>I linked to <i>example.com</i></a>’
        soup = BeautifulSoup(markup)
        soup.prettify()
    """)
markup = """<a href="http://example.com/">I linked to <i>example.com</i></a>"""
soup = BeautifulSoup(markup, 'html.parser')
soup.prettify() # ‘<html>n <head>n </head>n <body>n <a href=”http://example.com/“>n...’

print(soup.prettify())
            # <html> # <head> # </head> # <body> # <a href=”http://example.com/“>
            #   I linked to # <i> # example.com # </i> # </a> # </body> # </html>

print("\n        You can call prettify() on the top-level BeautifulSoup object, or on any of its Tag objects:\n")

print(soup.a.prettify())
            # <a href="http://example.com/">
            #  I linked to
            #  <i>
            #   example.com
            #  </i>
            # </a>

print("""
2.  Non-pretty printing

    If you just want a string, with no fancy formatting, you can call unicode() or str()
    on a BeautifulSoup object, or a Tag within it:

        str(soup)""")
print(f'        {str(soup)}')
            # '<html><head></head><body><a href="http://example.com/">I linked to <i>example.com</i></a></body></html>'

print("""
        The str() function returns a string encoded in UTF-8. See Encodings for other options.

        You can also call encode() to get a bytestring, and decode() to get Unicode.
    """)


print("""\n\n3.  Output formatters\n
    If you give Beautiful Soup a document that contains HTML entities like “&lquot;”,
    they’ll be converted to Unicode characters:

        soup = BeautifulSoup("&ldquo;Dammit!&rdquo; he said.")
    """)
soup = BeautifulSoup("&ldquo;Dammit!&rdquo; he said.", "html.parser")
print("""    If you then convert the document to a string, the Unicode characters will be encoded as UTF-8.
    You won’t get the HTML entities back:
    """)

print(f'        str(soup) : {str(soup)}')
            # '<html><head></head><body>\xe2\x80\x9cDammit!\xe2\x80\x9d he said.</body></html>'

print("""
    By default, the only characters that are escaped upon output are bare ampersands and angle brackets.
    These get turned into “&amp;”, “&lt;”, and “&gt;”, so that Beautiful Soup doesn’t inadvertently generate
    invalid HTML or XML:

        soup = BeautifulSoup("<p>The law firm of Dewey, Cheatem, & Howe</p>")""")
soup = BeautifulSoup("<p>The law firm of Dewey, Cheatem, & Howe</p>",'html.parser')
print(f'        soup.p : {soup.p}')
            # <p>The law firm of Dewey, Cheatem, &amp; Howe</p>

print("""
        soup = BeautifulSoup('<a href="http://example.com/?foo=val1&bar=val2">A link</a>')""")
soup = BeautifulSoup('<a href="http://example.com/?foo=val1&bar=val2">A link</a>','html.parser')
print(f'        soup.a : {soup.a}')
            # <a href="http://example.com/?foo=val1&amp;bar=val2">A link</a>

print("""
    You can change this behavior by providing a value for the formatter argument to prettify(), encode(), or decode().
    Beautiful Soup recognizes six possible values for formatter.

    The default is formatter="minimal". Strings will only be processed enough to ensure that Beautiful Soup generates
    valid HTML/XML:

        french = "<p>Il a dit &lt;&lt;Sacr&eacute; bleu!&gt;&gt;</p>"
        soup = BeautifulSoup(french)
    """)
french = "<p>Il a dit &lt;&lt;Sacr&eacute; bleu!&gt;&gt;</p>"
soup = BeautifulSoup(french, 'html.parser')
example = print(soup.prettify(formatter="minimal"))
print(f'            print(soup.prettify(formatter="minimal") : {example}')
            # <html>
            #  <body>
            #   <p>
            #    Il a dit &lt;&lt;Sacré bleu!&gt;&gt;
            #   </p>
            #  </body>
            # </html>

print("""
    If you pass in formatter="html", Beautiful Soup will convert Unicode characters to HTML entities whenever possible:

        print(soup.prettify(formatter="html"))
    """)
print(soup.prettify(formatter="html"))
            # <html>
            #  <body>
            #   <p>
            #    Il a dit &lt;&lt;Sacr&eacute; bleu!&gt;&gt;
            #   </p>
            #  </body>
            # </html>

print("""
    If you pass in ``formatter="html5"``, it's the same as
    formatter="html5", but Beautiful Soup will omit the closing slash in HTML void tags like “br”:

        soup = BeautifulSoup("<br>")
        print(soup.encode(formatter="html"))
    """)
soup = BeautifulSoup("<br>",'html.parser')
print(f'         html:  {soup.encode(formatter="html")}')
            # <html><body><br/></body></html>
print(f'        html5:  {soup.encode(formatter="html5")}')
#             # <html><body><br></body></html>

print("""
    If you pass in formatter=None, Beautiful Soup will not modify strings at all on output. This is the fastest
    option, but it may lead to Beautiful Soup generating invalid HTML/XML, as in these examples:
    """)
print(f'        ',end='')
print(soup.prettify(formatter=None))
            # <html>
            #  <body>
            #   <p>
            #    Il a dit <<Sacré bleu!>>
            #   </p>
            #  </body>
            # </html>

link_soup = BeautifulSoup('<a href="http://example.com/?foo=val1&bar=val2">A link</a>','html.parser')
print(f'        ',end='')
print(link_soup.a.encode(formatter=None))
            # <a href="http://example.com/?foo=val1&bar=val2">A link</a>

print("""
    Finally, if you pass in a function for formatter, Beautiful Soup will call that function once for every string
    and attribute value in the document. You can do whatever you want in this function. Here’s a formatter that
    converts strings to uppercase and does absolutely nothing else:

        def uppercase(str):
            return str.upper()

        print(soup.prettify(formatter=uppercase))""")

def uppercase(str):
    return str.upper()
print(f'            {soup.prettify(formatter=uppercase)}')
            # <html>
            #  <body>
            #   <p>
            #    IL A DIT <<SACRÉ BLEU!>>
            #   </p>
            #  </body>
            # </html>
print("\n    link_soup.a.prettify(formatter=uppercase)")
print(link_soup.a.prettify(formatter=uppercase))
    # <a href="HTTP://EXAMPLE.COM/?FOO=VAL1&BAR=VAL2">
    #  A LINK
    # </a>

print("""
    If you’re writing your own function, you should know about the EntitySubstitution class in the bs4.dammit module.
    This class implements Beautiful Soup’s standard formatters as class methods: the “html” formatter is
    EntitySubstitution.substitute_html, and the “minimal” formatter is EntitySubstitution.substitute_xml. You can use
    these functions to simulate formatter=html or formatter==minimal, but then do something extra.

    Here’s an example that replaces Unicode characters with HTML entities whenever possible,
    but also converts all strings to uppercase:

        from bs4.dammit import EntitySubstitution
        def uppercase_and_substitute_html_entities(str):
            return EntitySubstitution.substitute_html(str.upper())
    """)
from bs4.dammit import EntitySubstitution
def uppercase_and_substitute_html_entities(str):
    return EntitySubstitution.substitute_html(str.upper())

print(soup.prettify(formatter=uppercase_and_substitute_html_entities))
            # <html>
            #  <body>
            #   <p>
            #    IL A DIT &lt;&lt;SACR&Eacute; BLEU!&gt;&gt;
            #   </p>
            #  </body>
            # </html>

print("""
    One last caveat: if you create a CData object, the text inside that object is always presented exactly as it
    appears, with no formatting. Beautiful Soup will call the formatter method, just in case you’ve written a custom
    method that counts all the strings in the document or something, but it will ignore the return value:

        from bs4.element import CData
        soup = BeautifulSoup("<a></a>")
        soup.a.string = CData("one < three")
    """)
from bs4.element import CData
soup = BeautifulSoup("<a></a>", 'html.parser')
soup.a.string = CData("one < three")

print(soup.a.prettify(formatter="xml"))
            # <a>
            #  <![CDATA[one < three]]>
            # </a>

print("""4.  get_text()
    If you only want the text part of a document or tag, you can use the get_text() method.
    It returns all the text in a document or beneath a tag, as a single Unicode string:

        markup = '<a href="http://example.com/">\nI linked to <i>example.com</i>\n</a>'
        soup = BeautifulSoup(markup,'html.parser')
    """)
markup = '<a href="http://example.com/">\nI linked to <i>example.com</i>\n</a>'
soup = BeautifulSoup(markup, 'html.parser')

print(f'        soup.get_text() : {soup.get_text()}')
            # u'\nI linked to example.com\n'
print(f'        soup.i.get_text() : {soup.i.get_text()}')
            # u'example.com'

print("\n    You can specify a string to be used to join the bits of text together:")

print(f'        soup.get_text("|") : {soup.get_text("|")}')
            # u'\nI linked to |example.com|\n'
            # You can tell Beautiful Soup to strip whitespace from the beginning and end of each bit of text:

print(f'        soup.get_text("|", strip=True) : {soup.get_text("|", strip=True)}')
            # u'I linked to|example.com'

print("\n    But at that point you might want to use the .stripped_strings generator instead,")
print("    and process the text yourself:")

print([text for text in soup.stripped_strings])
            # [u'I linked to', u'example.com']
