#!/usr/bin/env python3

# This script is used to create the REST API documentation.  It executes
# example requests and then prints a markdown documentation to stdout.
#
# Start `blob-testapp`, configure an API key for the testing user, and run:
#
#   ./tools/bin/gen-apidoc-nog-blob-upload >apidoc-upload.md
#

toc = [
    'startUpload',
    'putS3',
    'getNextParts',
    'postComplete'
]

md = {}


md['startUpload'] = '''

<!--
DO NOT EDIT.
This file has been automatically generated by gen-apidoc-nog-blob-upload.
-->

## Upload Blob Data

Uploading data requires a sequence of coordinated requests.  The upload starts
with a `POST`, which returns an upload id and descriptions how parts should be
upload to S3.  Depending on the total upload size, multiple parts need to be
uploaded to S3.  More upload parts should be requested only when needed and
used immediately, because the S3 URLs expire after a couple of minutes.  After
uploading the binary data to S3, the upload is completed by posting the `ETag`
headers that S3 returned to the completion href that has been returned by the
initial `POST`.

### Start Upload

```
{method} {path}
```

The sha1 of the file is specified in the path.

`:limit` is used to restrict the number of upload parts that are initially
returned.  Use a limit of 1 if your upload code works sequentially.  You may
use a larger number if your upload code handles multiple concurrent uploads to
S3.

**Request body**

 - `size` (`Number`): The file size.
 - `name` (`String`): The local file name.

Example:

```
{method} {url}
```

```json
{reqbody}
```

**Response**

```
Status: {statusCode}
```

```json
{resbody}
```

'''


md['putS3'] = '''

### Upload Individual Parts to S3

```
{method} <data.parts.items[n].href>
```

Binary data for each part must be send to S3 using `PUT`.  The data slice for
each part is specified as `[start; end[` in the part descriptions.  The start
is inclusive; the end is exclusive.

Pay attention to the indexing: S3 part numbers start with index 1.  The
pagination starts with index 0.

Example:

```
{method} {url}

Body with binary data for [start; end[
```

**Response**

```
Status: {statusCode}
ETag: {etag}
```

'''


md['getNextParts'] = '''

### Get Additional Part Descriptions

```
{method} <parts.next>
```

More parts can be retrieved from the `next` href of the previous parts.  The
URL automatically uses the same limit as the initial request.

An alternative is to explicitly construct the URL to get more parts:

```
{method} {path}
```

Example:

```
{method} {url}
```

**Response**

```
Status: {statusCode}
```

```json
{resbody}
```

'''


md['postComplete'] = '''

### Complete the Upload

The upload is completed by a `POST` to the href that has been returned as
`upload.href` when starting the upload.

```
{method} <upload.href>
```

The format is:

```
{method} {path}
```

Example:

```
{method} {url}
```

**Request body**

The body contains a field `s3Parts` with an array with the part numbers and
ETag headers returned by S3.

Example:

```json
{reqbody}
```

**Response**

```
Status: {statusCode}
```

```json
{resbody}
```

'''


from copy import copy
from subprocess import check_output
from textwrap import dedent
import hashlib
import json
import os
import random
import re
import requests
import string
import sys

size = 6000000
filename = 'testdata.dat'

baseUrl = 'http://localhost:3000/api'
apiPath = '/repos'
headers = {'Content-Type': 'application/json'}

userName = 'fred'
repoName = 'hello-world'

# `examples` is used to store the HTTP requests in `request()`.  The recorded
# requests are then used when formatting the documentation.
examples = {}


def main():
    data = randomdata(size)
    sha1 = hashlib.sha1(data).hexdigest()
    res = startUpload(sha1)
    completeUploadUrl = res["upload"]["href"]
    s3Parts = []
    parts = res["parts"]
    while True:
        part = parts["items"][0]
        etag = putS3(part, data)
        s3Parts.append({
                'PartNumber': part["partNumber"],
                'ETag': etag
            })
        nextPartUrl = parts["next"]
        if not nextPartUrl:
            break
        parts = getNextParts(nextPartUrl)
    res = postComplete(completeUploadUrl, s3Parts)
    printFullDoc()


def startUpload(sha1):
    res = request(
        key='startUpload',
        method='POST',
        path=apiPath +
            '/{repoOwner}/{repoName}/db/blobs/{sha1}/uploads?limit={limit}',
        params={
            'repoOwner': userName,
            'repoName': repoName,
            'sha1': sha1,
            'limit': 1
        },
        reqbody={
            'size': size,
            'name': filename
        },
        statusCode=201
    )
    return res.json()["data"]


def getNextParts(url):
    res = request(
        key='getNextParts',
        method='GET',
        path=apiPath +
            '/{repoOwner}/{repoName}/db/blobs/{sha1}/uploads/{uploadId}' +
            '?offset={offset}&limit={limit}',
        url=url,
        statusCode=200
    )
    return res.json()["data"]


# Don't use request(), because this request PUTs binary data.
def putS3(part, data):
    key = 'putS3'
    start = part["start"]
    end = part["end"]
    res = requests.put(part["href"], data=data[start:end])
    res.raise_for_status()
    etag = res.headers["etag"]
    examples[key] = {
        'method': 'PUT',
        'path': part["href"],
        'url': part["href"],
        'reqbody': '<binary-data>',
        'statusCode': res.status_code,
        'etag': etag,
        'resbody': None,
        'data': None
    }
    return etag


def postComplete(url, parts):
    res = request(
        key='postComplete',
        method='POST',
        path=apiPath +
            '/{repoOwner}/{repoName}/db/blobs/{sha1}/uploads/{uploadId}',
        url=url,
        reqbody={
            's3Parts': parts
        },
        statusCode=201
    )
    return res.json()["data"]


# Process request and record it in examples.
def request(key, method, statusCode, url=None, path=None, reqbody=None,
        params=None):
    params = params or {}
    if not url:
        url = baseUrl + path.format(**params)
    signedUrl = sign_req(method, url)
    if method == 'GET':
        res = requests.get(signedUrl, headers=headers)
    elif method == 'POST':
        res = requests.post(signedUrl, headers=headers, data=json.dumps(reqbody))
    else:
        raise NotImplementedError(
                "Method '{0}' not implemented.".format(method)
                )
    if res.status_code != statusCode:
        raise_res(res)
    resbody = res.json()
    data = resbody["data"]
    examples[key] = {
        'method': method,
        'path': path,
        'url': url,
        'reqbody': reqbody,
        'statusCode': res.status_code,
        'resbody': resbody,
        'data': data
    }
    return res


def printFullDoc():
    for k in toc:
        doc = fmtDoc(k)
        print(doc)


def fmtDoc(key):
    ex = copy(examples[key])
    ex["reqbody"] = stringify(ex["reqbody"])
    ex["resbody"] = stringify(ex["resbody"])
    ex["data"] = stringify(ex["data"])
    # Change Python format fields to Express-style params:
    # `/api/{foo}/db` -> `/api/:foo/db`.
    if ex["path"]:
        ex["path"] = re.sub(r'[{]([^}]+)[}]', r':\1', ex["path"])
    # Remove AWS key details.
    for k in ('reqbody', 'resbody', 'data', 'url'):
        ex[k] = re.sub(r'AK[A-Z0-9]*', r'AKXXXXXXXXXXXXXXXXXX', ex[k])
        ex[k] = re.sub(r'https://[^/]*.amazonaws.com',
                        r'https://some-s3-bucket.amazonaws.com', ex[k])
        ex[k] = re.sub(r'http://localhost:3000', 'https://nog.zib.de',
                        ex[k])
    return dedent(md[key]).format(**ex)


class ApiError(RuntimeError):
    def __init__(self, arg):
        self.arg = arg


def raise_res(res):
    try:
        data = res.json()
    except ValueError:
        data = 'no response JSON.'
    raise ApiError("Unexpected status code {0}: {1}".format(
        res.status_code, data))


def stringify(data):
    return json.dumps(data, indent=2, sort_keys=True)


def sign_req(method, url):
    res = check_output(['../../tools/bin/sign-req', method, url])
    return res.decode('utf-8').rstrip()


def randomdata(length):
    return os.urandom(length)


main()
