#!/usr/bin/python
# 
# Copyright 2014 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Raw network client for HTTP(S) communication with ERMREST service.
"""

import os
import subprocess
import hashlib
import json
import base64
import urlparse
from httplib import HTTPConnection, HTTPSConnection, HTTPException, OK, CREATED, ACCEPTED, NO_CONTENT, CONFLICT, FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, BadStatusLine, CannotSendRequest, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED, NOT_FOUND
import sys
import traceback
import time
import shutil
import smtplib
import urllib
import re
import mimetypes
from email.mime.text import MIMEText
import socket
from dateutil.parser import parse

import httplib
import httplib2
import random

from apiclient.discovery import build
from apiclient.errors import HttpError
from apiclient.http import MediaFileUpload
from oauth2client.client import flow_from_clientsecrets
from oauth2client.file import Storage
from oauth2client.tools import argparser, run_flow

from client_delete import ClientDeleteYouTubeVideo

mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

YOUTUBE_UPLOAD_SCOPE = set(["https://www.googleapis.com/auth/youtube", "https://www.googleapis.com/auth/youtube.upload"])
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
    
httplib2.RETRIES = 1
MAX_RETRIES = 10            
RETRIABLE_EXCEPTIONS = (    
    httplib2.HttpLib2Error, 
    IOError, 
    httplib.NotConnected,
    httplib.IncompleteRead, 
    httplib.ImproperConnectionState,
    httplib.CannotSendRequest, 
    httplib.CannotSendHeader,
    httplib.ResponseNotReady, 
    httplib.BadStatusLine)

RETRIABLE_STATUS_CODES = [500, 502, 503, 504]

MISSING_CLIENT_SECRETS_MESSAGE = "WARNING: Please configure OAuth 2.0"

class ErmrestHTTPException(Exception):
    def __init__(self, value, status, retry=False):
        super(ErmrestHTTPException, self).__init__(value)
        self.value = value
        self.status = status
        self.retry = retry
        
    def __str__(self):
        message = "%s." % self.value
        return message

class ErmrestException(Exception):
    def __init__(self, value, cause=None):
        super(ErmrestException, self).__init__(value)
        self.value = value
        self.cause = cause
        
    def __str__(self):
        message = "%s." % self.value
        if self.cause:
            message += " Caused by: %s." % self.cause
        return message

class MalformedURL(ErmrestException):
    """MalformedURL indicates a malformed URL.
    """
    def __init__(self, cause=None):
        super(MalformedURL, self).__init__("URL was malformed", cause)

class UnresolvedAddress(ErmrestException):
    """UnresolvedAddress indicates a failure to resolve the network address of
    the Ermrest service.
    
    This error is raised when a low-level socket.gaierror is caught.
    """
    def __init__(self, cause=None):
        super(UnresolvedAddress, self).__init__("Could not resolve address of host", cause)

class NetworkError(ErmrestException):
    """NetworkError wraps a socket.error exception.
    
    This error is raised when a low-level socket.error is caught.
    """
    def __init__(self, cause=None):
        super(NetworkError, self).__init__("Network I/O failure", cause)

class ProtocolError(ErmrestException):
    """ProtocolError indicates a protocol-level failure.
    
    In other words, you may have tried to add a tag for which no tagdef exists.
    """
    def __init__(self, message='Network protocol failure', errorno=-1, response=None, cause=None):
        super(ProtocolError, self).__init__("Ermrest protocol failure", cause)
        self._errorno = errorno
        self._response = response
        
    def __str__(self):
        message = "%s." % self.value
        if self._errorno >= 0:
            message += " HTTP ERROR %d: %s" % (self._errorno, self._response)
        return message
    
class NotFoundError(ErmrestException):
    """Raised for HTTP NOT_FOUND (i.e., ERROR 404) responses."""
    pass


class ErmrestClient (object):
    """Network client for ERMREST.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.baseuri = kwargs.get("baseuri")
        o = urlparse.urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.cookie = kwargs.get("cookie")
        self.client_secrets_file = kwargs.get("client_secrets_file")
        self.client_oauth2_file = kwargs.get("client_oauth2_file")
        self.data_scratch = kwargs.get("data_scratch")
        self.ffmpeg = kwargs.get("ffmpeg")
        self.ffprobe = kwargs.get("ffprobe")
        self.category = kwargs.get("category")
        self.keywords = kwargs.get("keywords")
        self.privacyStatus = kwargs.get("privacyStatus")
        self.timeout = kwargs.get("timeout") * 60
        self.limit = kwargs.get("limit")
        self.chunk_size = kwargs.get("chunk_size")
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.header = None
        self.webconn = None
        argparser.add_argument("--config", required=True, help="YouTube configuration file")
        self.args = argparser.parse_args()
        self.args.category = self.category
        self.args.keywords = self.keywords
        self.args.privacyStatus = self.privacyStatus
        self.args.noauth_local_webserver = True
        self.logger.debug('Client initialized.')

    """
    Send requests to the Ermrest service.
    """
    def send_request(self, method, url, body='', headers={}, sendData=False, ignoreErrorCodes=[]):
        try:
            request_headers = headers.copy()
            url = self.url_cid(url)
            if self.header:
                headers.update(self.header)
            self.logger.debug('Sending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
            retry = False
            try:
                if sendData == False:
                    self.webconn.request(method, url, body, headers)
                else:
                    """ 
                    For file upload send the request step by step 
                    """
                    self.webconn.putrequest(method, url)
                    for key,value in headers.iteritems():
                        self.webconn.putheader(key,value)
                    self.webconn.endheaders()
                    self.webconn.send(body)
                resp = self.webconn.getresponse()
                self.logger.debug('Response: %d' % resp.status)
            except socket.error, e:
                retry = True
                self.logger.debug('Socket error: %d' % (e.errno))
            except (BadStatusLine, CannotSendRequest):
                retry = True
            except:
                raise
            if retry:
                """ 
                Resend the request 
                """
                self.close()
                self.connect()
                self.sendMail('WARNING Video: The HTTPSConnection has been restarted', 'The HTTPSConnection has been restarted on "%s://%s".\n' % (self.scheme, self.host))
                self.logger.debug('Resending request: method="%s", url="%s://%s%s"' % (method, self.scheme, self.host, url))
                if sendData == False:
                    self.webconn.request(method, url, body, headers)
                else:
                     self.webconn.putrequest(method, url)
                     for key,value in headers.iteritems():
                         self.webconn.putheader(key,value)
                     self.webconn.endheaders()
                     self.webconn.send(body)
                resp = self.webconn.getresponse()
                self.logger.debug('Response: %d' % resp.status)
            if resp.status in [INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT]:
                """ 
                Resend the request 
                """
                self.close()
                self.connect()
                self.sendMail('WARNING Video: The HTTPSConnection has been restarted', 'HTTP exception: %d.\nThe HTTPSConnection has been restarted on "%s://%s".\n' % (resp.status, self.scheme, self.host))
                self.logger.debug('Resending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
                if sendData == False:
                    self.webconn.request(method, url, body, headers)
                else:
                     self.webconn.putrequest(method, url)
                     for key,value in headers.iteritems():
                         self.webconn.putheader(key,value)
                     self.webconn.endheaders()
                     self.webconn.send(body)
                resp = self.webconn.getresponse()
                self.logger.debug('Response: %d' % resp.status)
            if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
                errmsg = resp.read()
                if resp.status not in ignoreErrorCodes:
                    self.logger.error('Error response: method="%s", url="%s://%s%s", status=%i, error: %s' % (method, self.scheme, self.host, url, resp.status, errmsg))
                else:
                    self.logger.error('Error response: %s' % (errmsg))
                raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, errmsg), resp.status, retry)
            return resp
        except ErmrestHTTPException:
            raise
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got HTTP exception: method="%s", url="%s://%s%s", error="%s"' % (method, self.scheme, self.host, url, str(ev)))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Video: Unexpected Exception', 'Error generated during the HTTP request: method="%s", url="%s://%s%s", error="\n%s\n%s"' % (method, self.scheme, self.host, url, str(ev), ''.join(traceback.format_exception(et, ev, tb))))
            raise

    """
    Open the connection to the Ermrest service.
    """
    def connect(self, reconnect=False):
        if self.scheme == 'https':
            self.webconn = HTTPSConnection(host=self.host, port=self.port)
        elif self.scheme == 'http':
            self.webconn = HTTPConnection(host=self.host, port=self.port)
        else:
            raise ValueError('Scheme %s is not supported.' % self.scheme)

        """
        if self.use_goauth:
            auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
            headers = dict(Authorization='Basic %s' % auth)
            resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, reconnect)
            goauth = json.loads(resp.read())
            self.access_token = goauth['access_token']
            self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
        else:
            #headers = {}
            #headers["Content-Type"] = "application/x-www-form-urlencoded"
            #resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, reconnect)
            #self.header = dict(Cookie=resp.getheader("set-cookie"))
        """
        self.header = {'Cookie': self.cookie}
        
    """
    Close the connection to the Ermrest service.
    The underlying python documentation is not very helpful but it would
    appear that the HTTP[S]Connection.close() could raise a socket.error.
    Thus, this method potentially raises a 'NetworkError'.
    """
    def close(self):
        assert self.webconn
        try:
            self.webconn.close()
        except socket.error as e:
            raise NetworkError(e)
        finally:
            self.webconn = None

    """
    Send email notification
    """
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification.')
                    ready = True
                except socket.gaierror as e:
                    if e.errno == socket.EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    """
    Start the process for uploading videos to YouTube
    """
    def start(self):
        self.connect()
        try:
            self.youtube_authenticated_service()
            if self.youtube is not None:
                self.logger.debug('Authenticated to the YouTube service.')
                self.uploadVideo()
                self.clearYouTube()
            self.deleteVideo()
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Video Processing: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
            raise
        
    """
    Get the YouTube credentials
    """
    def youtube_authenticated_service(self):
        flow = flow_from_clientsecrets(self.client_secrets_file, scope=YOUTUBE_UPLOAD_SCOPE, message=MISSING_CLIENT_SECRETS_MESSAGE)
        storage = Storage(self.client_oauth2_file)
        credentials = storage.get()
        if credentials is None or credentials.invalid:
            credentials = run_flow(flow, storage, self.args)
        self.youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, http=credentials.authorize(httplib2.Http()))
        
    """
    Create a request to upload a video to YouTube
    """
    def youtube_request(self):
        tags = None
        if self.args.keywords:
            tags = self.args.keywords.split(",")
        body=dict(
            snippet=dict(
                title=self.args.title,
                description=self.args.description,
                tags=tags,
                categoryId=self.args.category
            ),
            status=dict(
                privacyStatus=self.args.privacyStatus
            )
        )
        request = self.youtube.videos().insert(
            part=",".join(body.keys()),
            body=body,
            media_body=MediaFileUpload(self.args.file, chunksize=-1, resumable=True)
        )
        return request

    """
    Upload a video to YouTube
    """
    def youtube_upload(self, request):
        response = None
        retry = 0
        id = None
        while response is None:
            error = None
            try:
                self.logger.debug('Uploading file...')
                status, response = request.next_chunk()
                if 'id' in response:
                    id = response['id']
                    self.logger.debug("Video id '%s' was successfully uploaded." % id)
                else:
                    self.logger.error("The upload failed with an unexpected response: %s" % response)
            except HttpError, e:
                if e.resp.status in RETRIABLE_STATUS_CODES:
                    error = "A retriable HTTP error %d occurred:\n%s" % (e.resp.status, e.content)
                else:
                    raise
            except RETRIABLE_EXCEPTIONS, e:
                error = "A retriable error occurred: %s" % e
            if error is not None:
                self.logger.error(error)
                retry += 1
                if retry > MAX_RETRIES:
                    self.logger.error("No longer attempting to retry.")
                    break
                max_sleep = 2 ** retry
                sleep_seconds = random.random() * max_sleep  
                self.logger.debug("Sleeping %f seconds and then retrying..." % sleep_seconds)
                time.sleep(sleep_seconds)
        return id
                
    """
    Delete videos from YouTube
    """
    def clearYouTube(self):
        url = '%s/entity/Common:Delete_Youtube/Youtube_Deleted=FALSE&Record_Type=Immunofluorescence%%3ASlide_Video&!YouTube_MD5::null::&!YouTube_URI::null::' % (self.path)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        videos = json.loads(resp.read())
        videoids = []
        for video in videos:
            videoids.append((video['YouTube_URI'], video['RID']))
                
        self.logger.debug('Deleting from YouTube %d video(s).' % (len(videoids))) 
        if len(videoids) > 0:
            client_delete = ClientDeleteYouTubeVideo(client_secrets_file=self.client_secrets_file,
                                                     client_oauth2_file=self.client_oauth2_file,
                                                     logger=self.logger)
            for youtube_uri,rid in videoids:
                youtube_deleted = client_delete.youtube_delete(youtube_uri)
                if youtube_deleted == True:
                    columns = ["Youtube_Deleted"]
                    columns = ','.join([urllib.quote(col, safe='') for col in columns])
                    url = '%s/attributegroup/Common:Delete_Youtube/RID;%s' % (self.path, columns)
                    body = []
                    obj = {'RID': rid,
                           "Youtube_Deleted": True
                           }
                    body.append(obj)
                    headers = {'Content-Type': 'application/json'}
                    resp = self.send_request('PUT', url, json.dumps(body), headers, False)
                    resp.read()
                    self.logger.debug('SUCCEEDED updated the Common:Delete_Youtube table entry for the YouTube URL: "%s".' % (youtube_uri)) 
        
    """
    Delete videos from hatrac
    """
    def deleteVideo(self):
        url = '%s/entity/Common:Delete_Hatrac/Hatrac_Deleted=FALSE' % (self.path)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        files = json.loads(resp.read())
        fileids = []
        for f in files:
            fileids.append((f['Hatrac_URI'], f['RID']))
                
        self.logger.debug('Deleting from hatrac %d files(s).' % (len(fileids))) 
        for hatrac_uri,rid in fileids:
            parts = []
            for part in hatrac_uri.split('/'):
                parts.append(urllib.quote(part, safe=''))
            url = '%s' % ('/'.join(parts))
            headers = {}
            resp = self.send_request('DELETE', url, '', headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED deleted from hatrac the "%s" file.' % (hatrac_uri)) 
            columns = ["Hatrac_Deleted"]
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Common:Delete_Hatrac/RID;%s' % (self.path, columns)
            body = []
            obj = {'RID': rid,
                   "Hatrac_Deleted": True
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Common:Delete_Hatrac table entry for the Hatrac URL: "%s".' % (hatrac_uri)) 
        
    """
    Upload videos to YouTube
    """
    def uploadVideo(self):
        url = '%s/entity/Immunofluorescence:Slide_Video/!Identifier::null::&!Name::null::&!Bytes::null::&Media_Type=video%%2Fmp4&Processing_Status::null::' % (self.path)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        videos = json.loads(resp.read())
        videoids = []
        for video in videos:
            videoids.append((video['Accession_ID'], video['Name'], video['Title'], video['Description'], video['Identifier'], video['MD5'], video['YouTube_MD5'], video['YouTube_URI'], video['RID'], video['Consortium'], video['MP4_URI'], video['RCT'], video['RMT']))
                
        self.logger.debug('Processing %d video(s).' % (len(videoids))) 
        for movieId,fileName,title,description,uri,md5,youtube_md5,youtube_uri,rid,consortium,mp4_uri,rct,rmt in videoids:
            if description == None:
                description = ''
            consortium_url = ''
            if consortium == 'GUD':
                consortium_url = 'gudmap.org'
            elif consortium == 'RBK':
                consortium_url = 'rebuildingakidney.org'
            f, MP4_URI= self.getVideoFile(fileName, uri, consortium_url, md5)
            if f == None or MP4_URI == None:
                self.reportFailure(movieId, 'error_no_video_file')
                continue
                
            if youtube_uri != None and youtube_md5 != md5:
                """
                We have an update.
                Delete the video from YouTube
                """
                client_delete = ClientDeleteYouTubeVideo(client_secrets_file=self.client_secrets_file,
                                                         client_oauth2_file=self.client_oauth2_file,
                                                         logger=self.logger)
                youtube_deleted = client_delete.youtube_delete(youtube_uri)
                """
                We have an update.
                Insert the old video into the Delete_Youtube table
                """
                self.logger.debug('Inserting the old video "%s" file into the Delete_Youtube table.' % (fileName))
                url = '%s/entity/Common:Delete_Youtube' % (self.path)
                body = []
                obj = {'YouTube_MD5': youtube_md5,
                       'YouTube_URI': youtube_uri,
                       'Youtube_Deleted': youtube_deleted,
                       'Record_Type': 'Immunofluorescence:Slide_Video',
                       'Record_RID': rid,
                       'Record_Deleted': False
                       }
                body.append(obj)
                headers = {'Content-Type': 'application/json'}
                resp = self.send_request('PUT', url, json.dumps(body), headers, False)
                resp.read()
                self.logger.debug('SUCCEEDED updated the entry for the "%s" file in the Common:Delete_Youtube table.' % (fileName)) 
                
                if mp4_uri != None:
                    """
                    We have an update.
                    Insert the old video into the Delete_Hatrac table
                    """
                    self.logger.debug('Inserting the old MP4 video "%s" file into the Delete_Hatrac table.' % (fileName))
                    url = '%s/entity/Common:Delete_Hatrac' % (self.path)
                    body = []
                    obj = {'Hatrac_MD5': mp4_uri.split('/')[-1],
                           'Hatrac_URI': mp4_uri,
                           'Hatrac_Deleted': False,
                           'Record_Type': 'Immunofluorescence:Slide_Video',
                           'Record_RID': rid,
                           'Record_RCT': rct,
                           'Record_RMT': rmt,
                           'Record_Deleted': False
                           }
                    body.append(obj)
                    headers = {'Content-Type': 'application/json'}
                    resp = self.send_request('PUT', url, json.dumps(body), headers, False)
                    resp.read()
                    self.logger.debug('SUCCEEDED updated the entry for the "%s" file in the Common:Delete_Youtube table.' % (fileName)) 
            
            self.logger.debug('Uploading the video "%s" to YouTube' % (fileName))
            
            """
            Get the video properties
            """
            cfg = self.getVideoProperties(f)
            if cfg != None:
                width,height = self.getVideoResolution(cfg)
                self.logger.debug('Video resolution: (%d x %d).' % (width, height)) 
            else:
                self.logger.debug('Could not get the video resolution.') 
                
            """
            Initialize YouTube video parameters
            """
            self.args.file = f
            self.args.title = ('%s:\n%s' % (consortium_url, title))[:64]
            self.args.description = description
            
            """
            Upload video to YouTube
            """
            try:
                request = self.youtube_request()
                if request is not None:
                    id = self.youtube_upload(request)
                    returncode = 0
                else:
                    returncode = 1
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Video: YouTube ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                returncode = 1
            
            if returncode != 0:
                self.logger.error('Can not upload to YouTube the "%s" file.' % (fileName)) 
                self.sendMail('FAILURE YouTube', 'Can not upload to YouTube the "%s" file.' % (fileName))
                os.remove(f)
                """
                Update the Slide_Video table with the failure result.
                """
                self.reportFailure(movieId, 'error_youtube_upload')
                continue
                
            """
            Upload the Slide_Video table with the SUCCESS status
            """
            columns = ["MP4_URI", "YouTube_MD5", "YouTube_URI", "Processing_Status"]
            #youtube_uri = "https://www.youtube.com/embed/%s?showinfo=0&rel=0" % id
            youtube_uri = "https://www.youtube.com/embed/%s?rel=0" % id
            os.remove(f)
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Immunofluorescence:Slide_Video/Accession_ID;%s' % (self.path, columns)
            body = []
            obj = {'Accession_ID': movieId,
                   'MP4_URI': MP4_URI,
                   'YouTube_URI': youtube_uri,
                   'YouTube_MD5': md5,
                   'Processing_Status': 'success'
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the entry for the "%s" file.' % (fileName)) 
        self.logger.debug('Ended uploading videos to YouTube.') 
        
    """
    Update the Slide_Video table with the ERROR status
    """
    def reportFailure(self, movieId, error_message):
        """
            Update the Slide_Video table with the YouTube Upload failure result.
        """
        try:
            columns = ["Processing_Status"]
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Immunofluorescence:Slide_Video/Accession_ID;%s' % (self.path, columns)
            body = []
            obj = {'Accession_ID': movieId,
                   "Processing_Status": '%s' % error_message
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Slide_Video table for the video Accession_ID "%s"  with the Processing_Status result "%s".' % (movieId, error_message)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Video: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
        
    """
    Get the video file from hatrac
    """
    def getVideoFile(self, fileName, uri, consortium_url, md5):
        try:
            MP4_URI = None
            self.logger.debug('Processing file: "%s".' % (fileName)) 
            movieFile = '%s/%s' % (self.data_scratch, fileName)
            url = '%s' % (uri)
            headers = {'Accept': '*'}
            resp = self.send_request('GET', url, '', headers, False)
            self.logger.debug('content-length: %s.' % (resp.getheader('content-length'))) 
            #self.logger.debug('response headers: %s.' % (resp.getheaders())) 
            block_sz = 8192
            f = open(movieFile, 'wb')
            while True:
                buffer = resp.read(block_sz)
                if not buffer:
                    break
                f.write(buffer)
            f.close()
            self.logger.debug('File "%s", %d bytes.' % (movieFile, os.stat(movieFile).st_size)) 
            #"""
            drawFile = self.drawVideoText(movieFile, consortium_url, 'upper left')
            if drawFile != None:
                self.logger.debug('File with text "%s", %d bytes.' % (drawFile, os.stat(drawFile).st_size)) 
                os.remove(movieFile)
                newFile = drawFile
                file_size = os.path.getsize(newFile)
                new_md5sum = self.md5sum(newFile, self.chunk_size)
                new_md5 = self.md5hex(newFile)
                new_sha256 = self.sha256sum(newFile)
                new_uri = '%s%s' % (uri[0:-len(md5)], new_md5)
                metadata = {"content_disposition": "filename*=UTF-8''%s" % fileName}
                if self.get_md5sum(new_uri) == new_md5sum:
                    self.logger.info('Skipping the upload of the file "%s" as it already exists hatrac.' % fileName)
                    MP4_URI = new_uri
                else:
                    try:
                        self.uploadFile(new_uri, newFile, self.chunk_size, metadata)
                        MP4_URI = new_uri
                    except:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('Can not transfer file "%s" in namespace "%s". Error: "%s"' % (fileName, new_uri, str(ev)))
                        self.sendMail('Failure YouTube: HATRAC Error', 'Can not upload file "%s" in namespace "%s". Error: "%s"' % (fileName, new_uri, str(ev)))
                        self.reportFailure(movieId, 'error_hatrac_upload')
                return (drawFile, MP4_URI)
            #"""
            return (movieFile,MP4_URI)
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not get from hatrac the video file "%s"\n"%s"' % (fileName, str(ev)))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE YouTube: getVideoFile ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            return (None, None)

    """
    Append the cid=video string to the url query
    """
    def url_cid(self, url):
        """
        """
        ret = url
        o = urlparse.urlparse(url)
        if o.path.startswith('/ermrest/'):
            delimiter = '?'
            try:
                o = urlparse.urlparse(url)
                if o.query != '':
                    delimiter = '&'
                ret = '%s%scid=youtube' % (url, delimiter)
            except:
                pass
        return ret

    """
    Get the properties of the video file
    """
    def getVideoProperties(self, filename):
        """
        """
        ret = None
        try:
            args = [self.ffprobe, '-v', 'quiet', '-print_format', 'json', '-show_streams', '-i', '%s' % filename]
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdoutdata, stderrdata = p.communicate()
            returncode = p.returncode
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE YouTube: ffprobe ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            returncode = 1
            
        if returncode != 0:
            self.logger.error('Can not get the video properties of the "%s" file.\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
            self.sendMail('FAILURE YouTube', 'Can not get the video properties of the "%s" file.\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
        else:
            ret = json.loads(stdoutdata)

        return ret

        
    """
    Get the properties of the video file
    """
    def getVideoResolution(self, video):
        """
        """
        return (video.get('streams',None)[0].get('width',0), video.get('streams',None)[0].get('height',0))

    """
    Get the offset of the video text
    """
    def getTextOffset(self, position):
        """
        """
        if position == 'upper left':
            x = '5'
            y = '0'
        elif position == 'lower left':
            x = '5'
            y = '(h-text_h-5)'
        elif position == 'upper right':
            x = '(w-text_w-5)'
            y = '0'
        elif position == 'lower right':
            x = '(w-text_w-5)'
            y = '(h-text_h-5)'
        else:
            x = None
            y = None
            
        return (x, y)

    """
    Get the fontsize of the video text
    """
    def getTextFontSize(self, width, height):
        """
        """
        if height >= 1080:
            fontsize = 44
        elif height >= 720:
            fontsize = 36
        else:
            fontsize = None
            
        return fontsize

    """
    Draw the video text
    """
    def drawVideoText(self, filename, videoText, position):
        """
        """
        ret = None
        videoProperties = self.getVideoProperties(filename)
        if videoProperties != None:
            width,height = self.getVideoResolution(videoProperties)
            if height != None:
                x,y = self.getTextOffset(position)
                if x != None and y != None:
                    fontsize = self.getTextFontSize(width, height)
                    if fontsize != None:
                        file_name, file_extension = os.path.splitext(filename)
                        outputFile = '%s_%d%s' % (file_name, height, file_extension)
                        try:
                            #args = [self.ffmpeg, '-y', '-v', 'quiet', '-i', '%s' % filename, '-vf', 'drawtext=\'text=\'\'%s\'\': fontcolor=blue: fontsize=%d: box=1: boxcolor=white@0.5: boxborderw=5: x=%s: y=%s\'' % (videoText, fontsize, x, y), '-codec:a', 'copy', '%s' %  outputFile]
                            args = [self.ffmpeg, '-y', '-i', '%s' % filename, '-vf', 'drawtext=\'text=\'\'%s\'\': fontcolor=blue: fontsize=%d: box=1: boxcolor=white@0.5: boxborderw=5: x=%s: y=%s\'' % (videoText, fontsize, x, y), '-codec:a', 'copy', '%s' %  outputFile]
                            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                            stdoutdata, stderrdata = p.communicate()
                            returncode = p.returncode
                        except:
                            et, ev, tb = sys.exc_info()
                            self.logger.error('got unexpected exception "%s"' % str(ev))
                            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                            self.sendMail('FAILURE YouTube: ffmpeg ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                            returncode = 1
                            
                        if returncode != 0:
                            print 'returncode=%d' % returncode
                            self.logger.error('Can not draw text to the video "%s" file.\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
                            self.sendMail('FAILURE YouTube', 'Can not draw text to the video "%s" file.\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
                        else:
                            ret = outputFile

        return ret

    """
    Upload a file.
    """
    def uploadFile(self, object_url, filePath, chunk_size, metadata):
        try:
            job_id = self.createUploadJob(object_url, filePath, chunk_size, metadata)
            self.chunksUpload(object_url, filePath, job_id, chunk_size)
            self.chunksUploadFinalization(object_url, job_id)
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not upload file "%s" in namespace "%s://%s%s". Error: "%s"' % (filePath, self.scheme, self.host, object_url, str(ev)))
            raise

    """
    Create a job for uploading a file.
    """
    def createUploadJob(self, object_url, filePath, chunk_size, metadata):
        try:
            md5,sha256 = self.content_checksum(filePath, chunk_size)
            content_checksum = {"content-md5": md5,
                              "content-sha256": sha256}
            content_disposition = metadata.get('content_disposition', None)
            file_size = os.path.getsize(filePath)
            url = '%s;upload' % object_url
            headers = {'Content-Type': 'application/json'}
            if mimetypes.inited == False:
                mimetypes.init()
            content_type,encoding = mimetypes.guess_type(filePath)
            if content_type == None:
                content_type = 'application/octet-stream'
            obj = {"chunk-length": chunk_size,
                   "content-length": file_size,
                   "content-type": content_type}
            obj.update(content_checksum)
            if content_disposition != None:
                obj['content-disposition'] = self.encode_disposition(content_disposition)
            self.logger.debug('hatrac metadata: "%s"\n' % (json.dumps(obj)))
            resp = self.send_request('POST', url, body=json.dumps(obj), headers=headers)
            res = resp.read()
            job_id = res.split('/')[-1][:-1]
            self.logger.debug('Created job_id "%s" for url "%s://%s%s".' % (job_id, self.scheme, self.host, url))
            return job_id
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not create job for uploading file "%s" in object "%s://%s%s". Error: "%s"' % (filePath, self.scheme, self.host, object_url, str(ev)))
            raise

    """
    Upload a file through chunks.
    """
    def chunksUpload(self, object_url, filePath, job_id, chunk_size):
        try:
            file_size = os.path.getsize(filePath)
            chunk_no = file_size / chunk_size
            last_chunk_size = file_size % chunk_size
            f = open(filePath, "rb")
            for index in range(chunk_no):
                position = index
                body = f.read(chunk_size)
                url = '%s;upload/%s/%d' % (object_url, job_id, position)
                headers = {'Content-Type': 'application/octet-stream', 'Content-Length': '%d' % chunk_size}
                resp = self.send_request('PUT', url, body=body, headers=headers, sendData=True)
                res = resp.read()
            if last_chunk_size > 0:
                position = chunk_no
                body = f.read(chunk_size)
                url = '%s;upload/%s/%d' % (object_url, job_id, position)
                headers = {'Content-Type': 'application/octet-stream', 'Content-Length': '%d' % last_chunk_size}
                resp = self.send_request('PUT', url, body=body, headers=headers, sendData=True)
                res = resp.read()
            f.close()
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not upload chunk for file "%s" in namespace "%s://%s%s" and job_id "%s". Error: "%s"' % (filePath, self.scheme, self.host, url, job_id, str(ev)))
            try:
                f.close()
                self.cancelJob(object_url, job_id)
            except:
                pass
            raise
            
    """
    Finalize the chunks upload.
    """
    def chunksUploadFinalization(self, object_url, job_id):
        try:
            url = '%s;upload/%s' % (object_url, job_id)
            headers = {}
            resp = self.send_request('POST', url, headers=headers)
            res = resp.read()
            return res
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not finalize job "%s" for object "%s://%s%s". Error: "%s"' % (job_id, self.scheme, self.host, url, str(ev)))
            raise
            
    """
    Get the base64 digest strings like the sha256 and the md5 utilities would compute.
    """
    def content_checksum(self, fpath, chunk_size):
        hmd5 = hashlib.md5()
        hsha256 = hashlib.sha256()
        try:
            f = open(fpath, 'rb')
            try:
                b = f.read(chunk_size)
                while b:
                    hmd5.update(b)
                    hsha256.update(b)
                    b = f.read(chunk_size)
                return (base64.b64encode(hmd5.digest()), base64.b64encode(hsha256.digest()))
            finally:
                f.close()
        except:
            return (None, None)

    """
    Encode the content-disposition.
    """
    def encode_disposition(self, orig):
        m = re.match("^filename[*]=UTF-8''(?P<name>[-_.~A-Za-z0-9%]+)$", orig)
        if m:
            return orig
        elif not orig.startswith("filename*=UTF-8''"):
            raise ValueError('Cannot accept content-disposition "%s"; it must start with "filename*=UTF-8\'\'".' % orig)
        else:
            ret = ["filename*=UTF-8''"]
            for c in orig[len("filename*=UTF-8''"):]:
                m = m = re.match("(?P<name>[-_.~A-Za-z0-9%]+)$", c)
                if m:
                    ret.append(c)
                else:
                    #In case we want an URL encoding
                    #ret.append('%%%s' % c.encode('hex').upper())
                    ret.append('_')
            return ''.join(ret)

    """
    Cancel a job.
    """
    def cancelJob(self, object_url, job_id):
        try:
            url = '%s;upload/%s' % (object_url, job_id)
            headers = {}
            resp = self.send_request('DELETE', url, headers=headers)
            res = resp.read()
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not cancel job "%s" for object "%s://%s%s". Error: "%s"' % (job_id, self.scheme, self.host, url, str(ev)))
            raise
            
    """
    Get the md5sum file from hatrac.
    """
    def get_md5sum(self, url):
        """
            Retrieve the md5sum of a file
        """
        ret = None
        if url != None:
            headers = {'Accept': '*'}
            try:
                resp = self.send_request('HEAD', url, '', headers=headers, ignoreErrorCodes=[NOT_FOUND])
                resp.read()
                ret = resp.getheader('content-md5', None)
            except:
                pass
        return ret
                
    """
    Get the hexa md5 checksum of the file.
    """
    def md5hex(self, fpath):
        h = hashlib.md5()
        try:
            f = open(fpath, 'rb')
            try:
                b = f.read(4096)
                while b:
                    h.update(b)
                    b = f.read(4096)
                return h.hexdigest()
            finally:
                f.close()
        except:
            return None

    """
    Get the checksum of the file.
    """
    def sha256sum(self, fpath):
        h = hashlib.sha256()
        try:
            f = open(fpath, 'rb')
            try:
                b = f.read(4096)
                while b:
                    h.update(b)
                    b = f.read(4096)
                return h.hexdigest()
            finally:
                f.close()
        except:
            return None

    """
    Get the base64 digest string like md5 utility would compute.
    """
    def md5sum(self, fpath, chunk_size):
        h = hashlib.md5()
        try:
            f = open(fpath, 'rb')
            try:
                b = f.read(chunk_size)
                while b:
                    h.update(b)
                    b = f.read(chunk_size)
                return base64.b64encode(h.digest())
            finally:
                f.close()
        except:
            return None

        
