#!/usr/bin/python3
# 
# Copyright 2014 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Raw network client for HTTP(S) communication with ERMREST service.
"""

import os
import subprocess
import json
import base64
from http.client import HTTPConnection, HTTPSConnection, HTTPException, OK, CREATED, ACCEPTED, NO_CONTENT, CONFLICT, FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, BadStatusLine, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED
import sys
import traceback
import time
import shutil
import smtplib
import urllib
from urllib.parse import urlparse
import re
import czifile
from email.mime.text import MIMEText
from .bioformats import BioformatsClient
from socket import gaierror, EAI_AGAIN
from lxml.etree import XMLSyntaxError
from lxml import etree
from dateutil.parser import parse
mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

class ErmrestHTTPException(Exception):
    def __init__(self, value, status):
        super(ErmrestHTTPException, self).__init__(value)
        self.value = value
        self.status = status
        
    def __str__(self):
        message = "%s." % self.value
        return message

class ErmrestException(Exception):
    def __init__(self, value, cause=None):
        super(ErmrestException, self).__init__(value)
        self.value = value
        self.cause = cause
        
    def __str__(self):
        message = "%s." % self.value
        if self.cause:
            message += " Caused by: %s." % self.cause
        return message

class MalformedURL(ErmrestException):
    """MalformedURL indicates a malformed URL.
    """
    def __init__(self, cause=None):
        super(MalformedURL, self).__init__("URL was malformed", cause)

class UnresolvedAddress(ErmrestException):
    """UnresolvedAddress indicates a failure to resolve the network address of
    the Ermrest service.
    
    This error is raised when a low-level socket.gaierror is caught.
    """
    def __init__(self, cause=None):
        super(UnresolvedAddress, self).__init__("Could not resolve address of host", cause)

class NetworkError(ErmrestException):
    """NetworkError wraps a socket.error exception.
    
    This error is raised when a low-level socket.error is caught.
    """
    def __init__(self, cause=None):
        super(NetworkError, self).__init__("Network I/O failure", cause)

class ProtocolError(ErmrestException):
    """ProtocolError indicates a protocol-level failure.
    
    In other words, you may have tried to add a tag for which no tagdef exists.
    """
    def __init__(self, message='Network protocol failure', errorno=-1, response=None, cause=None):
        super(ProtocolError, self).__init__("Ermrest protocol failure", cause)
        self._errorno = errorno
        self._response = response
        
    def __str__(self):
        message = "%s." % self.value
        if self._errorno >= 0:
            message += " HTTP ERROR %d: %s" % (self._errorno, self._response)
        return message
    
class NotFoundError(ErmrestException):
    """Raised for HTTP NOT_FOUND (i.e., ERROR 404) responses."""
    pass


class ErmrestClient (object):
    """Network client for ERMREST.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.metadata = kwargs.get("metadata")
        self.baseuri = kwargs.get("baseuri")
        o = urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.use_goauth = kwargs.get("use_goauth")
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.thumbnails = kwargs.get("thumbnails")
        self.dzi = kwargs.get("dzi")
        self.czi2dzi = kwargs.get("czi2dzi")
        self.viewer = kwargs.get("viewer")
        self.czi = kwargs.get("czi")
        self.czirules = kwargs.get("czirules")
        self.showinf = kwargs.get("showinf")
        self.convert = kwargs.get("convert")
        self.data_scratch = kwargs.get("data_scratch")
        self.timeout = kwargs.get("timeout") * 60
        self.limit = kwargs.get("limit")
        self.hatrac = kwargs.get("hatrac")
        self.namespace = kwargs.get("namespace")
        self.cookie = kwargs.get("cookie")
        self.chunk_size = kwargs.get("chunk_size")
        self.header = None
        self.webconn = None
        self.logger.debug('Client initialized.')

    def send_request(self, method, url, body='', headers={}, reconnect=False):
        if self.header:
            headers.update(self.header)
        try:
            self.webconn.request(method, url, body, headers)
        except:
            self.logger.debug('Could not sent the request. Reconnecting...') 
            self.close()
            self.connect(True)
            self.webconn.request(method, url, body, headers)
            
        try:
            resp = self.webconn.getresponse()
        except BadStatusLine:
            if not reconnect:
                self.close()
                self.connect(True)
                #self.sendMail('WARNING Tiles: HTTP BadStatusLine exception', 'The HTTPSConnection has been restarted\n')
                self.webconn.request(method, url, body, headers)
                resp = self.webconn.getresponse()
            else:
                raise
        if not reconnect and resp.status in [FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED, CONFLICT]:
            self.close()
            self.connect(True)
            #self.sendMail('WARNING Tiles: HTTP exception: %d' % resp.status, 'The HTTPSConnection has been restarted\n')
            self.webconn.request(method, url, body, headers)
            resp = self.webconn.getresponse()
        if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
            raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, resp.read()), resp.status)
        return resp

    def connect(self, reconnect=False):
        if self.scheme == 'https':
            self.webconn = HTTPSConnection(host=self.host, port=self.port)
        elif self.scheme == 'http':
            self.webconn = HTTPConnection(host=self.host, port=self.port)
        else:
            raise ValueError('Scheme %s is not supported.' % self.scheme)

        """
        if self.use_goauth:
            auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
            headers = dict(Authorization='Basic %s' % auth)
            resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, reconnect)
            goauth = json.loads(resp.read())
            self.access_token = goauth['access_token']
            self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
        else:
            #headers = {}
            #headers["Content-Type"] = "application/x-www-form-urlencoded"
            #resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, reconnect)
            #self.header = dict(Cookie=resp.getheader("set-cookie"))
        """
        self.header = {'Cookie': self.cookie}
        
    def close(self):
        """Closes the connection to the Ermrest service.
        
        The underlying python documentation is not very helpful but it would
        appear that the HTTP[S]Connection.close() could raise a socket.error.
        Thus, this method potentially raises a 'NetworkError'.
        """
        assert self.webconn
        try:
            self.webconn.close()
        except socket.error as e:
            raise NetworkError(e)
        finally:
            self.webconn = None

    def writeThumbnailFile(self, slide_id, scan_id):
        scanDir='%s/%s' % (self.dzi, scan_id)
        channels = []
        for channel in os.listdir(scanDir):
            if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
               channels.append( channel)
        outdir = '%s/%s' % (self.thumbnails, slide_id)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        try:
            args = [self.convert, '-border', '2', '-bordercolor', 'grey', '%s/%s/%s/0/0_0.jpg' % (self.dzi, scan_id, channels[0]), '%s/%s.jpg' % (outdir, scan_id)]
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdoutdata, stderrdata = p.communicate()
            returncode = p.returncode
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got convert exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            returncode = 1
        
        if returncode != 0:
            self.logger.error('Can not add border for thumbnail "%s/%s/%s/0/0_0.jpg".\nstdoutdata: %s\nstderrdata: %s\n' % (self.dzi, scan_id, channels[0], stdoutdata, stderrdata)) 
            shutil.copyfile('%s/%s/%s/0/0_0.jpg' % (self.dzi, scan_id, channels[0]), '%s/%s.jpg' % (outdir, scan_id))
        
        thumbnail = '/thumbnails/%s/%s.jpg' % (urllib.parse.quote(slide_id, safe=''), urllib.parse.quote(scan_id, safe=''))
        urls = []
        for channel in channels:
            urls.append('url=/data/%s/%s/ImageProperties.xml' % (urllib.parse.quote(scan_id, safe=''), channel))
        return (thumbnail, '&'.join(urls))
            
        
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification')
                    ready = True
                except gaierror as e:
                    if e.errno == EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    def start(self):
        self.connect()
        ready = False
        bftools_only = os.getenv('BFTOOLS_ONLY', 'f').lower() in ['t', 'true']
        while ready == False:
            try:
                if self.namespace == 'resources/histological_images':
                    if bftools_only:
                        self.bftoolsHistologicalImages()
                    else:
                        self.processHistologicalImages()
                else:
                    if bftools_only:
                        self.bftoolsScans()
                    else:
                        self.processScans()
                ready = True
                #time.sleep(self.timeout)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
                raise
        
    def processScans(self):
        url = '%s/entity/Scan/!bytes::null::&DZI::null::&czi2dzi::null::@sort(%s::desc::)?limit=%d' % (self.path,urllib.parse.quote('File Date', safe=''),self.limit)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        scans = json.loads(resp.read())
        scanids = []
        for scan in scans:
            if self.hasCziFile(scan['slide_id'], scan['id'], scan['Disambiguator']) == True:
                scanids.append((scan['slide_id'], scan['id'], scan['Disambiguator']))
            else:
                self.reportFailure(scan['slide_id'], scan['id'], 'missing file')
                
        self.logger.debug('Processing %d scan(s).' % (len(scanids))) 
        for slideId,scanId,disambiguator in scanids:
            f = self.getCziFile(slideId, scanId, disambiguator)
            mdate = self.getAcquisitionDate(f)
            self.logger.debug('Converting czi to dzi tiles for slide "%s", scan "%d"' % (slideId, scanId))
            
            try:
                currentDirectory=os.getcwd()
                os.chdir(self.dzi)
                args = [self.czi2dzi, f, '%s-%d' % (slideId, disambiguator)]
                p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdoutdata, stderrdata = p.communicate()
                returncode = p.returncode
                os.chdir(currentDirectory)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.chdir(currentDirectory)
                returncode = 1
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s", slide "%s", scan "%d".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s", slide "%s", scan "%d".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata))
                os.remove(f)
                #os.rename('%s', '%s.err' % (f, f))
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'czi2dzi error')
                continue
            try:
                thumbnail,urls = self.writeThumbnailFile(slideId, '%s-%d' % (slideId, disambiguator))
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.remove(f)
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'DZI failure')
                continue
                
            self.logger.debug('Extracting metadata for slide "%s", scan "%d"' % (slideId, scanId)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            columns = ["Thumbnail","DZI","czi2dzi", "HTTP URL", "uri", "Acquisition Date"]
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
                columns.extend(self.metadata)
            
            os.remove(f)
            columns = ','.join([urllib.parse.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/id;%s' % (self.path, columns)
            body = []
            obj = {'id': scanId,
                   'Thumbnail': thumbnail,
                   'Acquisition Date': mdate,
                   'DZI': '/%s?%s' % (self.viewer, urls),
                   'uri': '/%s?%s' % (self.viewer, urls),
                   'HTTP URL': '%s/%s/%s/%s-%d.czi' % (self.hatrac, self.namespace, urllib.parse.quote(slideId, safe=''), urllib.parse.quote(slideId, safe=''), disambiguator),
                   "czi2dzi": 'success'
                   }
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the tiles directory for the slide id "%s" and scan id "%d".' % (slideId, scanId)) 
            #self.sendMail('SUCCEEDED Tiles', 'The tiles directory for the slide id "%s" and scan id "%s" was created.\n' % (slideId, scanId))
        self.logger.debug('Ended Scans Processing.') 
        
    def bftoolsScans(self):
        url = '%s/entity/Scan/!bytes::null::&Channels::null::&czi2dzi=success' % (self.path)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        scans = json.loads(resp.read())
        scanids = []
        for scan in scans:
            scanids.append((scan['slide_id'], scan['id'], scan['HTTP URL']))
                
        self.logger.debug('Extracting metadata for %d scan(s).' % (len(scanids))) 
        for slideId,scanId,file_url in scanids:
            f = self.getCziScanFile(slideId, scanId, file_url)
            self.logger.debug('Extracting metadata for slide "%s", scan "%d"' % (slideId, scanId)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            columns = []
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
                columns.extend(self.metadata)
            
            os.remove(f)
            
            if returncode == 1:
                continue
            
            columns = ','.join([urllib.parse.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/id;%s' % (self.path, columns)
            body = []
            obj = {'id': scanId
                   }
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED extracted metadata for the slide id "%s" and scan id "%d".' % (slideId, scanId)) 
        self.logger.debug('Ended Extracting Metadata for Scans.') 
        
    def reportFailure(self, slideId, scanId, error_message):
        """
            Update the Scan table with the czi2dzi failure result.
        """
        try:
            columns = ["Thumbnail","czi2dzi"]
            columns = ','.join([urllib.parse.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/id;%s' % (self.path, columns)
            body = []
            obj = {'id': scanId,
                   'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                   "czi2dzi": '%s' % error_message
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Scan table for the slide id "%s" and scan id "%d" with the czi2dzi result.' % (slideId, scanId)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
        
    def hasCziFile(self, slideId=None, scanId=None, disambiguator=None):
        """
            Check if the file exists in hatrac
        """
        ret = False
        try:
            url = '%s/%s/%s/%s-%d.czi' % (self.hatrac, self.namespace, urllib.parse.quote(slideId, safe=''), urllib.parse.quote(slideId, safe=''), disambiguator)
            headers = {'Accept': '*'}
            resp = self.send_request('HEAD', url, '', headers, False)
            resp.read()
            ret = True
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('HEAD exception "%s" for the slide id "%s" and scan id "%d"' % (str(ev), slideId, scanId))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: Missing file', '%s\n' % 'HEAD exception "%s" for the slide id "%s" and scan id "%d"' % (str(ev), slideId, scanId))
            pass
        return ret
                
    def getCziFile(self, slideId, scanId, disambiguator):
        cziFile = '%s/%s/%s-%d.czi' % (self.data_scratch, self.namespace, slideId, disambiguator)
        outdir = '%s/%s' % (self.data_scratch, self.namespace)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        url = '%s/%s/%s/%s-%d.czi;versions' % (self.hatrac, self.namespace, urllib.parse.quote(slideId, safe=''), urllib.parse.quote(slideId, safe=''), disambiguator)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        srcFile = urllib.parse.unquote('%s%s'  % (self.czi, json.loads(resp.read())[0]))
        shutil.copyfile(srcFile, cziFile)
        return cziFile

    def getCziScanFile(self, slideId, scanId, file_url):
        cziFile = '%s/%s/%s-1.czi' % (self.data_scratch, self.namespace, slideId)
        outdir = '%s/%s' % (self.data_scratch, self.namespace)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        url = '%s;versions' % (file_url)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        self.logger.debug('getCziScanFile URL: "%s"' % (url))
        resp = self.send_request('GET', url, '', headers, False)
        srcFile = urllib.parse.unquote('%s%s'  % (self.czi, json.loads(resp.read())[0]))
        shutil.copyfile(srcFile, cziFile)
        return cziFile

    def getAcquisitionDate(self, cziFile):
        mdate = time.strftime('%Y-%m-%d')
        try:
            cf = czifile.CziFile(cziFile)
            cm = etree.fromstring(cf.metadata.encode('utf-8'))
            mdate = cm.findall('Metadata/Information/Image/AcquisitionDateAndTime')[0].text[:10]
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('Can not extract the Acquisition Date from the "%s" file.' % (cziFile))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
        return mdate

    def getHistologicalFile(self, filename, file_url):
        cziFile = '%s/%s/%s' % (self.data_scratch, self.namespace, filename)
        outdir = '%s/%s' % (self.data_scratch, self.namespace)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        srcFile = urllib.parse.unquote('%s%s'  % (self.czi, self.getHatracLocation(file_url))) 
        shutil.copyfile(srcFile, cziFile)
        return cziFile

    def writeThumbnailImage(self, year, md5):
        scanDir='%s/%s/%s' % (self.dzi, year, md5)
        channels = []
        for channel in os.listdir(scanDir):
            if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
               channels.append( channel)
        outdir = '%s/%s' % (self.thumbnails, year)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        shutil.copyfile('%s/%s/%s/%s/0/0_0.jpg' % (self.dzi, year, md5, channels[0]), '%s/%s.jpg' % (outdir, md5))
        thumbnail = '/thumbnails/%s/%s.jpg' % (urllib.parse.quote(year, safe=''), urllib.parse.quote(md5, safe=''))
        urls = []
        for channel in channels:
            urls.append('url=/data/%s/%s/%s/ImageProperties.xml' % (year, md5, channel))
        return (thumbnail, '&'.join(urls))
            
    def reportImageFailure(self, slideId, filename, error_message):
        """
            Update the Image table with the czi2dzi failure result.
        """
        try:
            columns = ["Thumbnail","Processing_Status"]
            columns = ','.join([urllib.parse.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Histological_Images:HE_Slide/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': slideId,
                   'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                   "Processing_Status": '%s' % error_message
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Image table for the file "%s" with the czi2dzi result "%s".' % (filename, error_message)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
        
    def processHistologicalImages(self):
        url = '%s/entity/Histological_Images:HE_Slide/!File_Bytes::null::&Pyramid_URL::null::&Processing_Status::null::@sort(%s::desc::)?limit=%d' % (self.path,urllib.parse.quote('RCT', safe=''),self.limit)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        slides = json.loads(resp.read())
        slideids = []
        for slide in slides:
            slideids.append((slide['ID'], slide['Filename'], slide['File_URL'], slide['RCT'], slide['File_MD5'], slide['Name']))
                
        self.logger.debug('Processing %d slides(s).' % (len(slideids))) 
        for slideId,filename,file_url,creation_time,md5,name in slideids:
            f = self.getHistologicalFile(filename, file_url)
            mdate = self.getAcquisitionDate(f)
            year = parse(creation_time).strftime("%Y")
            self.logger.debug('Converting czi to dzi tiles for file "%s"' % (filename))
            
            try:
                currentDirectory=os.getcwd()
                os.chdir(self.dzi)
                outdir = '%s/%s/%s' % (self.dzi, year, md5)
                if not os.path.exists(outdir):
                    os.makedirs(outdir)
                args = [self.czi2dzi, f, '%s' % outdir]
                p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdoutdata, stderrdata = p.communicate()
                returncode = p.returncode
                os.chdir(currentDirectory)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.chdir(currentDirectory)
                returncode = 1
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, stdoutdata, stderrdata))
                os.remove(f)
                """
                Update the slide table with the failure result.
                """
                self.reportImageFailure(slideId, filename, 'czi2dzi error')
                continue
            try:
                thumbnail,urls = self.writeThumbnailImage(year, md5)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.remove(f)
                """
                Update the slide table with the failure result.
                """
                self.reportImageFailure(slideId, filename, 'DZI failure')
                continue
                
            self.logger.debug('Extracting metadata for filename "%s"' % (filename)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
            
            os.remove(f)
            columns = ["Thumbnail","Pyramid_URL","Processing_Status","uri"]
            columns = ','.join([urllib.parse.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Histological_Images:HE_Slide/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': slideId,
                   'Thumbnail': thumbnail,
                   'Pyramid_URL': '/%s?%s' % (self.viewer, urls),
                   'uri': '/%s?%s' % (self.viewer, urls),
                   "Processing_Status": 'success'
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the tiles directory for the file "%s".' % (filename)) 
            
            """
            INSERT/UPDATE the image entry with the metadata
            """
            url = '%s/entity/Histological_Images:HE_Image' % self.path
            body = []
            obj = {}
            obj['ID'] = slideId
            obj['Name'] = name
            obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the image entry for the file "%s".' % (filename)) 
        self.logger.debug('Ended Slides Processing.') 
        
    def bftoolsHistologicalImages(self):
        url = '%s/entity/Histological_Images:HE_Image/Channels::null::/Histological_Images:HE_Slide/!File_Bytes::null::&!Pyramid_URL::null::&Processing_Status=success' % (self.path)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        slides = json.loads(resp.read())
        slideids = []
        for slide in slides:
            slideids.append((slide['ID'], slide['Filename'], slide['File_URL'], slide['Name']))
                
        self.logger.debug('Extracting metadata for %d slides(s).' % (len(slideids))) 
        for slideId,filename,file_url,name in slideids:
            f = self.getHistologicalFile(filename, file_url)
            self.logger.debug('Extracting metadata for filename "%s"' % (filename)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
            
            os.remove(f)
            
            if returncode == 1:
                continue
            
            """
            INSERT/UPDATE the image entry with the metadata
            """
            url = '%s/entity/Histological_Images:HE_Image' % self.path
            body = []
            obj = {}
            obj['ID'] = slideId
            obj['Name'] = name
            obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the image entry for the file "%s".' % (filename)) 
        self.logger.debug('Ended Metadata Extracting.') 
        
    def getHatracLocation(self, object_url):
        """
        Retrieve the object hatrac location.
        """
        ret = None
        url = '%s' % (object_url)
        if url != None:
            headers = {'Accept': '*'}
            try:
                resp = self.send_request('HEAD', url, headers=headers)
                resp.read()
                ret = resp.getheader('content-location', None)
            except:
                pass
        return ret
