from datetime import datetime, date, timedelta
strptime = datetime.strptime
strftime = datetime.strftime
from astroquery.eso import EsoClass
from . import config
from .date_manips import mjd_to_datetime, datetime_to_mjd
from .log import Log, ERROR, WARNING, NOTICE
import os
from bs4 import BeautifulSoup
from astropy.table import Table, Column
from astroquery.utils import schema, system_tools
from astroquery.exceptions import LoginError, RemoteServiceError, NoResultsWarning
import sys
from subprocess import Popen


debug = config.debug
class RunError(ValueError):
    pass

class GraviQuery(EsoClass, Log):
    filters = config.filters
    date_format = "%Y-%m-%dT%H:%M:%S"
    date_format2 = "%Y-%m-%dT%H:%M:%S.%f"
    night_format = "%Y-%m-%d"
    time_formats = ["%Y-%m-%dT%H:%M:%S.%f",
                    "%Y-%m-%dT%H:%M:%S",
                    "%Y-%m-%dT%H:%M",
                    "%Y-%m-%dT%H",
                    "T%H", "T%H:%M", "T%H:%M:%S",  "T%H:%M:%S.%f"]

    last_query = None
    def __init__(self, night=None, stime=None, etime=None, filters={}, output_dir=None,
                 ROW_LIMIT=None, USERNAME=None, instrument=None, debug=debug):
        """this a wrapper of astroquery.eso.EsoClass object for Gravity


        Parameters
        ----------
        night : string the nitgh  date e.g 2015-10-13
        stime / etime : string start/end UT time.
            If night is given they should be of the form Thh or Thh:mm or Thh:mm:ss
             e.i. they must gives the start hour and end hour.
             Becarefull, nights start from 12 to 12 UT time.
              if hour<12 it is interpreted as the night after

                example:
                    night = "2015-10-13", stime="T22", etime="T04"
                    is interpreted as:  - start ut time  2015-10-13T22:00:00
                                        - end   ut time  2015-10-14T04:00:00

                    night = "2015-10-13", stime="T01", etime="T04"
                    is interpreted as:  - start ut time  2015-10-14T01:00:00
                                        - end   ut time  2015-10-14T04:00:00

            If night is no given etime, stime can be a full timestamps.

        filters : additional column_filters (default are in config.py)
        output_dir : the ouput root director. With the download method, the files
            will be partitioned into subdirectories of night (like in ESO wrokstations)

        ROW_LIMIT : a new RAO_LIMIT (default in config.py)
        USERNAME :  a user name for eso portail (default in config.py)


        Examples:
        ---------
        >>> q = GraviQuery( '2015-10-15')
        >>> q.query()
        >>> q.download()

        """
        # make a copy of the class filters and update it from filters

        EsoClass.__init__(self)
        self.filters = dict(self.filters, **filters)
        self.output_dir = output_dir or config.output_dir
        self.request_dir = self.output_dir+"/.requests"
        self.gunzip_list = []

        if night is not None:
            try:
                night = strptime(night, self.night_format)
            except ValueError:
                raise RunError( "Error when reading the night date should be of the form, e.g. '2015-10-23'")


        if stime is not None:
            stime = self.read_time(stime)
            if stime.year > 1900: ## a date was given
                if night is not None:

                    check_night = (stime -timedelta(1)).date() if stime.hour<12 else stime.date()
                    if night.date()!= check_night:
                         raise RunError("start time %s (of night %s) is in conflict with the date parsed in night : %s"%(stime, check_night, night))

            elif night is None:
                raise RunError("stime should have a date if no night is given")
            else:
                stime = datetime(night.year, night.month, night.day, stime.hour, stime.minute, stime.second)
                if stime.hour<12:
                    # this is the day after
                    stime = stime + timedelta(1)

        else:
            if night is None:
                raise RunError("No night neither start time  given")
            else:
                stime = datetime(night.year, night.month, night.day, 12)

        if etime is not None:
            etime = self.read_time(etime)
            if etime.year > 1900: ## a date was given
                if night is not None:
                    check_night = (etime -timedelta(1)).date() if etime.hour<12 else etime.date()
                    if night.date()!= check_night:
                         raise RunError("end time %s (of night %s) is in conflict with the date parsed in night : %s"%(etime, check_night, night))

                    if (etime.hour<12 and  (night.date()!=  (etime -timedelta(1)).date())) or \
                       (etime.hour>=12 and (night.date()!= etime.date())):
                         raise RunError("etime  has a date in conflict with the date parsed in 'night'")

            elif night is None:
                raise ValueError("etime should have a date if no night is given")
            else:
                etime = datetime(night.year, night.month, night.day, etime.hour, etime.minute, etime.second)
                if etime.hour<12:
                    # this is the day after
                    etime = etime + timedelta(1)
        else:
            if night is None:
                raise RunError("No night neither end time given")
            else:
                etime = datetime(night.year, night.month, night.day, 12)+timedelta(1)# from 12 to 12 so add one day


        if stime > etime:
            raise RunError("stime is after etime")

        self.etime = etime
        self.stime = stime

        self.ROW_LIMIT = ROW_LIMIT or config.ROW_LIMIT
        self.USERNAME = USERNAME or config.USERNAME
        self.instrument = instrument or config.filters.get("instrument", "GRAVITY")
        self.filters.update(etime=etime.isoformat(), stime=stime.isoformat(),
                            instrument=self.instrument
                            )
        self.debug = debug


    def query2(self, requests=None):

        if isinstance(requests, (str,int)):
            requests = [requests]
        if requests is None:
            self.log("query the list of requests for Gravity",1, NOTICE)
            requests = self._query_request()
            self.log("%d requests found"%len(requests), 1, NOTICE)
        ids = []
        for request in requests:
            self.log("query file from request %s"%request, 1, NOTICE)
            qf = self._query_files(request)
            ids.extend(list(zip(qf, [request]*len(qf))))


        self.log("extracted %d files from all request"%len(ids),1,NOTICE)
        dates = self._extract_date(ids)


        self._select_dates(dates)
        self.log("%d files are within query dates"%len(dates),1,NOTICE)

        mjds = [datetime_to_mjd(d[0]) for d in  list(dates.values())]
        qids = [d[1] for d in  list(dates.values())]
        self.last_query = Table( [list(dates.keys()), mjds, qids], names=["Dataset ID", "MJD-OBS", "REQUEST-ID"])
        return self.last_query

    def query1(self):
        self.log("sending query to eso archive from %s to %s"%(self.filters["stime"],self.filters["etime"]), 1, NOTICE)
        self.last_query = self.query_instrument('eso_archive_main', column_filters=self.filters)
        if self.last_query is not None:
            self.log("query retrieved %d files ready for download"%(len(self.last_query)), 1, NOTICE)
        return self.last_query

    def query(self, username=None, request=None):
        if self.instrument.lower() == "gravity":
            if not self.authenticated():
                self.login(username or self.USERNAME)
            return self.query2(request)
        else:
            return self.query1()

    def download2(self, table=None, only=None, username=None, output_dir=None):
        by_requests = {}

        if table is None:
            if self.last_query is None:
                raise ValueError("You should run query() first or give a query table")
            else:
                table = self.last_query
        if isinstance(only, int):
            table = [table[only]]
        if isinstance(only, str):
            raise ValueError("only must be a int, slice, array like not a string")
        elif only is not None:
            table = table[only]

        for r in table:
            rid = r["REQUEST-ID"]
            if not rid in by_requests:
                by_requests[rid] = {}
            d = mjd_to_datetime(r["MJD-OBS"])
            if d.hour<12:
                # the True night subdir is the day before
                d = d - timedelta(1)
            night =  strftime(d, "%Y-%m-%d")
            if not night in by_requests[rid]:
                by_requests[rid][night] = []

            by_requests[rid][night].append(r["Dataset ID"])

        path = []
        output_dir = output_dir or self.output_dir
        for request, nights in by_requests.items():
            for night,files in nights.items():
                self.cache_location = output_dir+"/"+night
                if not os.path.exists(self.cache_location):
                    self.log("making directory '%s'"%self.cache_location, 1, NOTICE)
                    os.mkdir(self.cache_location)
                self.log("downloading files of night '%s'"%night, 1, NOTICE)
                if self.debug:
                    path.extend(self.cache_location+"/"+id for id in files)
                else:
                    path.extend(self.download_file_from_request(files, request))

        # Wait for all gunzip processes to complete
        while self.gunzip_list:
            self.gunzip_list.pop(0).wait()

        return path

    def background_gunzip(self, filename):
        self.gunzip_list.append(Popen(["gzip", "-d", "{0}".format(filename)]))
        return filename.strip(".Z")

    def download_file_from_request(self, req_files, request):

        if not len(req_files):
            return []
        self.log("processing download of %d files in request %s "%(len(req_files), request), 1, NOTICE)

        data_download_form = self._request("GET",
                                        "http://dataportal.eso.org/rh/requests/GRAVarchive/%s"%request,
                                        cache=False)
        root = BeautifulSoup(data_download_form.content, 'html5lib')
        state = root.select('span[id=requestState]')[0].text
        #print("{0:20.0f}s elapsed".format(time.time()-t0), end='\r')
        sys.stdout.flush()

        if state == 'ERROR':
            raise RemoteServiceError("There was a remote service error;"
                                     " perhaps the requested file could not be found?")

        self.log("Downloading files...", 1, NOTICE)
        files = []
        for fileId in root.select('input[name=fileId]'):
                selected = False
                fid ="_dummy_"
                for f in req_files:
                    if f in fileId.attrs['value']:
                        selected = True
                        fid = f
                        break
                else:
                    continue

                fpath = self.cache_location+"/"+fid+".fits"
                if os.path.exists(fpath):
                    self.log( "file '%s' already there skiping"%fpath, 1,NOTICE)
                    continue
                fileLink = "http://dataportal.eso.org/dataPortal"+fileId.attrs['value'].split()[1]
                filename = self._request("GET", fileLink, save=True)
                files.append(self.background_gunzip(filename))
        self._session.redirect_cache.clear() # Empty the redirect cache of this request session
        return files

    def download(self, table=None, only=None, username=None, output_dir=None):

        if not self.debug and not self.authenticated():
            self.login(username or self.USERNAME)
        if self.instrument.lower() == "gravity":
            return self.download2(table=table, only=only, username=username)
        else:
            return self.download1(table=table, only=only, username=username)

    def download1(self, table=None, only=None, username=None, output_dir=None):
        """ download file retrieve from the last query

        The file are partitioned in observation date (e.g. subdirectories like 2015-10-14)


        Parameters:
        -----------
            table (Optional) : astropy.table
                table returned by q.query(), if None use the last q.query() result
            only (Optional) : table row index
                limit the download to the given row indexes -> table = table[only]
            username (Optioanl) : string
                a new username for this download. default in the query instance or config.py
            output_dir (Optional) : string
                output root directory. Default set in the query instance or in config.py

        Returns:
        --------
            path : list of string
                a flat list of all file path

        """
        if table is None:
            if self.last_query is None:
                raise ValueError("You should run query() first or give a query table")
            else:
                table = self.last_query

        if not self.debug and not self.authenticated():
            self.login(username or self.USERNAME)


        # separate the query by night
        nights = {}

        if isinstance(only, int):
            table = [table[only]]
        if isinstance(only, str):
            raise ValueError("only must be a int, slice, array like not a string")
        elif only is not None:
            table = table[only]

        for r in table:
            d = mjd_to_datetime(r["MJD-OBS"])
            if d.hour<12:
                # the True night subdir is the day before
                d = d - timedelta(1)

            night =  strftime(d, "%Y-%m-%d")

            if not night in nights:
                nights[night] = []
            nights[night].append(r['Dataset ID'])

        path = []
        output_dir = output_dir or self.output_dir

        for night,ids in nights.items():
            self.cache_location = output_dir+"/"+night
            if not os.path.exists(self.cache_location):
                self.log("making directory '%s'"%self.cache_location, 1, NOTICE)
                os.mkdir(self.cache_location)
            self.log("downloading files of night '%s'"%night, 1, NOTICE)
            if self.debug:
                path.extend( self.cache_location+"/"+id for id in ids )
            else:
                path.extend( self.retrieve_data(ids) )

        return path
        #return eso.retrieve_data(table['Dataset ID'])

    def read_time(self,ts):

        try: # try if it is a numerical form
            tn = float(ts)
        except ValueError:
            pass
        else:
            d = datetime.fromtimestamp(3600*tn)
            return datetime(1900, 1, 1, d.hour-1, d.minute, d.second)

        for f in self.time_formats:
            try:
                t = strptime(ts, f)
            except ValueError:
                continue
            else:
                return t
        raise RunError("'%s' is not a valid time format"%ts)


    def _extract_date(self, ids):
        dates = {}
        for id,qid in ids:
            if not id.strip(): continue
            sdate = id.split(".",1)[1]
            dates[id] = (strptime(sdate, self.date_format2), qid)
        return dates

    def _select_dates(self, dates):
        for id, (date,qid) in list(dates.items()):
            if (date<self.stime) or (date>self.etime):
                dates.pop(id)


    def _query_request(self, cache=False):
        list_retrieval = self._request("GET",
                        "http://dataportal.eso.org/rh/requests/GRAVarchive",
                        cache=cache)
        root = BeautifulSoup(list_retrieval.content)
        request_ids = []

        for tr in root.select("tr"):
            if not "Special Access to GRAVITY" in tr.text:
                continue

            for td in tr.select("td[class=right]"):
                a = td.select("a")
                if len(a):
                    request_ids.append(a[0].text.strip())
        return request_ids

    def _query_files(self, request, cache=True):
        if cache:
            cached = self.request_dir+"/"+request+".lst"
            if os.path.exists(cached):
                self.log("request cached in %s"%cached, 1, NOTICE)
                return [f.strip() for f in open(cached).read().split("\n")]

        if not self.authenticated():
            self.login()
        self.log("retrieve new request in http://dataportal.eso.org/rh/requests/GRAVarchive/%s"%request, 1, NOTICE)
        list_retrieval = self._request("GET",
                        "http://dataportal.eso.org/rh/requests/GRAVarchive/%s"%request,
                        cache=cache)
        root = BeautifulSoup(list_retrieval.content)
        file_ids = []

        for e in root.select("td[class=left]"):
            if "SAF+GRAVI" in e.text:
                ## patch for some bad requested
                if "SAF+SAF" in e.text: continue
                file_ids.append(e.text.strip()[4:])

        if cache:
            if not os.path.exists(self.request_dir):
                os.mkdir(self.request_dir)
            cached = self.request_dir+"/"+request+".lst"
            try:
                g = open(cached, "w")
            except:
                pass
            else:
                g.write("\n".join(file_ids)+"\n")
        return file_ids

    def _download_files(self, files):
        if not self.authenticated():
            self.login()