[NbConvertApp] Converting notebook /home/aavsonet/scripts/reports/pipeline_report.ipynb to HTML
[NbConvertApp] Executing notebook with kernel: python3
[NbConvertApp] ERROR | Error while converting '/home/aavsonet/scripts/reports/pipeline_report.ipynb'
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/nbconvert/nbconvertapp.py", line 410, in export_single_notebook
    output, resources = self.exporter.from_filename(notebook_filename, resources=resources)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/exporter.py", line 179, in from_filename
    return self.from_file(f, resources=resources, **kw)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/exporter.py", line 197, in from_file
    return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/html.py", line 95, in from_notebook_node
    return super(HTMLExporter, self).from_notebook_node(nb, resources, **kw)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/templateexporter.py", line 307, in from_notebook_node
    nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/exporter.py", line 139, in from_notebook_node
    nb_copy, resources = self._preprocess(nb_copy, resources)
  File "/usr/lib/python3/dist-packages/nbconvert/exporters/exporter.py", line 316, in _preprocess
    nbc, resc = preprocessor(nbc, resc)
  File "/usr/lib/python3/dist-packages/nbconvert/preprocessors/base.py", line 47, in __call__
    return self.preprocess(nb, resources)
  File "/usr/lib/python3/dist-packages/nbconvert/preprocessors/execute.py", line 405, in preprocess
    nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
  File "/usr/lib/python3/dist-packages/nbconvert/preprocessors/base.py", line 69, in preprocess
    nb.cells[index], resources = self.preprocess_cell(cell, resources, index)
  File "/usr/lib/python3/dist-packages/nbconvert/preprocessors/execute.py", line 448, in preprocess_cell
    raise CellExecutionError.from_cell_and_msg(cell, out)
nbconvert.preprocessors.execute.CellExecutionError: An error occurred while executing the following cell:
------------------
#!/usr/bin/python3
'''

Created on Feb 5, 2020

@author: CLKotnik

This notebook prototypes textual and graphical reporting on the images
and photometry produced during a pipeline run.

It is assumed the image_info database has been updated with the extended metrics
on each image for the date/telescope.

KCLA Changes for pypline phase 3
Obtain config file location, observatory(=telescope) and datenite from environment

'''
import os
import glob
import requests
import sys
from simplejson.errors import JSONDecodeError 
import json
import fnmatch
import astropy.io.fits as pyfits
import string
import datetime
import MySQLdb
import time
import configargparse
import numpy as np
import math
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
#%matplotlib notebook
import logging
from tabulate import tabulate
from IPython.display import display, HTML
from IPython.display import Image
import fileinput
import datetime
import pytz
import re
from shutil import move, copymode

from astroplan import Observer
from astropy.coordinates import EarthLocation
import astropy.units as u
from datetime import datetime
from astropy.time import Time
from astropy.coordinates import get_sun, get_moon, AltAz    
    
def setup_logging(verbose,filename):
    """
    Setup the logging subsystem configuration
    """
    if (verbose):
        lev = logging.DEBUG
    else:
        lev = logging.INFO
    mode = 'a'
    form = '%(asctime)s - %(levelname)s - %(module)s - %(funcName)s - %(lineno)d - %(message)s'
    # Following to log to stdout
    #logging.basicConfig(level=lev,stream=sys.stdout, filemode = mode,format=form)
    logging.basicConfig(level=lev,filename = filename, filemode = mode,format=form)

#def create_link(filename):
#    # create a link to JPG image
#    return 'http://occam.aavso.org/' + \
#            telescope + '/' + \
#            telescope + '_' + yymmdd + '_images/' + \
#            filename[:-4] + 'jpg'
def create_imglink(imgurl):
    # create a link to JPG image
    link = '<img src="{}" />'.format(imgurl)
    #'<img src="http://occam.aavso.org/bsm_nh2/bsm_nh2_200122_images/a0101.jpg" style=max-height:124px;/>'
    return link
def create_tnlink(imgurl):
    # create a link to JPG image
    tn='<img src="{}"  style=max-height:400px;/>'.format(imgurl)
    tnlink = '<a href="{}"  target="_blank">{}</a>'.format(imgurl,tn)   
    return tnlink
def create_dual_imglink(imgurl,cenurl):
    figlink = """<figure> <img src="{}"  align="left" style="width:49%"> <img src="{}"  align="right" style="width:40%"> </figure>""".format(imgurl,cenurl)
    return figlink

def create_imgurl(filename,center):
    # create a URL to JPG image
    path = 'http://occam.aavso.org/' + \
            telescope + '/' + \
            telescope + '_' + yymmdd + '_images/' + \
            filename[:-5] 
    if center:   
        path = path + 'c.jpg'
    else:
        path = path + '.jpg'
        
    return path
    
    

def create_review(comments,reviewer):
    # create a URL to JPG image
    if (comments == ""):
        if (reviewer == ""):
            return "not reviewed"
        else:
            return "reviewed"
    else:
        return comments
    
    
def get_directory(telescope,yymmdd,img_base_dir,doYyyy=False):
    """
    Construct the directory name that corresponds to the telescope and date. 
    Check that the directory exists.  
       
    If doYyyy is true, look within a yyyy directory for the yymmdd directory
    
    Input:
    telescope: name of telescope
    yymmdd: date
    img_base_dir : base path name of images
    doYyyy : boolean, true to look for directory <telescope>/yyyy/yymmdd
    
    Return the directory name 
    or None if it does not exist
    """
    yyyy = '20' + yymmdd[:2]
    if telescope.lower() == 'sro50':
        telescope = 'sro'
    else:
        telescope = telescope.lower()
    
    if (len(telescope) > 0) and (len(yymmdd) > 0):
        if not doYyyy:
            imgdir = img_base_dir + '/' + telescope + '/' + yymmdd 
            if os.path.exists(imgdir):
                return imgdir
            else:
                # Sometimes the yymmdd directory is within a yyyy directory
                return get_directory(telescope,yymmdd,img_base_dir,True)
        else:
            imgdir = img_base_dir + '/' + telescope + '/' + yyyy + '/' + yymmdd 
            if os.path.exists(imgdir):
                return imgdir
            else:
                return None
    else:
        return None
    
def isActive(x,bins,temps,filters):
    if x.imagetyp == 'flat':
        if int(x.binning) in bins and x['filter'] in filters:
            return 'Y'
        else:
            return ' '
    else:
        if int(x.binning) in bins and int(x.set_temp) in temps:
            return 'Y'
        else:
            return ' '


def get_telescope_parms(parms):
    """
    Issue the web service call to the telescopes service and return the list
    of telescops returned or None if nothing is returned or there is an 
    exception doing the call.
    """
    
    url=r'https://aavso.org/apps/aavsonet/telescopes2/'

    try:
        r1= requests.get(url= url, params= parms)
    except requests.exceptions.RequestException as e:
        logging.exception("Exception issuing the telescopes web service call")
        logging.critical("URL {}; params {}".format(url,parms))
        return None

    if r1 is None:
        logging.error("Nothing returned - URL {}; params {}".format(url,parms))
        return None

    try:
        d1= r1.json() 
        logging.info("URL {}; params {} returned {} telescopes".format(url,parms,len(d1)))
    except JSONDecodeError as e:
        logging.exception("JSONDecodeError URL {}; params {}; resp {}".format(self.url,parms,r1))
        
        return None
    return d1

def get_telescope_datenite(yyyymmdd,observatory,args):
    """
    Return the telescop info for the most recent date 
    that has changed date on or before the specified datenite.
    Return None if not found.  Name is case
    insensitive.  The telescop information is a dictionary.
    """
    parms={'name':observatory}
    telescopes = get_telescope_parms(parms)
    # list of telescopes is order most recent date first

    errmsg = "Failed to find telescopes for {} datenite {}".format(observatory,yyyymmdd)
    
    if telescopes is None:
        logging.error(errmsg)
        haltwarn(args,True,errmsg)
        return None
    yyyy_mm_dd = yyyymmdd[:4] + '-' + yyyymmdd[4:6] + '-' + yyyymmdd[6:8]
    for telescop in telescopes:
        if telescop['changed'] < yyyy_mm_dd:
            logging.debug("Obs {}, datenite {} found {}".format(
                observatory,yyyy_mm_dd,telescop))
            return telescop

    logging.error(errmsg)
    haltwarn(args,True,errmsg)
    return  None   
    

def new_exp(lin_limit,exp,maxadu):
    if maxadu is None or math.isnan(maxadu) or maxadu == 'None' or maxadu == '' or maxadu <= lin_limit:
        return math.nan
    target_adu = 0.7 * lin_limit
    new_exp = (target_adu / maxadu) * exp
    return new_exp
    
def format_jd(x):
    try:
        JD = "{0:.5f}".format(float(x[3:]))
    except ValueError:
        JD = ''
    return JD
    
def get_pgm_parms():
    """
    Get the programs config file and command line arguments.
    
    Return a configargparse arguments object
    """
    parser = configargparse.ArgumentParser(description='AAVSOnet pipeline daily report')
    parser.add_argument('-c', '--my-config', required=True, is_config_file=True, help='config file path', env_var="MY_CONFIG")
    parser.add_argument('-o', '--observatory', required=True, help='observatory name', env_var ='OBSERVATORY')
    parser.add_argument('-d', '--yyyymmdd',    required=True, help='datenite process date yyyymmdd', env_var = 'YYYYMMDD')
    parser.add_argument('-D', '--procdir',     required=True, help='base directory for processed images')
    parser.add_argument('-n','--df_export',help='Filename to (im)export dataframe to excluding path')
    parser.add_argument('-l','--logfile',      required=True, help='Filename to log messages to')
    parser.add_argument('-v', '--verbose',                    help='Turn on verbose output', action='store_true',default=False, env_var='VERBOSE')
    parser.add_argument('-m', '--homedir',     required=True, help='home directory')
    parser.add_argument('-t', '--thumbdir',    required=True, help='base directory for thumbnails')
    parser.add_argument('-E', '--haltfile',    required=True, help='filename to signal HALT completion')
    parser.add_argument('-W', '--warnfile',    required=True, help='filename to signal WARNING completion')


    

    args,unknown = parser.parse_known_args()

    logname = os.path.join(args.homedir,args.observatory)
    logname = os.path.join(logname,args.logfile)
    setup_logging(args.verbose,filename=logname)
    
    for arg in vars(args):
        attr = getattr(args, arg)
        if isinstance(attr, str) and arg.startswith("db_"):
            t = attr.split(";")
            if len(t) == 4:
                attr = "{};{};;{}".format(t[0],t[1],t[3])
        logging.debug ("        {}  {}".format(arg, attr))

                
    return args

def get_scheduler_rpt(args):
    """
    Form this night's pathname for scheduler report and return it if it exists.  If not
    return None
    """
    pathname = os.path.join(args.thumbdir,args.observatory)
    filename = "{}_{}_schedrpt.txt".format(args.observatory,args.yyyymmdd[2:])
    pathaname = os.path.join(pathname,filename)
    if os.path.exists(pathname):
        return os.path.join(args.observatory,filename)
    else:
        logging.warning("No scheduler report found - {}".format(pathname))
        return None
    
def haltwarn(args,isHalt,desc):
    # Write a HALT or WARN message
    if isHalt:
        filepath = os.path.join(args.homedir,args.observatory)
        filepath = os.path.join(filepath,args.haltfile)
    else:
        filepath = os.path.join(args.homedir,args.observatory)
        filepath = os.path.join(filepath,args.warnfile)
        
    with open(filepath, 'a') as hfile:
        hfile.write(desc+'\n')
    if isHalt:
        logging.error("HALTing {}".format(desc))
    else:
        logging.warning("WARNing {}".format(desc))
        
    return










#######################################################################################
#### MAIN PROCESSING STARTS HERE ######################################################
#######################################################################################















# Get program parameters and setup logging
args = get_pgm_parms()

telescope = args.observatory
yymmdd = args.yyyymmdd[2:]

# Get telescope parameters from JSon web service
telescop = get_telescope_datenite('20' + yymmdd,telescope,args)

# Get pathname for scheduler report if it exists
schedrpt = get_scheduler_rpt(args)
if schedrpt is not None:
    display(HTML("<a href='http://occam.aavso.org/{}'>Scheduler Report</a> ".format(schedrpt)))


# Get UTC offset and stick in telescop
d= datetime.strptime(yymmdd, '%y%m%d')
tz = pytz.timezone(telescop['timezone']) #  eg. 'US/Aleutian'
telescop['utc_offset']= tz.localize(d).utcoffset().total_seconds()/3600
# sunrise and sunset times
location = EarthLocation.from_geodetic(telescop['longitude']*u.deg, telescop['latitude']*u.deg, telescop['elevation']*u.m)
observatory = Observer(location=location, name= telescop['name_long'], timezone= telescop['timezone'])
#now = Time(datetime.strptime(yymmdd, '%y%m%d'), location= location)
#telescop['sunset']= observatory.sun_set_time(now, which="next")
#telescop['sunrise']= observatory.sun_rise_time(now+ 1, which="next")
tr= observatory.datetime_to_astropy_time(datetime.strptime(yymmdd+ ' 13', '%y%m%d %H')) # 1300 of datenite (local) as utc
telescop['sunset']= observatory.astropy_time_to_datetime(observatory.sun_set_time(tr, which="next"))
telescop['sunrise']= observatory.astropy_time_to_datetime(observatory.sun_rise_time(tr, which="next"))

#Use the following when running from the command line 
#args,unknown = parser.parse_known_args()





logging.info("Begin execution, telescope {}, date {}".format(args.observatory,args.yyyymmdd))

hdf_file = get_directory(telescope,yymmdd,args.procdir)
if hdf_file is None:
    logging.error("Cannot locate image directory {} {} {}".format(args.procdir,telescope,yymmdd))
    got_images = False
else:    
    hdf_file += "/" + args.df_export
    if not os.path.exists(hdf_file):
        logging.error("Cannot locate HDF file {} {} {} {}".format(args.procdir,telescope,yymmdd,args.df_export))
        got_images = False
    else:
        got_images = True

if got_images:
    df = pd.read_hdf(hdf_file,key='science')
    logging.info("Loaded {} science images from {}".format(len(df),hdf_file))

    cdf = pd.read_hdf(hdf_file,key='calibration')
    logging.info("Loaded {} calibration images from {}".format(len(cdf),hdf_file))
    
    # Adjust for new database based calibration files
    if 'rel_filepath' in cdf.columns:
        cdf.rename(columns = {'rel_filepath':'filename'}, inplace = True)

        bins = [int(b) for b in df.xbinning.unique()]
        temps = [int(t) for t in df.set_temp.unique()]
        filters = df['filter'].unique()
        cdf['Active'] = cdf.apply(lambda x: isActive(x,bins,temps,filters),axis=1)
        cdf=cdf.sort_values(["Active", "imagetyp","Age(d)"], ascending = (False, True, True))


    # Make booleans 1=true, 0-false
    df['has_wcs'] = df.apply(lambda x: 1 if x['has_wcs'] == 'True' else 0,axis=1)
    df['image_available'] = df['image_available'].fillna(0)

    # Review columns blank is null
    
    if 'comments' not in df.columns:
        df['comments'] = ""
        df['reviewer'] = ""
    else:
        df['comments'] = df['comments'].fillna("")
        df['reviewer'] = df['reviewer'].fillna("")
        
    df['xbinning'] = df['xbinning'].fillna (0)
    df['ybinning'] = df['ybinning'].fillna (0)
    df['img_type'] = df['img_type'].fillna ("")

    # Convert strings to floats
    for col in ['set_temp','ccd_temp','obj_ra_deg', 'obj_dec_deg', 'center_ra_deg', 'center_dec_deg',
        'fov_radius_amin', 'center2obj', 'hr_ang','airmass', 'amb_temp', 'bjd','gain', 'focallen', 'fwhm',
        'exptime', 'ccd_tpwr',
        ]:
        df[col]=df.apply(lambda x: float(x[col]) if ((x[col] != 'None') and (x[col] != '') and (x[col] is not None)) else None,axis=1)
        
    # Calculate how much warmer CCD was than the set point
    df['deltaT']= abs(df['ccd_temp']-df['set_temp'])

    # CLKotnik 10/30/23 Change required by upgrade to Pandas 2
    #df['DateTime'] = pd.to_datetime(df['date_obs'],format='%Y-%m-%dT%H:%M:%S') 
    df['DateTime'] = pd.to_datetime(df['date_obs'],format='ISO8601')

    # URL column for image
    df['imgurl'] = df.apply(lambda x: create_imgurl(x['filename'],False),axis=1)
    
    # URL column for center cutout image (if available)
    img_first = df['filename'].iloc[0]
    cenpath = '/raid/aavsonet_TNimages/' + telescope + '/' + \
            telescope + '_' + yymmdd + '_images/' + \
            img_first[:-5] + 'c.jpg'
    got_center = os.path.exists(cenpath)
    df['tnurl'] = df.apply(lambda x: create_imgurl(x['filename'],got_center),axis=1)

    # Create review column
    # URL column for image
    df['review'] = df.apply(lambda x: create_review(x['comments'],x['reviewer']),axis=1)


    # Inline image link
    df['tnimage'] = df.apply(lambda x: create_tnlink(x['tnurl']),axis=1)

    #Inline image
    df['image'] = df.apply(lambda x: create_imglink(x['imgurl']),axis=1)

    # Side by side image
    df['sxsimage'] = df.apply(lambda x: create_dual_imglink(x['imgurl'],x['tnurl']),axis=1)

    # Combined X and Y binning
    df['binning'] = df.apply(lambda x: str(int(x.xbinning)) + 'x' + str(int(x.ybinning)),axis=1)

    # Format JD as string
    df['jd'] = df['jd'].fillna(0)
    df['JD'] = df.apply(lambda x: format_jd(x.jd) ,axis=1)
    
    # Create a display version of the project column with spaces so it will line wrap
    df['project_'] = df.apply(lambda x: x['project'].replace("_","_ "),axis=1)

    # Rename some columns
    # Fixup so even if no astrometry at all we still run
    for c in ["obj_mag","obj_mag_err","obj_maxadu"]:
        if c not in df.columns:
            df[c] = None
    df = df.rename(columns={"obj_mag": "instmag", "obj_mag_err": "err","obj_maxadu":"maxadu"})

    #********************  New exposure detect and computation ****************************    
    # Is saturated, calculate recommended new exposure
    df['OE'] = df.apply(lambda x: "!OE!" if x['maxadu'] is not None and x['maxadu'] != 'None' and x['maxadu'] != '' and x['maxadu'] > telescop['linearitylimit'] else '',axis=1)
    df['new_exp'] = df.apply(lambda x: new_exp(telescop['linearitylimit'],x['exptime'],x['maxadu']),axis=1)
    df['UE'] = df.apply(lambda x: "!UE!" if x['err'] is not None and x['err'] != 'None' and x['err'] != '' and x['err'] > 0.05 else '',axis=1) # snr < 20
    
------------------

---------------------------------------------------------------------------
JSONDecodeError                           Traceback (most recent call last)
<ipython-input-3-b5f4f874c2ce> in get_telescope_parms(parms)
    187     try:
--> 188         d1= r1.json()
    189         logging.info("URL {}; params {} returned {} telescopes".format(url,parms,len(d1)))

/usr/lib/python3/dist-packages/requests/models.py in json(self, **kwargs)
    896                     pass
--> 897         return complexjson.loads(self.text, **kwargs)
    898 

/usr/lib/python3/dist-packages/simplejson/__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, use_decimal, **kw)
    517             and not use_decimal and not kw):
--> 518         return _default_decoder.decode(s)
    519     if cls is None:

/usr/lib/python3/dist-packages/simplejson/decoder.py in decode(self, s, _w, _PY3)
    369             s = str(s, self.encoding)
--> 370         obj, end = self.raw_decode(s)
    371         end = _w(s, end).end()

/usr/lib/python3/dist-packages/simplejson/decoder.py in raw_decode(self, s, idx, _w, _PY3)
    399                 idx += 3
--> 400         return self.scan_once(s, idx=_w(s, idx).end())

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

NameError                                 Traceback (most recent call last)
<ipython-input-3-b5f4f874c2ce> in <module>
    342 
    343 # Get telescope parameters from JSon web service
--> 344 telescop = get_telescope_datenite('20' + yymmdd,telescope,args)
    345 
    346 # Get pathname for scheduler report if it exists

<ipython-input-3-b5f4f874c2ce> in get_telescope_datenite(yyyymmdd, observatory, args)
    202     """
    203     parms={'name':observatory}
--> 204     telescopes = get_telescope_parms(parms)
    205     # list of telescopes is order most recent date first
    206 

<ipython-input-3-b5f4f874c2ce> in get_telescope_parms(parms)
    189         logging.info("URL {}; params {} returned {} telescopes".format(url,parms,len(d1)))
    190     except JSONDecodeError as e:
--> 191         logging.exception("JSONDecodeError URL {}; params {}; resp {}".format(self.url,parms,r1))
    192 
    193         return None

NameError: name 'self' is not defined
NameError: name 'self' is not defined