#%% ###################################################################################################################
"""
Python Glacier Evolution Model "PyGEM" V1.0
Prepared by David Rounce with support from Regine Hock.
This work was funded under the NASA-ROSES program (grant no. NNX17AB27G).

PyGEM is an open source glacier evolution model written in python.  The model expands upon previous models from
Radic et al. (2013), Bliss et al. (2014), and Huss and Hock (2015).
"""
#######################################################################################################################
# This is the main script that provides the architecture and framework for all of the model runs. All input data is
# included in a separate module called pygem_input.py. It is recommended to not make any changes to this file unless
# you are a PyGEM developer and making changes to the model architecture.
#
#%%========= IMPORT PACKAGES ==========================================================================================
# Various packages are used to provide the proper architecture and framework for the calculations used in this script.
# Some packages (e.g., datetime) are included in order to speed up calculations and simplify code.
import pandas as pd
import numpy as np
#import matplotlib.pyplot as plt
from datetime import datetime
import os # os is used with re to find name matches
#import re # see os
import xarray as xr
import netCDF4 as nc
from time import strftime
import timeit
from scipy.optimize import minimize
from scipy.stats import linregress
import matplotlib.pyplot as plt
import cartopy

#========== IMPORT INPUT AND FUNCTIONS FROM MODULES ===================================================================
import pygem_input as input
import pygemfxns_modelsetup as modelsetup
import pygemfxns_climate as climate
import pygemfxns_massbalance as massbalance
import pygemfxns_output as output

#%% ===== LOAD GLACIER DATA ===========================================================================================
# RGI glacier attributes
main_glac_rgi = modelsetup.selectglaciersrgitable()
# For calibration, filter data to those that have calibration data
if input.option_calibration == 1:
    # Select calibration data from geodetic mass balance from David Shean
    main_glac_calmassbal = modelsetup.selectcalibrationdata(main_glac_rgi)
    # Concatenate massbal data to the main glacier
    main_glac_rgi = pd.concat([main_glac_rgi, main_glac_calmassbal], axis=1)
    # Drop those with nan values
    main_glac_calmassbal = main_glac_calmassbal.dropna()
    main_glac_rgi = main_glac_rgi.dropna()
# Glacier hypsometry [km**2], total area
main_glac_hyps = modelsetup.import_Husstable(main_glac_rgi, input.rgi_regionsO1, input.hyps_filepath,
                                             input.hyps_filedict, input.hyps_colsdrop)
elev_bins = main_glac_hyps.columns.values.astype(int)
# Ice thickness [m], average
main_glac_icethickness = modelsetup.import_Husstable(main_glac_rgi, input.rgi_regionsO1, input.thickness_filepath,
                                                     input.thickness_filedict, input.thickness_colsdrop)
# Width [km], average
main_glac_width = modelsetup.import_Husstable(main_glac_rgi, input.rgi_regionsO1, input.width_filepath,
                                              input.width_filedict, input.width_colsdrop)
# Add volume [km**3] and mean elevation [m a.s.l.] to the main glaciers table
main_glac_rgi['Volume'], main_glac_rgi['Zmean'] = modelsetup.hypsometrystats(main_glac_hyps, main_glac_icethickness)
# Model time frame
dates_table, start_date, end_date = modelsetup.datesmodelrun(input.startyear, input.endyear, input.spinupyears)
# Quality control - if ice thickness = 0, glacier area = 0 (problem identified by glacier RGIV6-15.00016 on 03/06/2018)
main_glac_hyps[main_glac_icethickness == 0] = 0

#%%=== LOAD CLIMATE DATA ================================================================================
if input.option_gcm_downscale == 1:
    # Air Temperature [degC] and GCM dates
    main_glac_gcmtemp, main_glac_gcmdate = climate.importGCMvarnearestneighbor_xarray(
            input.gcm_temp_filename, input.gcm_temp_varname, main_glac_rgi, dates_table, start_date, end_date)
    # Precipitation [m] and GCM dates
    main_glac_gcmprec, main_glac_gcmdate = climate.importGCMvarnearestneighbor_xarray(
            input.gcm_prec_filename, input.gcm_prec_varname, main_glac_rgi, dates_table, start_date, end_date)
    # Elevation [m a.s.l] associated with air temperature  and precipitation data
    main_glac_gcmelev = climate.importGCMfxnearestneighbor_xarray(
            input.gcm_elev_filename, input.gcm_elev_varname, main_glac_rgi)
    # Add GCM time series to the dates_table
    dates_table['date_gcm'] = main_glac_gcmdate
    # Lapse rates [degC m-1]
    if input.option_lapserate_fromgcm == 1:
        main_glac_gcmlapserate, main_glac_gcmdate = climate.importGCMvarnearestneighbor_xarray(
                input.gcm_lapserate_filename, input.gcm_lapserate_varname, main_glac_rgi, dates_table, start_date,
                end_date)
elif input.option_gcm_downscale == 2:
    # Import air temperature, precipitation, and elevation from pre-processed csv files for a given region
    #  this simply saves time from re-running the fxns above
    main_glac_gcmtemp_all = np.genfromtxt(input.gcm_filepath_var + input.gcmtemp_filedict[input.rgi_regionsO1[0]],
                                          delimiter=',')
    main_glac_gcmprec_all = np.genfromtxt(input.gcm_filepath_var + input.gcmprec_filedict[input.rgi_regionsO1[0]],
                                          delimiter=',')
    main_glac_gcmelev_all = np.genfromtxt(input.gcm_filepath_var + input.gcmelev_filedict[input.rgi_regionsO1[0]],
                                          delimiter=',')
    # Lapse rates [degC m-1]
    main_glac_gcmlapserate_all = np.genfromtxt(input.gcm_filepath_var +
                                               input.gcmlapserate_filedict[input.rgi_regionsO1[0]], delimiter=',')
    # Select the climate data for the glaciers included in the study
    main_glac_gcmtemp = main_glac_gcmtemp_all[main_glac_rgi['O1Index'].values]
    main_glac_gcmprec = main_glac_gcmprec_all[main_glac_rgi['O1Index'].values]
    main_glac_gcmelev = main_glac_gcmelev_all[main_glac_rgi['O1Index'].values]
    main_glac_gcmlapserate = main_glac_gcmlapserate_all[main_glac_rgi['O1Index'].values]

#%%=== STEP FOUR: CALIBRATION =========================================================================================
#timestart_step4 = timeit.default_timer()
if input.option_calibration == 1:
    #----- ENTER CALIBRATION RUN --------------------------------------------------------------------------------------
    # [INSERT REGIONAL LOOP HERE] if want to do all regions at the same time.  Separate netcdf files will be generated
    #  for each loop to reduce file size and make files easier to read/share
    regionO1_number = input.rgi_regionsO1[0]
    # Create output netcdf file
#    if input.output_package != 0:
#        output.netcdfcreate(regionO1_number, main_glac_hyps, dates_table, annual_columns)

    # CREATE A SEPARATE OUTPUT FOR CALIBRATION with only data relevant to calibration
    #   - annual glacier-wide massbal, area, ice thickness, snowline
    # Model parameter output
    main_glac_modelparamsopt = np.zeros((main_glac_rgi.shape[0], 8))
    main_glac_massbal_compare = np.zeros((main_glac_rgi.shape[0],4))

    for glac in range(main_glac_rgi.shape[0]):
#    for glac in range(25):
#    for glac in [0]:
#        glac = 3472
        print(main_glac_rgi.loc[glac,'RGIId'])

        # Set model parameters
        modelparameters = [input.lrgcm, input.lrglac, input.precfactor, input.precgrad, input.ddfsnow, input.ddfice,
                           input.tempsnow, input.tempchange]
        # Select subset of variables to reduce the amount of data being passed to the function
        glacier_rgi_table = main_glac_rgi.loc[glac, :]
        glacier_gcm_elev = main_glac_gcmelev[main_glac_rgi.loc[glac,'O1Index']]
        glacier_gcm_prec = main_glac_gcmprec[main_glac_rgi.loc[glac,'O1Index'],:]
        glacier_gcm_temp = main_glac_gcmtemp[main_glac_rgi.loc[glac,'O1Index'],:]
        glacier_gcm_lrgcm = main_glac_gcmlapserate[main_glac_rgi.loc[glac,'O1Index']]
        glacier_gcm_lrglac = glacier_gcm_lrgcm.copy()
        glacier_area_t0 = main_glac_hyps.iloc[glac,:].values.astype(float)
        # Inclusion of ice thickness and width, i.e., loading values may be only required for Huss mass redistribution!
        icethickness_t0 = main_glac_icethickness.iloc[glac,:].values.astype(float)
        width_t0 = main_glac_width.iloc[glac,:].values.astype(float)

        # Record the calibration round
        calround = 0

        # Run calibration only for glaciers that have calibration data
        if np.isnan(main_glac_rgi.loc[glac,input.massbal_colname]) == False:
            # OPTIMIZATION FUNCTION: Define the function that you are trying to minimize
            #  - modelparameters are the parameters that will be optimized
            #  - return value is the value is the value used to run the optimization
            def objective(modelparameters):
                (glac_wide_massbaltotal, glac_wide_runoff, glac_wide_snowline, glac_wide_snowpack,
                 glac_wide_area_annual, glac_wide_volume_annual, glac_wide_ELA_annual) = (
                    massbalance.runmassbalance(modelparameters, glacier_rgi_table, glacier_area_t0, icethickness_t0,
                                               width_t0, elev_bins, glacier_gcm_temp, glacier_gcm_prec,
                                               glacier_gcm_elev, glacier_gcm_lrgcm, glacier_gcm_lrglac, dates_table))
                # Compare calibration data
                # Column index for start and end year based on dates of geodetic mass balance observations
                massbal_idx_start = int(glacier_rgi_table.loc[input.massbal_time1] - input.startyear)
                massbal_idx_end = int(massbal_idx_start + glacier_rgi_table.loc[input.massbal_time2] -
                                      glacier_rgi_table.loc[input.massbal_time1] + 1)
                # Annual glacier-wide mass balance [m w.e.]
                glac_wide_massbaltotal_annual = np.sum(glac_wide_massbaltotal.reshape(-1,12), axis=1)
                # Average annual glacier-wide mass balance [m w.e.a.]
                glac_wide_massbaltotal_annual_avg = (
                        glac_wide_massbaltotal_annual[massbal_idx_start:massbal_idx_end].mean())
                #  units: m w.e. based on initial area
                # Difference between geodetic and modeled mass balance
                massbal_difference = abs(glacier_rgi_table[input.massbal_colname] - glac_wide_massbaltotal_annual_avg)
                return massbal_difference
            # CONSTRAINTS
            #  everything goes on one side of the equation compared to zero
            #  ex. return x[0] - input.lr_gcm with an equality constraint means x[0] = input.lr_gcm (see below)
            def constraint_lrgcm(modelparameters):
                return modelparameters[0] - input.lrgcm
            def constraint_lrglac(modelparameters):
                return modelparameters[1] - input.lrglac
            def constraint_precfactor(modelparameters):
                return modelparameters[2] - input.precfactor
            def constraint_precgrad(modelparameters):
                return modelparameters[3] - input.precgrad
            def constraint_ddfsnow(modelparameters):
                return modelparameters[4] - input.ddfsnow
            def constraint_ddfice(modelparameters):
                return modelparameters[5] - input.ddfice
            def constraint_tempsnow(modelparameters):
                return modelparameters[6] - input.tempsnow
            def constraint_tempchange(modelparameters):
                return modelparameters[7] - input.tempchange
            def constraint_ddficefxsnow(modelparameters):
                return modelparameters[4] - input.ddfsnow_iceratio * modelparameters[5]
            def constraint_ddficegtsnow(modelparameters):
                return modelparameters[5] - modelparameters[4]
            def constraint_lrsequal(modelparameters):
                return modelparameters[0] - modelparameters[1]
            # Define constraint type for each function
            con_lrgcm = {'type':'eq', 'fun':constraint_lrgcm}
            con_lrglac = {'type':'eq', 'fun':constraint_lrglac}
            con_precfactor = {'type':'eq', 'fun':constraint_precfactor}
            con_precgrad = {'type':'eq', 'fun':constraint_precgrad}
            con_ddfsnow = {'type':'eq', 'fun':constraint_ddfsnow}
            con_ddfice = {'type':'eq', 'fun':constraint_ddfice}
            con_tempsnow = {'type':'eq', 'fun':constraint_tempsnow}
            con_tempchange = {'type':'eq', 'fun':constraint_tempchange}
            con_ddficefxsnow = {'type':'eq', 'fun':constraint_ddficefxsnow}
            con_ddficegtsnow = {'type':'ineq', 'fun':constraint_ddficegtsnow}
            con_lrsequal = {'type':'eq', 'fun':constraint_lrsequal}
            # INITIAL GUESS
            modelparameters_init = modelparameters
            # PARAMETER BOUNDS
            lrgcm_bnds = (-0.008,-0.004)
            lrglac_bnds = (-0.008,-0.004)
            precfactor_bnds = (0.9,1.2)
            precgrad_bnds = (0.0001,0.00025)
            ddfsnow_bnds = (0.0036, 0.0046)
            #  Braithwaite (2008)
            ddfice_bnds = (ddfsnow_bnds[0]/input.ddfsnow_iceratio, ddfsnow_bnds[1]/input.ddfsnow_iceratio)
            tempsnow_bnds = (0,2)
            tempchange_bnds = (-1,1)
            modelparameters_bnds = (lrgcm_bnds, lrglac_bnds, precfactor_bnds, precgrad_bnds, ddfsnow_bnds, ddfice_bnds,
                                    tempsnow_bnds, tempchange_bnds)
            # OPTIMIZATION ROUND #1: optimize precfactor, DDFsnow, tempchange
            # Select constraints used to optimize precfactor
            cons = [con_lrgcm, con_lrglac, con_ddficefxsnow, con_tempsnow]
            # Run the optimization
            #  'L-BFGS-B' - much slower
            modelparameters_opt = minimize(objective, modelparameters_init, method='SLSQP', bounds=modelparameters_bnds,
                                           constraints=cons, tol=1e-3)
            # Record the calibration round
            calround = calround + 1
            # Record the optimized parameters
            main_glac_modelparamsopt[glac] = modelparameters_opt.x
            # Re-run the optimized parameters in order to see the mass balance
            (glac_wide_massbaltotal, glac_wide_runoff, glac_wide_snowline, glac_wide_snowpack, glac_wide_area_annual,
             glac_wide_volume_annual, glac_wide_ELA_annual) = (
                massbalance.runmassbalance(main_glac_modelparamsopt[glac], glacier_rgi_table, glacier_area_t0,
                                           icethickness_t0, width_t0, elev_bins, glacier_gcm_temp, glacier_gcm_prec,
                                           glacier_gcm_elev, glacier_gcm_lrgcm, glacier_gcm_lrglac, dates_table))
            # Compare calibration data
            # Column index for start and end year based on dates of geodetic mass balance observations
            massbal_idx_start = int(glacier_rgi_table.loc[input.massbal_time1] - input.startyear)
            massbal_idx_end = int(massbal_idx_start + glacier_rgi_table.loc[input.massbal_time2] -
                                  glacier_rgi_table.loc[input.massbal_time1] + 1)
            # Annual glacier-wide mass balance [m w.e.]
            glac_wide_massbaltotal_annual = np.sum(glac_wide_massbaltotal.reshape(-1,12), axis=1)
            # Average annual glacier-wide mass balance [m w.e.a.]
            glac_wide_massbaltotal_annual_avg = glac_wide_massbaltotal_annual[massbal_idx_start:massbal_idx_end].mean()
            #  units: m w.e. based on initial area
            # Difference between geodetic and modeled mass balance
            massbal_difference = abs(glacier_rgi_table[input.massbal_colname] - glac_wide_massbaltotal_annual_avg)
            main_glac_massbal_compare[glac] = [glac_wide_massbaltotal_annual_avg,
                                               glacier_rgi_table.loc[input.massbal_colname], massbal_difference,
                                               calround]

            print('precfactor:', main_glac_modelparamsopt[glac,2])
            print('precgrad:', main_glac_modelparamsopt[glac,3])
            print('ddfsnow:', main_glac_modelparamsopt[glac,4])
            print('tempchange:', main_glac_modelparamsopt[glac,7])
            print(glacier_rgi_table.loc[input.massbal_colname],
                  glacier_rgi_table.loc[input.massbal_uncertainty_colname], glac_wide_massbaltotal_annual_avg,
                  massbal_difference)

            # OPTIMIZATION ROUND #2: if tolerance not reached, increase bounds
            if massbal_difference > input.massbal_tolerance:
                # Constraints
                cons = [con_lrgcm, con_lrglac, con_precgrad, con_ddficefxsnow, con_tempsnow]
                # Bounds
                lrgcm_bnds = (-0.008,-0.004)
                lrglac_bnds = (-0.008,-0.004)
                precfactor_bnds = (0.75,1.5)
                precgrad_bnds = (0.0001,0.00025)
                ddfsnow_bnds = (0.0031, 0.0051)
                ddfice_bnds = (ddfsnow_bnds[0]/input.ddfsnow_iceratio, ddfsnow_bnds[1]/input.ddfsnow_iceratio)
                tempsnow_bnds = (0,2)
                tempchange_bnds = (-2,2)
                modelparameters_bnds = (lrgcm_bnds, lrglac_bnds, precfactor_bnds, precgrad_bnds, ddfsnow_bnds,
                                        ddfice_bnds, tempsnow_bnds, tempchange_bnds)
                # Run optimization
                modelparameters_opt = minimize(objective, main_glac_modelparamsopt[glac], method='SLSQP',
                                               bounds=modelparameters_bnds, constraints=cons, tol=1e-3)
                # Record the calibration round
                calround = calround + 1
                # Record the optimized parameters
                main_glac_modelparamsopt[glac] = modelparameters_opt.x
                # Re-run the optimized parameters in order to see the mass balance
                (glac_wide_massbaltotal, glac_wide_runoff, glac_wide_snowline, glac_wide_snowpack,
                 glac_wide_area_annual, glac_wide_volume_annual, glac_wide_ELA_annual) = (
                    massbalance.runmassbalance(main_glac_modelparamsopt[glac], glacier_rgi_table, glacier_area_t0,
                                               icethickness_t0, width_t0, elev_bins, glacier_gcm_temp, glacier_gcm_prec,
                                               glacier_gcm_elev, glacier_gcm_lrgcm, glacier_gcm_lrglac, dates_table))
                # Compare calibration data
                # Column index for start and end year based on dates of geodetic mass balance observations
                massbal_idx_start = int(glacier_rgi_table.loc[input.massbal_time1] - input.startyear)
                massbal_idx_end = int(massbal_idx_start + glacier_rgi_table.loc[input.massbal_time2] -
                                      glacier_rgi_table.loc[input.massbal_time1] + 1)
                # Annual glacier-wide mass balance [m w.e.]
                glac_wide_massbaltotal_annual = np.sum(glac_wide_massbaltotal.reshape(-1,12), axis=1)
                # Average annual glacier-wide mass balance [m w.e.a.]
                glac_wide_massbaltotal_annual_avg = (
                        glac_wide_massbaltotal_annual[massbal_idx_start:massbal_idx_end].mean())
                #  units: m w.e. based on initial area
                # Difference between geodetic and modeled mass balance
                massbal_difference = abs(glacier_rgi_table[input.massbal_colname] - glac_wide_massbaltotal_annual_avg)
                main_glac_massbal_compare[glac] = [glac_wide_massbaltotal_annual_avg,
                                                   glacier_rgi_table.loc[input.massbal_colname], massbal_difference,
                                                   calround]

                print('precfactor:', main_glac_modelparamsopt[glac,2])
                print('precgrad:', main_glac_modelparamsopt[glac,3])
                print('ddfsnow:', main_glac_modelparamsopt[glac,4])
                print('tempchange:', main_glac_modelparamsopt[glac,7])
                print(glacier_rgi_table.loc[input.massbal_colname],
                      glacier_rgi_table.loc[input.massbal_uncertainty_colname], glac_wide_massbaltotal_annual_avg,
                      massbal_difference)

            # OPTIMIZATION ROUND #3: if tolerance not reached, increase bounds again
            if massbal_difference > input.massbal_tolerance:
                # Constraints
                cons = [con_lrgcm, con_lrglac, con_precgrad, con_ddficefxsnow, con_tempsnow]
                # Bounds
                lrgcm_bnds = (-0.008,-0.004)
                lrglac_bnds = (-0.008,-0.004)
                precfactor_bnds = (0.5,2)
                precgrad_bnds = (0.0001,0.00025)
                ddfsnow_bnds = (0.0026, 0.0056)
                ddfice_bnds = (ddfsnow_bnds[0]/input.ddfsnow_iceratio, ddfsnow_bnds[1]/input.ddfsnow_iceratio)
                tempsnow_bnds = (0,2)
                tempchange_bnds = (-5,5)
                modelparameters_bnds = (lrgcm_bnds, lrglac_bnds, precfactor_bnds, precgrad_bnds, ddfsnow_bnds,
                                        ddfice_bnds, tempsnow_bnds, tempchange_bnds)
                # Run optimization
                modelparameters_opt = minimize(objective, main_glac_modelparamsopt[glac], method='SLSQP',
                                               bounds=modelparameters_bnds, constraints=cons, tol=1e-4)
                #  requires higher tolerance due to the increase in parameters being optimized
                # Record the calibration round
                calround = calround + 1
                # Record the optimized parameters
                main_glac_modelparamsopt[glac] = modelparameters_opt.x
                # Re-run the optimized parameters in order to see the mass balance
                (glac_wide_massbaltotal, glac_wide_runoff, glac_wide_snowline, glac_wide_snowpack,
                 glac_wide_area_annual, glac_wide_volume_annual, glac_wide_ELA_annual) = (
                    massbalance.runmassbalance(main_glac_modelparamsopt[glac], glacier_rgi_table, glacier_area_t0,
                                               icethickness_t0, width_t0, elev_bins, glacier_gcm_temp, glacier_gcm_prec,
                                               glacier_gcm_elev, glacier_gcm_lrgcm, glacier_gcm_lrglac, dates_table))
                # Compare calibration data
                # Column index for start and end year based on dates of geodetic mass balance observations
                massbal_idx_start = int(glacier_rgi_table.loc[input.massbal_time1] - input.startyear)
                massbal_idx_end = int(massbal_idx_start + glacier_rgi_table.loc[input.massbal_time2] -
                                      glacier_rgi_table.loc[input.massbal_time1] + 1)
                # Annual glacier-wide mass balance [m w.e.]
                glac_wide_massbaltotal_annual = np.sum(glac_wide_massbaltotal.reshape(-1,12), axis=1)
                # Average annual glacier-wide mass balance [m w.e.a.]
                glac_wide_massbaltotal_annual_avg = (
                        glac_wide_massbaltotal_annual[massbal_idx_start:massbal_idx_end].mean())
                #  units: m w.e. based on initial area
                # Difference between geodetic and modeled mass balance
                massbal_difference = abs(glacier_rgi_table[input.massbal_colname] - glac_wide_massbaltotal_annual_avg)
                main_glac_massbal_compare[glac] = [glac_wide_massbaltotal_annual_avg,
                                                   glacier_rgi_table.loc[input.massbal_colname],
                                                   massbal_difference, calround]

                print('precfactor:', main_glac_modelparamsopt[glac,2])
                print('precgrad:', main_glac_modelparamsopt[glac,3])
                print('ddfsnow:', main_glac_modelparamsopt[glac,4])
                print('tempchange:', main_glac_modelparamsopt[glac,7])
                print(glacier_rgi_table.loc[input.massbal_colname],
                      glacier_rgi_table.loc[input.massbal_uncertainty_colname], glac_wide_massbaltotal_annual_avg,
                      massbal_difference)
        else:
        # if calibration data not available for a glacier, then insert NaN into calibration output
            main_glac_modelparamsopt[glac] = float('NaN')
            main_glac_massbal_compare[glac] = float('NaN')

        # Output calibration results to .csv file
        #  pandas dataframe used instead of numpy arrays here, so column headings can be exported
        main_glac_caloutput = main_glac_rgi.copy()
        main_glac_caloutput['MB_model_mwea'] = main_glac_massbal_compare[:,0]
        main_glac_caloutput['MB_geodetic_mwea'] = main_glac_massbal_compare[:,1]
        main_glac_caloutput['MB_difference_mwea'] = main_glac_massbal_compare[:,2]
        main_glac_caloutput['calround'] = main_glac_massbal_compare[:,3]
        main_glac_caloutput['lrgcm'] = main_glac_modelparamsopt[:,0]
        main_glac_caloutput['lrglac'] = main_glac_modelparamsopt[:,1]
        main_glac_caloutput['precfactor'] = main_glac_modelparamsopt[:,2]
        main_glac_caloutput['precgrad'] = main_glac_modelparamsopt[:,3]
        main_glac_caloutput['ddfsnow'] = main_glac_modelparamsopt[:,4]
        main_glac_caloutput['ddfice'] = main_glac_modelparamsopt[:,5]
        main_glac_caloutput['tempsnow'] = main_glac_modelparamsopt[:,6]
        main_glac_caloutput['tempchange'] = main_glac_modelparamsopt[:,7]
        # export csv
        cal_output_fullfile = (input.output_filepath + input.calibrationcsv_filenameprefix + 'R' + str(regionO1_number)
                               + '_' + str(strftime("%Y%m%d")) + '.csv')
        main_glac_caloutput.to_csv(cal_output_fullfile)

# GRID SEARCH CALIBRATION
elif input.option_calibration == 2:
    #----- IMPORT CALIBRATION DATASETS --------------------------------------------------------------------------------
    # Import geodetic mass balance from David Shean
    main_glac_calmassbal = modelsetup.selectcalibrationdata(main_glac_rgi)

    #----- ENTER GRID SEARCH RUN --------------------------------------------------------------------------------------
    # [INSERT REGIONAL LOOP HERE] if want to do all regions at the same time.  Separate netcdf files will be generated
    #  for each loop to reduce file size and make files easier to read/share
    regionO1_number = input.rgi_regionsO1[0]
    # Create output netcdf file
#    if input.output_package != 0:
#        output.netcdfcreate(regionO1_number, main_glac_hyps, dates_table, annual_columns)

    # Create grid of all parameter sets
    grid_precfactor = np.arange(0.75, 2, 0.25)
    grid_tempbias = np.arange(-4, 6, 2)
    grid_ddfsnow = np.arange(0.0031, 0.0056, 0.0005)
    grid_precgrad = np.arange(0.001, 0.007, 0.002)

    grid_modelparameters = np.zeros((grid_precfactor.shape[0] * grid_tempbias.shape[0] * grid_ddfsnow.shape[0] *
                                     grid_precgrad.shape[0],8))
    grid_count = 0
    for n_precfactor in range(grid_precfactor.shape[0]):
        for n_tempbias in range(grid_tempbias.shape[0]):
            for n_ddfsnow in range(grid_ddfsnow.shape[0]):
                for n_precgrad in range(grid_precgrad.shape[0]):
                    # Set grid of model parameters
                    grid_modelparameters[grid_count,:] = [input.lrgcm, input.lrglac, grid_precfactor[n_precfactor],
                          grid_precgrad[n_precgrad], grid_ddfsnow[n_ddfsnow], grid_ddfsnow[n_ddfsnow] / 0.7,
                          input.tempsnow, grid_tempbias[n_tempbias]]
                    grid_count = grid_count + 1

#    # Create netcdf HERE
#    # Netcdf file path and name
#    filename = input.calibrationnetcdf_filenameprefix + str(regionO1_number) + '_' + str(strftime("%Y%m%d")) + '.nc'
#    fullfile = input.output_filepath + filename
#    # Create netcdf file ('w' will overwrite existing files, 'r+' will open existing file to write)
#    netcdf_output = nc.Dataset(fullfile, 'w', format='NETCDF4')
#    # Global attributes
#    netcdf_output.description = 'Results from glacier evolution model'
#    netcdf_output.history = 'Created ' + str(strftime("%Y-%m-%d %H:%M:%S"))
#    netcdf_output.source = 'Python Glacier Evolution Model'
#    # Dimensions
#    glacier = netcdf_output.createDimension('glacier', None)
#    binelev = netcdf_output.createDimension('binelev', main_glac_hyps.shape[1])
#    if input.timestep == 'monthly':
#        time = netcdf_output.createDimension('time', dates_table.shape[0] - input.spinupyears * 12)
#    year = netcdf_output.createDimension('year', annual_columns.shape[0] - input.spinupyears)
#    year_plus1 = netcdf_output.createDimension('year_plus1', annual_columns.shape[0] - input.spinupyears + 1)
#    gridround = netcdf_output.createDimension('gridround', grid_modelparameters.shape[0])
#    glacierinfo = netcdf_output.createDimension('glacierinfo', 3)
#    # Variables associated with dimensions
#    glaciers = netcdf_output.createVariable('glacier', np.int32, ('glacier',))
#    glaciers.long_name = "glacier number associated with model run"
#    glaciers.standard_name = "GlacNo"
#    glaciers.comment = ("The glacier number is defined for each model run. The user should look at the main_glac_rgi"
#                           + " table to determine the RGIID or other information regarding this particular glacier.")
#    binelevs = netcdf_output.createVariable('binelev', np.int32, ('binelev',))
#    binelevs.standard_name = "center bin_elevation"
#    binelevs.units = "m a.s.l."
#    binelevs[:] = main_glac_hyps.columns.values
#    binelevs.comment = ("binelev are the bin elevations that were used for the model run.")
#    times = netcdf_output.createVariable('time', np.float64, ('time',))
#    times.standard_name = "date"
#    times.units = "days since 1900-01-01 00:00:00"
#    times.calendar = "gregorian"
#    if input.timestep == 'monthly':
#        times[:] = (nc.date2num(dates_table.loc[input.spinupyears*12:dates_table.shape[0]+1,'date'].astype(datetime),
#                                units = times.units, calendar = times.calendar))
#    years = netcdf_output.createVariable('year', np.int32, ('year',))
#    years.standard_name = "year"
#    if input.option_wateryear == 1:
#        years.units = 'water year'
#    elif input.option_wateryear == 0:
#        years.units = 'calendar year'
#    years[:] = annual_columns[input.spinupyears:annual_columns.shape[0]]
#    # years_plus1 adds an additional year such that the change in glacier dimensions (area, etc.) is recorded
#    years_plus1 = netcdf_output.createVariable('year_plus1', np.int32, ('year_plus1',))
#    years_plus1.standard_name = "year with additional year to record glacier dimension changes"
#    if input.option_wateryear == 1:
#        years_plus1.units = 'water year'
#    elif input.option_wateryear == 0:
#        years_plus1.units = 'calendar year'
#    years_plus1 = np.concatenate((annual_columns[input.spinupyears:annual_columns.shape[0]],
#                                  np.array([annual_columns[annual_columns.shape[0]-1]+1])))
#    gridrounds = netcdf_output.createVariable('gridround', np.int32, ('gridround',))
#    gridrounds.long_name = "number associated with the calibration grid search"
#    glacierinfoheader = netcdf_output.createVariable('glacierinfoheader', str, ('glacierinfo',))
#    glacierinfoheader.standard_name = "information about each glacier from main_glac_rgi"
#    glacierinfoheader[:] = np.array(['RGIID','lat','lon'])
#    glacierinfo = netcdf_output.createVariable('glacierinfo',str,('glacier','glacierinfo',))
#
#    # Variables associated with the output
#    #  monthly glacier-wide variables (massbal_total, runoff, snowline, snowpack)
#    #  annual glacier-wide variables (area, volume, ELA)
#    massbaltotal_glac_monthly = netcdf_output.createVariable('massbaltotal_glac_monthly', np.float64,
#                                                             ('glacier', 'gridround', 'time'))
#    massbaltotal_glac_monthly.standard_name = "glacier-wide total mass balance"
#    massbaltotal_glac_monthly.units = "m w.e."
#    massbaltotal_glac_monthly.comment = ("total mass balance is the sum of the climatic mass balance and frontal "
#                                         + "ablation.")
#    runoff_glac_monthly = netcdf_output.createVariable('runoff_glac_monthly', np.float64,
#                                                       ('glacier', 'gridround', 'time'))
#    runoff_glac_monthly.standard_name = "glacier runoff"
#    runoff_glac_monthly.units = "m**3"
#    runoff_glac_monthly.comment = "runoff from the glacier terminus, which moves over time"
#    snowline_glac_monthly = netcdf_output.createVariable('snowline_glac_monthly', np.float64,
#                                                         ('glacier', 'gridround', 'time'))
#    snowline_glac_monthly.standard_name = "transient snowline"
#    snowline_glac_monthly.units = "m a.s.l."
#    snowline_glac_monthly.comment = "transient snowline is the line separating the snow from ice/firn"
#    snowpack_glac_monthly = netcdf_output.createVariable('snowpack_glac_monthly', np.float64,
#                                                         ('glacier', 'gridround', 'time'))
#    snowpack_glac_monthly.standard_name = "snowpack volume"
#    snowpack_glac_monthly.units = "km**3 w.e."
#    snowpack_glac_monthly.comment = "m w.e. multiplied by the area converted to km**3"
#    area_glac_annual = netcdf_output.createVariable('area_glac_annual', np.float64,
#                                                    ('glacier', 'gridround', 'year_plus1'))
#    area_glac_annual.standard_name = "glacier area"
#    area_glac_annual.units = "km**2"
#    area_glac_annual.comment = "the area that was used for the duration of the year"
#    volume_glac_annual = netcdf_output.createVariable('volume_glac_annual', np.float64,
#                                                      ('glacier', 'gridround', 'year_plus1'))
#    volume_glac_annual.standard_name = "glacier volume"
#    volume_glac_annual.units = "km**3 ice"
#    volume_glac_annual.comment = "the volume based on area and ice thickness used for that year"
#    ELA_glac_annual = netcdf_output.createVariable('ELA_glac_annual', np.float64, ('glacier', 'gridround', 'year'))
#    ELA_glac_annual.standard_name = "annual equilibrium line altitude"
#    ELA_glac_annual.units = "m a.s.l."
#    ELA_glac_annual.comment = "equilibrium line altitude is the elevation where the climatic mass balance is zero"
#    netcdf_output.close()

     # Loop through the glaciers for each grid of parameter sets
    for glac in range(main_glac_rgi.shape[0]):
#    for glac in range(25):
#    for glac in [1]:
        print(glac)
        # Run calibration only for glaciers that have calibration data
        if np.isnan(main_glac_calmassbal[glac,0]) == False:
            # Select subset of variables to reduce the amount of data being passed to the function
            glacier_rgi_table = main_glac_rgi.loc[glac, :]
            glacier_gcm_elev = main_glac_gcmelev[glac]
            glacier_gcm_prec = main_glac_gcmprec[glac,:]
            glacier_gcm_temp = main_glac_gcmtemp[glac,:]
            # Select annual divisor and columns
            if input.timestep == 'monthly':
                annual_divisor = 12
            annual_columns = np.unique(dates_table['wateryear'].values)
            # Set new output
            output_glac_wide_massbaltotal = np.zeros((grid_modelparameters.shape[0], glacier_gcm_temp.shape[0] -
                                                      input.spinupyears*12))
            output_glac_wide_runoff = np.zeros((grid_modelparameters.shape[0], glacier_gcm_temp.shape[0] -
                                                input.spinupyears*12))
            output_glac_wide_snowline = np.zeros((grid_modelparameters.shape[0], glacier_gcm_temp.shape[0] -
                                                  input.spinupyears*12))
            output_glac_wide_snowpack = np.zeros((grid_modelparameters.shape[0], glacier_gcm_temp.shape[0] -
                                                  input.spinupyears*12))
            output_glac_wide_area_annual = np.zeros((grid_modelparameters.shape[0], annual_columns.shape[0] -
                                                     input.spinupyears + 1))
            output_glac_wide_volume_annual = np.zeros((grid_modelparameters.shape[0], annual_columns.shape[0] -
                                                       input.spinupyears + 1))
            output_glac_wide_ELA_annual = np.zeros((grid_modelparameters.shape[0], annual_columns.shape[0] -
                                                    input.spinupyears))
            # Loop through each set of parameters
            for grid_round in range(grid_modelparameters.shape[0]):
                # Set model parameters
                modelparameters = grid_modelparameters[grid_round, :]
                # Set lapse rate based on options
                if input.option_lapserate_fromgcm == 1:
                    glacier_gcm_lrgcm = main_glac_gcmlapserate[glac]
                    glacier_gcm_lrglac = glacier_gcm_lrgcm.copy()
                elif input.option_lapserate_fromgcm == 0:
                    glacier_gcm_lrgcm = np.zeros(glacier_gcm_temp.shape) + modelparameters[0]
                    glacier_gcm_lrglac = np.zeros(glacier_gcm_temp.shape) + modelparameters[1]
                # Reset initial parameters
                glacier_area_t0 = main_glac_hyps.iloc[glac,:].values.astype(float)
                icethickness_t0 = main_glac_icethickness.iloc[glac,:].values.astype(float)
                width_t0 = main_glac_width.iloc[glac,:].values.astype(float)
                # MASS BALANCE
                # Run the mass balance function (spinup years have been removed from output)
                (glac_bin_temp, glac_bin_prec, glac_bin_acc, glac_bin_refreeze, glac_bin_snowpack, glac_bin_melt,
                 glac_bin_frontalablation, glac_bin_massbalclim, glac_bin_massbalclim_annual, glac_bin_area_annual,
                 glac_bin_icethickness_annual, glac_bin_width_annual, glac_bin_surfacetype_annual) = (
                         massbalance.runmassbalance(glac, modelparameters, regionO1_number, glacier_rgi_table,
                                                    glacier_area_t0, icethickness_t0, width_t0, glacier_gcm_temp,
                                                    glacier_gcm_prec, glacier_gcm_elev, glacier_gcm_lrgcm,
                                                    glacier_gcm_lrglac, elev_bins, dates_table, annual_columns,
                                                    annual_divisor))
                glac_wide_prec = np.zeros(glac_bin_temp.shape[1])
                glac_wide_acc = np.zeros(glac_bin_temp.shape[1])
                glac_wide_refreeze = np.zeros(glac_bin_temp.shape[1])
                glac_wide_melt = np.zeros(glac_bin_temp.shape[1])
                glac_wide_frontalablation = np.zeros(glac_bin_temp.shape[1])
                # Compute desired output
                glac_bin_area = glac_bin_area_annual[:,0:glac_bin_area_annual.shape[1]-1].repeat(12,axis=1)
                glac_wide_area = glac_bin_area.sum(axis=0)
                glac_wide_prec_mkm2 = (glac_bin_prec * glac_bin_area).sum(axis=0)
                glac_wide_prec[glac_wide_prec_mkm2 > 0] = (glac_wide_prec_mkm2[glac_wide_prec_mkm2 > 0] /
                                                           glac_wide_area[glac_wide_prec_mkm2 > 0])
                glac_wide_acc_mkm2 = (glac_bin_acc * glac_bin_area).sum(axis=0)
                glac_wide_acc[glac_wide_acc_mkm2 > 0] = (glac_wide_acc_mkm2[glac_wide_acc_mkm2 > 0] /
                                                         glac_wide_area[glac_wide_acc_mkm2 > 0])
                glac_wide_refreeze_mkm2 = (glac_bin_refreeze * glac_bin_area).sum(axis=0)
                glac_wide_refreeze[glac_wide_refreeze_mkm2 > 0] = (glac_wide_refreeze_mkm2[glac_wide_refreeze_mkm2 > 0]
                                                                   / glac_wide_area[glac_wide_refreeze_mkm2 > 0])
                glac_wide_melt_mkm2 = (glac_bin_melt * glac_bin_area).sum(axis=0)
                glac_wide_melt[glac_wide_melt_mkm2 > 0] = (glac_wide_melt_mkm2[glac_wide_melt_mkm2 > 0] /
                                                           glac_wide_area[glac_wide_melt_mkm2 > 0])
                glac_wide_frontalablation_mkm2 = (glac_bin_frontalablation * glac_bin_area).sum(axis=0)
                glac_wide_frontalablation[glac_wide_frontalablation_mkm2 > 0] = (
                        glac_wide_frontalablation_mkm2[glac_wide_frontalablation_mkm2 > 0] /
                        glac_wide_area[glac_wide_frontalablation_mkm2 > 0])
                glac_wide_massbalclim = glac_wide_acc + glac_wide_refreeze - glac_wide_melt
                glac_wide_massbaltotal = glac_wide_massbalclim - glac_wide_frontalablation
                glac_wide_runoff = (glac_wide_prec + glac_wide_melt - glac_wide_refreeze) * glac_wide_area * (1000)**2
                #  units: (m + m w.e. - m w.e.) * km**2 * (1000 m / 1 km)**2 = m**3
                glac_wide_snowline = (glac_bin_snowpack > 0).argmax(axis=0)
                glac_wide_snowline[glac_wide_snowline > 0] = (elev_bins[glac_wide_snowline[glac_wide_snowline > 0]] -
                                                              input.binsize/2)
                glac_wide_area_annual = glac_bin_area_annual.sum(axis=0)
                glac_wide_volume_annual = (glac_bin_area_annual * glac_bin_icethickness_annual / 1000).sum(axis=0)
                glac_wide_ELA_annual = (glac_bin_massbalclim_annual > 0).argmax(axis=0)
                glac_wide_ELA_annual[glac_wide_ELA_annual > 0] = (
                        elev_bins[glac_wide_ELA_annual[glac_wide_ELA_annual > 0]] - input.binsize/2)
                # Record desired output
                output_glac_wide_massbaltotal[grid_round,:] = glac_wide_massbaltotal
                output_glac_wide_runoff[grid_round,:] = glac_wide_runoff
                output_glac_wide_snowline[grid_round,:] = glac_wide_snowline
                output_glac_wide_snowpack[grid_round,:] = glac_wide_acc_mkm2 / 1000
                output_glac_wide_area_annual[grid_round,:] = glac_wide_area_annual
                output_glac_wide_volume_annual[grid_round,:] = glac_wide_volume_annual
                output_glac_wide_ELA_annual[grid_round,:] = glac_wide_ELA_annual
            # Netcdf file path and name
            filename = (input.calibrationnetcdf_filenameprefix + str(regionO1_number) + '_' + str(strftime("%Y%m%d"))
                        + '.nc')
            fullfile = input.output_filepath + filename
            # Open netcdf file to write to existing file ('r+')
            netcdf_output = nc.Dataset(fullfile, 'r+')
            # Write variables to netcdf
            netcdf_output.variables['glacierinfo'][glac,:] = np.array([glacier_rgi_table.loc['RGIId'],
                    glacier_rgi_table.loc[input.lat_colname], glacier_rgi_table.loc[input.lon_colname]])
            netcdf_output.variables['massbaltotal_glac_monthly'][glac,:,:] = output_glac_wide_massbaltotal
            netcdf_output.variables['runoff_glac_monthly'][glac,:,:] = output_glac_wide_runoff
            netcdf_output.variables['snowline_glac_monthly'][glac,:,:] = output_glac_wide_snowline
            netcdf_output.variables['snowpack_glac_monthly'][glac,:,:] = output_glac_wide_snowpack
            netcdf_output.variables['area_glac_annual'][glac,:,:] = output_glac_wide_area_annual
            netcdf_output.variables['volume_glac_annual'][glac,:,:] = output_glac_wide_volume_annual
            netcdf_output.variables['ELA_glac_annual'][glac,:,:] = output_glac_wide_ELA_annual
            netcdf_output.close()

#timeelapsed_step4 = timeit.default_timer() - timestart_step4
#print('Step 4 time:', timeelapsed_step4, "s\n")

#%%=== STEP FIVE: SIMULATION RUN ======================================================================================
#timestart_step5 = timeit.default_timer()

if input.option_calibration == 0:
    # [INSERT REGIONAL LOOP HERE] if want to do all regions at the same time.  Separate netcdf files will be generated
    #  for each loop to reduce file size and make files easier to read/share

    # Create output netcdf file
    if input.output_package != 0:
        netcdf_fn = (input.netcdf_fn_prefix + str(input.rgi_regionsO1[0]) + '_' + str(input.startyear) + '_' +
                     str(input.endyear) + '_' + str(strftime("%Y%m%d")) + '.nc')
        output.netcdfcreate(netcdf_fn, main_glac_rgi, main_glac_hyps, dates_table)

    # Load model parameters
    if input.option_loadparameters == 1:
        # Model parameters
        main_glac_modelparams_all = pd.read_csv(input.modelparams_filepath + input.modelparams_filename, index_col=0)
        main_glac_modelparams = main_glac_modelparams_all.loc[main_glac_rgi['O1Index'].values, :]
#        main_glac_modelparams_all = pd.read_csv(input.modelparams_filepath + input.modelparams_filename)
#        main_glac_modelparams = main_glac_modelparams_all.iloc[main_glac_rgi['O1Index'].values]
#        main_glac_modelparams.reset_index(drop=True, inplace=True)
#        main_glac_modelparams.index.name = input.indexname
    else:
        main_glac_modelparams = pd.DataFrame(np.repeat([input.lrgcm, input.lrglac, input.precfactor, input.precgrad,
            input.ddfsnow, input.ddfice, input.tempsnow, input.tempchange], main_glac_rgi.shape[0]).reshape(-1,
            main_glac_rgi.shape[0]).transpose(), columns=input.modelparams_colnames)

    # Test range
    #glac = 0
    #prec_factor_low = 0.8
    #prec_factor_high = 2.0
    #prec_factor_step = 0.005
    #prec_factor_range = np.arange(prec_factor_low, prec_factor_high + prec_factor_step, prec_factor_step)
    #glac_wide_massbal_record = np.zeros(prec_factor_range.shape)
    #for n in range(len(prec_factor_range)):
    #    prec_factor = prec_factor_range[n]

    # ENTER GLACIER LOOP
    for glac in range(main_glac_rgi.shape[0]):
#    for glac in [0]:  # for testing a single glacier
        print(main_glac_rgi.loc[glac,'RGIId'])
        # Select subset of variables to reduce the amount of data being passed to the function
#        modelparameters = main_glac_modelparams.loc[glac,:].values
        modelparameters = main_glac_modelparams.loc[main_glac_modelparams.index.values[glac],input.modelparams_colnames]
        glacier_rgi_table = main_glac_rgi.loc[glac, :]
        glacier_gcm_elev = main_glac_gcmelev[glac]
        glacier_gcm_prec = main_glac_gcmprec[glac,:]
        glacier_gcm_temp = main_glac_gcmtemp[glac,:]
        if input.option_lapserate_fromgcm == 1:
            glacier_gcm_lrgcm = main_glac_gcmlapserate[glac]
            glacier_gcm_lrglac = glacier_gcm_lrgcm.copy()
        elif input.option_lapserate_fromgcm == 0:
            glacier_gcm_lrgcm = np.zeros(glacier_gcm_temp.shape) + modelparameters[0]
            glacier_gcm_lrglac = np.zeros(glacier_gcm_temp.shape) + modelparameters[1]
        glacier_area_t0 = main_glac_hyps.iloc[glac,:].values.astype(float)
        # Inclusion of ice thickness and width, i.e., loading values may be only required for Huss mass redistribution!
        icethickness_t0 = main_glac_icethickness.iloc[glac,:].values.astype(float)
        width_t0 = main_glac_width.iloc[glac,:].values.astype(float)
        # MASS BALANCE
        # Run the mass balance function (spinup years have been removed from output)
        (glac_bin_temp, glac_bin_prec, glac_bin_acc, glac_bin_refreeze, glac_bin_snowpack, glac_bin_melt,
         glac_bin_frontalablation, glac_bin_massbalclim, glac_bin_massbalclim_annual, glac_bin_area_annual,
         glac_bin_icethickness_annual, glac_bin_width_annual, glac_bin_surfacetype_annual,
         glac_wide_massbaltotal, glac_wide_runoff, glac_wide_snowline, glac_wide_snowpack, glac_wide_area_annual,
         glac_wide_volume_annual, glac_wide_ELA_annual) = (
            massbalance.runmassbalance(modelparameters, glacier_rgi_table, glacier_area_t0, icethickness_t0,
                                       width_t0, elev_bins, glacier_gcm_temp, glacier_gcm_prec, glacier_gcm_elev,
                                       glacier_gcm_lrgcm, glacier_gcm_lrglac, dates_table))
        # Volume change [%]
        if icethickness_t0.max() > 0:
            glac_vol_change_prec = (
                (glac_wide_volume_annual[-1] - glac_wide_volume_annual[0]) / glac_wide_volume_annual[0] * 100)
        # Annual glacier-wide mass balance [m w.e.]
        glac_wide_massbaltotal_annual = np.sum(glac_wide_massbaltotal.reshape(-1,12), axis=1)
        # Average annual glacier-wide mass balance [m w.e.a.]
        massbal_idx_start = 0
        massbal_idx_end = 16
        mb_mwea = glac_wide_massbaltotal_annual[massbal_idx_start:massbal_idx_end].mean()
        # OUTPUT: Record variables according to output package
        #  must be done within glacier loop since the variables will be overwritten
        if input.output_package != 0:
            output.netcdfwrite(netcdf_fn, glac, modelparameters, glacier_rgi_table, elev_bins, glac_bin_temp,
                               glac_bin_prec, glac_bin_acc, glac_bin_refreeze, glac_bin_snowpack, glac_bin_melt,
                               glac_bin_frontalablation, glac_bin_massbalclim, glac_bin_massbalclim_annual,
                               glac_bin_area_annual, glac_bin_icethickness_annual, glac_bin_width_annual,
                               glac_bin_surfacetype_annual)
            # POTENTIAL IMPROVEMENT: OPEN THE FILE WITHIN THE MAIN SCRIPT AND THEN GO INTO THE FUNCTION TO WRITE TO THE
            #                        NETCDF FILE - THIS WAY NO LONGER HAVE TO OPEN AND CLOSE EACH TIME

#timeelapsed_step5 = timeit.default_timer() - timestart_step5
#print('Step 5 time:', timeelapsed_step5, "s\n")

##%%=== Model testing ===============================================================================
##netcdf_output = nc.Dataset(input.main_directory + '/' + netcdf_fullfilename, 'r+')


#!/usr/bin/python3

from itertools import groupby

filepath = input()
result = []
with open(filepath) as f:
	while True:
		line = f.readline()
		if not line:
			break
		if not line.startswith('#'):
			result.append(line)

split_tuples = [string.split() for string in result]
grouped_by_protein = groupby(split_tuples, key=lambda x: x[0])
max_tuples = [max(t[1], key=lambda x: x[2])   for t in grouped_by_protein]
tuples_to_print = [' '.join(l) for l in max_tuples]
print('\n'.join(tuples_to_print))


# This file is to use two kernels.

import numpy as np
import matplotlib.pyplot as plt
from sobol import sobol_rvs
from scipy import integrate
from scipy.linalg import eigh
from scipy.interpolate import interp1d

sig1=0.5
sig2=2
sigma=1
L=20
N=30     # samples
NN=int(N/2)
n_r=101  # raw image
n=101   # kernel
Ntime=3000

def kernel(x):
    # y = np.exp(-x)
    y = np.exp(-x * x / (2 * sig1 * sig1)) / np.sqrt(2 * sig1 * sig1 * np.pi)
    return y

def kernel2(x):
    # y = np.exp(-x)
    y = np.exp(-x * x / (2 * sig2 * sig2)) / np.sqrt(2 * sig2 * sig2 * np.pi)
    return y

# raw image
xi = np.linspace(-L/2, L/2, n_r)
yi = np.exp(-1*np.abs(xi))
integral = integrate.simps(yi)
yi = yi/integral

# kernel
xk1= np.linspace(-L/2, L/2, n)
yk1 = kernel(xk1)
xk2= np.linspace(-L/2, L/2, n)
yk2= kernel2(xk2)
xk = np.linspace(-L/2, L/2, n)
yk = np.linspace(1,1,n/4)
ker1 = np.convolve(yk1, yk, 'same')
integral = integrate.simps(ker1)
ker1 = ker1/integral
f_ker1=interp1d(xk1, ker1, kind='linear')
ker2 = np.convolve(yk2, yk, 'same')
integral = integrate.simps(ker2)
ker2 = ker2/integral
f_ker2=interp1d(xk1, ker2, kind='linear')
# Here, kernel is (-L/2, L/2), number of points is n.

image1=np.convolve(yi,ker1,'same')
f_image1=interp1d(xk1, image1, kind='linear')
observe1 = (sobol_rvs(NN,skip=1000)-0.5)*L

image2=np.convolve(yi,ker2,'same')
f_image2=interp1d(xk2, image2, kind='linear')
observe2 = (sobol_rvs(NN,skip=2000)-0.5)*L

observe=np.concatenate((observe1,observe2), axis=0)
# observe=observe[::-1]

# image is the convolution of raw image and kernel. range is with raw image, size=raw image
# values of samples, number is N. (-L/2, L/2)
X = np.zeros([N,n])
for k in range(NN):
    for i in range(n):
        if observe[k]-xk[i]<-L/2 or observe[k]-xk[i]>L/2:
            X[k,i]=0
        else:
            X[k,i]=f_ker1(observe[k]-xk[i])/sigma
for k in range(NN,N):
    for i in range(n):
        if observe[k]-xk[i]<-L/2 or observe[k]-xk[i]>L/2:
            X[k,i]=0
        else:
            X[k,i]=f_ker2(observe[k]-xk[i])/sigma
X_T=np.transpose(X)
# Resolving the resolution
M=np.dot(X_T,X)
print(M)
print(np.all(np.linalg.eigvals(M) > 0))
eigvalue,eigvector=eigh(M,eigvals_only=False)
print(eigvalue)
# for i in range(n):
#     if eigvalue[i]<0:
#         eigvalue[i]=0
Q = np.dot(np.dot(eigvector,np.sqrt(np.diag(np.abs(eigvalue)))),np.transpose(eigvector))
print('Q')
print(Q)
sl=Q.sum(axis=1)
Rl=np.transpose(np.transpose(Q)/np.transpose(sl))
# Rl.shape, (n,n)
print(Rl.sum(axis=0))
print(Rl.sum(axis=1))
cl=sl*sl
print('cl')
print(cl)
print(np.dot(Q,Q))
print(np.dot(np.dot(np.transpose(Rl),np.diag(cl)),Rl))

# SVD
[U,D,V]=np.linalg.svd(X,full_matrices=False)
print(D)
Dinv=1/D

value1 = f_image1(observe1)
value2 = f_image2(observe2)
value = np.concatenate((value1, value2), axis=0)
y = value/ sigma
print(value.mean())
a0=np.dot(np.transpose(V),np.dot(np.diag(Dinv),np.dot(np.transpose(U),y)))
atilt0=np.dot(Rl,a0)
amatrix = np.zeros([Ntime,n, n])
atiltmatrix = np.zeros([Ntime,n, n])
for time in range(Ntime):
    value1 = f_image1(observe1)
    value2 = f_image2(observe2)
    value = np.concatenate((value1, value2), axis=0)
    # print('value')
    # print(value1.mean())
    # print(value2.mean())
    value_2=np.zeros(N)
    for i in range(N):
        value_2[i]=(np.random.poisson(3000*value[i]+100,1)-100)/3000
    y = value_2/ sigma
    # print(value.mean())
    a=np.dot(np.transpose(V),np.dot(np.diag(Dinv),np.dot(np.transpose(U),y)))
    atilt=np.dot(Rl,a)
    for i in range(n):
        for j in range(n):
            amatrix[time, i, j] = (a[i] - a0[i]) * (a[j] - a0[j])
            atiltmatrix[time, i, j] = (atilt[i] - atilt0[i]) * (atilt[j] - atilt0[j])

amatrixmean = amatrix.mean(axis=0)
atiltmatrixmean = atiltmatrix.mean(axis=0)
ax = plt.subplot(1, 2, 1)
ax.imshow(amatrixmean, interpolation='nearest')
ax.set_title('a covariance matrix,two kernels,poisson,b=100,amplify=3000,3000 times')
ax = plt.subplot(1, 2, 2)
ax.imshow(atiltmatrixmean, interpolation='nearest')
ax.set_title('a-tilde covariance matrix')

# m=np.dot(X,a)
# # m is samples constructed by the model.
# chi=y-m
# chi_square=(chi*chi).sum()
# print('a')
# print(a)
# print(atilt)
# np.savetxt("result.txt", a,fmt='%.2e')
# np.savetxt("result.txt", atilt,fmt='%.2e')
# print(chi_square)
# print((y*y).sum())
#
# ax=plt.subplot(2,3,1)
# ax.plot(xi,yi,'r',label="raw")
# ax.plot(xk1,ker1,'b',label="kernel1")
# ax.plot(xk2,ker2,'y',label="kernel2")
# ax.plot(xi,image1,'k',label='image1')
# ax.plot(xi,image2,'c',label='image2')
# ax.plot(observe,y,'g+',label='sample')
# ax.plot(observe,m,'m+',label='model')
# ax.legend(loc='upper left')
# ax.xaxis.set_ticks(np.arange(-L/2,L/2+2,2))
# ax.set_title('L=20,N=30,n=101,sig1=0.5,sig2=2,sigma_N=0.003')
#
# nn=np.linspace(1,n,n)
# ax2=plt.subplot(2,3,2)
# ax2.plot(xk,atilt,'r--o',label='atilt')
# ax2.plot(xk,a,'b--o',label='a')
# ax2.legend(loc='upper left')
# ax2.set_title('a,atilde vs n')
# ax2.xaxis.set_ticks(np.arange(-L/2,L/2+2,2))
#
# # resamples
# N2=120
# N2N=int(N2/2)
# observe_2=np.linspace(-L/2,L/2,N2)
# # observe_2= (sobol_rvs(N2,skip=3000)-0.5)*L
# # observe2= (np.random.rand(N2)-0.5)*L
# # f2 = np.concatenate((f_image1(observe_2[0:N2N]),f_image2(observe_2[N2N:N2])),axis=0)
# # print(f2)
# f2 = f_image1(observe_2)
# X2 = np.zeros([N2,n])
# for k in range(N2):
#     for i in range(n):
#         if observe_2[k]-xk[i]<-L/2 or observe_2[k]-xk[i]>L/2:
#             X2[k,i]=0
#         else:
#             X2[k,i]=f_ker1(observe_2[k]-xk[i])/sigma
# # for k in range(N2N,N2):
# #     for i in range(n):
# #         if observe_2[k]-xk[i]<-L/2 or observe_2[k]-xk[i]>L/2:
# #             X2[k,i]=0
# #         else:
# #             X2[k,i]=f_ker2(observe_2[k]-xk[i])/sigma
# # X2.size (N2, n), a.size n*1
# m2=np.dot(X2,a)
# chi2=f2-m2
# chi2_square=(chi2*chi2).sum()
# print(chi2_square)
# print((f2*f2).sum())
# ax3=plt.subplot(2,3,3)
# ax3.plot(observe_2,f2,'b+',label='sample')
# ax3.plot(observe_2,m2,'r--+',label='model')
# ax3.legend(loc='upper right')
# ax3.set_title('resample')
# ax3.xaxis.set_ticks(np.arange(-L/2,L/2+2,2))
#
# # plots
# ax4=plt.subplot(2,3,4)
# ax4.plot(nn,cl,'r+',label='eigenvalue')
# ax4.legend(loc='lower center')
# ax4.set_title('eigenvalue vs n')
#
#
# ax5=plt.subplot(2,3,5)
# ax5.plot(xk,Rl[0],'r--o',label='eigenvector 1')
# ax5.plot(xk,Rl[24],'k--o',label='eigenvector 25')
# ax5.plot(xk,Rl[49],'b--o',label='eigenvector 50')
# ax5.plot(xk,Rl[74],'m--o',label='eigenvector 75')
# ax5.plot(xk,Rl[n-1],'g--o',label='eigenvector 101')
# ax5.set_xlabel('x')
# #ax5.set_ylim([0,0.3])
# ax5.set_title('eigenvector')
# ax5.xaxis.set_ticks(np.arange(-L/2,L/2+2,2))
#
# aa=np.dot(Rl,yi)
# ax6=plt.subplot(2,3,6)
# ax6.plot(xk,atilt,'r--o',label='atilde')
# ax6.plot(xk,aa,'b+',label='aa-tilde')
# # ax6.plot(xk,aa1,'k+',label='aa-25')
# # ax6.plot(xk,aa2,'m+',label='aa-75')
# ax6.legend(loc='upper left')
# ax6.set_xlabel('x')
# ax6.set_ylabel('a')
# ax6.set_title('atilde vs n, L=20')
# ax6.legend(loc='upper left')
# ax6.xaxis.set_ticks(np.arange(-L/2,L/2+2,2))

plt.show()


class Solution:
    def rob(self, nums):
        if not nums:
            return 0
        if len(nums) == 1:
            return nums[0]
        max_one = self.helper(nums[1:])
        max_two = self.helper(nums[0:-1])
        return max(max_one, max_two)

    def helper(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        if not nums:
            return 0

        if len(nums) == 1:
            return nums[0]

        result = [0] * len(nums)
        result[0] = nums[0]
        result[1] = nums[1]

        for i in range(2, len(nums)):
            if i == 2:
                result[i] = nums[i] + nums[i-2]
            else:
                result[i] = max(nums[i]+result[i-2], nums[i]+result[i-3])

        return max(result[-1], result[-2])


def find_parents(x):
    if x==parents[x] :
        return x
    parents[x]=find_parents(parents[x])
    return parents[x]
def union_set(x,y):
    x=find_parents(x)
    y=find_parents(y)
    if x==y : return
    parents[y]=x
    node_count[x]+=node_count[y]

T=int(input())
for TC in range(T):
    parents={}
    node_count={}
    N=int(input())
    for i in range(N):
        A, B=input().split(' ')
        if A not in parents:
            parents[A]=A
            node_count[A]=1
        if B not in parents:
            parents[B]=B
            node_count[B]=1

        if find_parents(A)!=find_parents(B):
            union_set(A,B)
        print(node_count[parents[A]])


#coding:utf-8
from django.shortcuts import render
from .models import ArticleColumn,ArticlePost
from django.core.paginator import Paginator,EmptyPage,PageNotAnInteger  #分页
from django.shortcuts import get_object_or_404
from django.contrib.auth.models import User

from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.http import  HttpResponse

import redis
from django.conf import settings
r= redis.StrictRedis(host=settings.REDIS_HOST,port=settings.REDIS_PORT,db=settings.REDIS_DB)

#根据username决定显示所有文章还是个人名下文章
def article_titles(request,username=None):
    if username:
        user = User.objects.get(username=username)
        article_title = ArticlePost.objects.filter(author=user)
        try:
            userinfo = user.userinfo
        except:
            userinfo = None
    else:
        article_title = ArticlePost.objects.all()

    paginator = Paginator(article_title,8)
    page = request.GET.get('page')#获取GET请求中的page值
    try:
        current_page = paginator.page(page)#得到指定页面内容
        articles = current_page.object_list #得到该页面所有的对象列表
    except PageNotAnInteger:
        current_page = paginator.page(1)#得到指定页面内容
        articles = current_page.object_list #得到该页面所有的对象列表
    except EmptyPage:
        current_page = paginator.page(paginator.num_pages)#得到指定页面内容
        articles = current_page.object_list #得到该页面所有的对象列表

    if username:
        return render(request,"article/list/author_articles.html",{"articles":articles,"page":current_page,"userinfo":userinfo,"user":user})
    else:
        return render(request,"article/list/article_titles.html",{"articles":articles,"page":current_page})

#文章详情
def article_detail(request,id,slug):
    article =get_object_or_404(ArticlePost,id=id,slug=slug)
    total_views = r.incr("article.{}.views".format(article.id))
    print("article.id:%s" %article.id)
    r.zincrby('article_ranking', 1, article.id)#文章被访问一次，article_rankig就将该文章的id值加1

    article_ranking = r.zrange('article_ranking',0,-1,desc=True)[:10]
    print("article_ranking:%s" %article_ranking)
    article_ranking_ids = [int(id) for id in article_ranking]
    print("article_ranking_ids:%s" %article_ranking_ids)
    most_viewed = list(ArticlePost.objects.filter(id__in=article_ranking_ids))
    most_viewed.sort(key=lambda x:article_ranking_ids.index(x.id))

    return render(request,"article/list/article_detail.html",{"article":article,"total_views":total_views,"most_viewed":most_viewed})

#文章点赞
@csrf_exempt
@require_POST
@login_required(login_url='/account/login/')
def like_article(request):
    article_id = request.POST.get("id")
    action = request.POST.get("action")
    if article_id and action:
        try:
            article = ArticlePost.objects.get(id=article_id)
            if action == "like":   #点赞
                article.users_like.add(request.user)#该用户点赞了该文章
                return HttpResponse("1")
            else:#踩你
                article.users_like.remove(request.user)
                return HttpResponse("2")
        except:
            return HttpResponse("no")


# -*- coding:utf-8 -*-
'''
Created on 2016年3月31日

@author: baoguodong.kevin
'''
from naas.db.sqlalchemy import session as sa
from naas.db.object.models import Endpoint
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class EndpointDao():

    def getEndpointById(self,endpoint_id,session=None):
        try:
            if session is None:
                session=sa.get_session()
            endpoint=session.query(Endpoint).filter(Endpoint.endpoint_id==endpoint_id).first()
            return endpoint
        except Exception as e:
            session.close()
            LOG.error(str(e))
            raise e


""" Języki skryptowe - Cwiczenia - Rafał Stepkowski GD30982 - alfabet - zajecia 2
Wydrukuj alfabet w formacie: AaBbCcDd….., a następnie w formacie:
aAbBcCdD….."""

#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import time

print("Alfabet w porządku duża litera, mała litera:")
x = 0
for i in range(65, 91):
    litera = chr(i)
    x += 1
    tmp = litera + litera.lower()
    if i > 65 and x % 5 == 0:
        x = 0

    print(tmp)

x = -1
print("\nAlfabet w porządku małą litera, duża litera:")
for i in range(65, 91):
    litera = chr(i)
    x += 1
    tmp = litera.lower() + litera.upper()
    if i > 65 and x % 5 == 0:
        x = 0
        tmp += "\n"
    print(tmp, end=" ")

time.sleep(60)


from xai.brain.wordbase.nouns._ellipsis import _ELLIPSIS

#calss header
class _ELLIPSES(_ELLIPSIS, ):
	def __init__(self,):
		_ELLIPSIS.__init__(self)
		self.name = "ELLIPSES"
		self.specie = 'nouns'
		self.basic = "ellipsis"
		self.jsondata = {}


import numpy as np

from src import params
from src.emm import Emm
from src.score_calc.base_score_calc import BaseScoreCalc
from src.utils.circular_buffer import CircularBuffer

class LogsumScoreCalc(BaseScoreCalc):

    def __init__(self, max_score=1000) -> None:
        super().__init__()
        self.threshold = params.LOGSUM_P_THR
        self.max_score = max_score
        self.next_threshold = 0

    @staticmethod
    def name():
        return 'LogSum'

    def score(self, q: CircularBuffer, emm: Emm):
        for k in range(q.size - 1):
            m = emm.get_pr(q[k], q[k + 1])
            self.pr[k] = -np.log(m) if m != 0 else self.max_score
        return np.mean(self.pr)

    def is_anomaly_score(self, score):
        return score > self.threshold

    def update_threshold(self):
        p_thr_new = self.next_threshold
        p_thr_new += self.max_score * params.THRESHOLD_TOLERANCE
        if p_thr_new > self.max_score:
            p_thr_new = self.max_score
        self.threshold = p_thr_new
        self.next_threshold = 0

    def update_next_threshold(self, score):
        if score > self.next_threshold:
            self.next_threshold = score

    def clone(self):
        score_calc = LogsumScoreCalc(self.max_score)
        score_calc.threshold = self.threshold
        return score_calc


#!/usr/bin/env python3

'''
This class takes the environment and represents it in a way that can be
specified by the user.  If no specification is given, the representation
will be equivalent to the os.environ dictionary.

The user
'''

import os
import sys
import json
import subprocess
from pprint import pprint

def make_decorator(dictionary):
    '''
    Creates a decorator that adds the function to a dictionary under the keys
    listed in args

    '''
    class env_decorator:
        def __init__(self, args):
            self.args = args
        def __call__(self, f):
            for var in self.args:
                dictionary[var] = f
            return f
    return env_decorator

# Functions taking string values and returning string, lists or dictionaries
# Decorating a function with this decorator with argument list_of_vars
# will register that function as the parser for the values of the variables in
# the list list_of_vars
parsers = {}
parses = make_decorator(parsers)

# Functions taking variable name and value and returning string
stringizers= {}
stringizes = make_decorator(stringizers)

# Functions taking variable name and value and returning string
pretty_stringizers = {}
pretty_stringizes = make_decorator(pretty_stringizers)

''' Dictionnary of comparison functions '''
# Functions taking object before and object after and returning a string
comparers = {}
compares = make_decorator(comparers)

updaters = {}
updates = make_decorator(updaters)

class EnvWrapper:
    ''' Class that encapsulates a dictionnary of environment variables
    The keys are variable names and the values are the parsed string values
    of the environment variables as defined by the 'processor' functions '''
    def __init__(self, d=None, representation=None):
        ''' Create an instance from an already made dictionary or from the
        environment dictionary from os.environ. '''
        if representation:
            self.env=representation
        elif d:
            self.env=self.decode_environment_dict(d)
        else:
            self.env = self.decode_environment_dict(os.environ)

    def __getitem__(self, key):
        return self.env[key]

    def __iter__(self):
        return iter(sorted(self.env))

    def __str__(self):
        return str(self.env)

    def get_str(self, key):
        '''Returns a string representing the environment variable. This string may or
        may not be equal to the string value of the variable using function
        registered as the 'stringizer' for that vaiable

        '''
        if key in self.env:
            if key in stringizers:
                return key + '=' + stringizers[key](self.env[key])
            else:
                return key + '=' + str(self.env[key])
        else:
            return key + ' is not in environment'

    def get_pretty_str(self, key):
        ''' Get a pretty representation of the variable '''
        if key in self.env:
            if key in pretty_stringizers:
                return key + '=\n' + pretty_stringizers[key](self.env[key])
            else:
                return key + '=' + str(self.env[key])
        else:
            return key + ' is not in environment'

    def json_dumps(self):
        ''' Dump the dictionary of variabl and their parsed values '''
        return json.dumps(self.env)

    def pretty(self):
        ''' Return a string formed by all the pretty printed variables '''
        return '\n'.join(self.get_pretty_str(key) for key in self)

    def to_file(self, filename):
        with open(filename, 'w') as f:
            json_dump(f, self.env, indent=4)

    @staticmethod
    def decode_environment_dict(d):
        ''' Transform the os.environ dictionary to the format that I use:
        Each variable can have a function that parsed the string value into a
        list or dictionary or what ever else you want. '''
        representation = {}
        for var, value in d.items():
            if var in parsers.keys():
                representation[var] = parsers[var](value)
            else:
                representation[var] = d[var]
        return representation

    @classmethod
    def from_environment_dict(cls, d=None):
        if not d:
            d = os.environ
        return cls(representation=cls.decode_environment_dict(d))

    @classmethod
    def from_file(cls, filename):
        with open(filename, 'r') as f:
            representation = json.load(f)
        return cls(representation=representation)

    def get_declaration(self, var):
        real_value = self.get_str
        if var in stringizers:
            real_value = stringizers[var](self.env[var])
        else:
            real_value = str(self.env[var])
        return f'{var}="{real_value}"'

    def get_unsetting(self, var):
        return f'unset {var}'

    def get_change(self, var, before, after):
        if isinstance(before, list):
            if after:
                new_elements = set(after) - set(before)
                return f"{var}=\"${var}:{':'.join(new_elements)}\""
            else:
                return f'{var}=""'
        elif var in stringizers:
            return f'{var}="{stringizers[var](after)}"'
        else:
            return f'{var}="{after}"'

'''
================================================================================
Definitions of the processing and string functions
For any variable, you can define a function that parses it (taking a string
to any type of object)
For any variable, you can define a function that will take a variable name and a
ivalue and return a string.
Same thing for the pretty_stringizes
================================================================================
'''
''' Dictionaries with accompanying decorators used to register the functions
that process variables from string values and puts them back as strings in a
pretty way or in a normal way'''

def get_command(args):
    env = EnvWrapper.from_environment_dict()
    if args.posargs:
        for var in args.posargs:
            print(env.get_pretty_str(var))
    else:
        print(env.pretty())


"""
In order to fully utilize multiple cores to do emulations or data processing, this script use process pool to do all the commands in parallel
You should copy this file and rename it, just like what you do with lua scripts
"""

# YOUR CODE BEGINS HERE
LUA_EXEC_TIMES = 300
PROCESS_COUNT = 15
SCRIPT_FILE = "200206_32kbps_emulator_ecc_8kbps.lua"  # your lua script filename
# def SCRIPT_INLINE(evaluate_idx, evaluate_cnt):  # your inline lua script
    # return "evaluate_idx = %d; evaluate_cnt = %d" % (evaluate_idx, evaluate_cnt)
def SCRIPT_INLINE(ecc_bytes):  # your inline lua script
    return "ecc_per_block = %d" % (ecc_bytes)

def jobs():  # register jobs to finish
    # for evaluate_idx in range(1):
    for ecc_bytes in [2]:
        for evaluate_idx in range(LUA_EXEC_TIMES):
            job = Job()
            job.output_filename = "job_8kbps_%d_%d.txt" % (ecc_bytes, evaluate_idx)
            # job.script_inline = SCRIPT_INLINE(evaluate_idx, 10)
            job.script_inline = SCRIPT_INLINE(ecc_bytes)
            job.register()

# YOUR CODE ENDS HERE

import multiprocessing, subprocess, os, sys, signal
job_list = []
class Job:
    def __init__(self):
        self.output_filename = None
        self.script_inline = None
    def register(self):
        assert self.output_filename is not None
        assert self.script_inline is not None
        global job_list
        self.job_idx = len(job_list)
        job_list.append(self)
def signal_handler(signum, frame):
    sys.exit(0)
def process(job):
    global job_list
    global SCRIPT_FILE
    # signal.signal(signal.SIGINT, signal_handler)
    outfile = open(job.output_filename, 'w')
    print("job", job.job_idx, "running")
    try:
        sid_assigned = "%04d" % job.job_idx
        subprocess.run(["python", "../WebGUI/lua/execlua_new_process.py", SCRIPT_FILE, job.script_inline, sid_assigned], stdout=outfile)
    except KeyboardInterrupt:
        print("job", job.job_idx, "exited")
    else:
        print("job", job.job_idx, "finished")
if __name__ == "__main__":
    jobs()
    original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
    pool = multiprocessing.Pool(processes=PROCESS_COUNT)
    signal.signal(signal.SIGINT, original_sigint_handler)
    try:
        pool.map(process, job_list)
    except KeyboardInterrupt:
        print("Caught KeyboardInterrupt, terminating workers")
        pool.terminate()
    else:
        pool.close()
    pool.close()
    pool.join()


# Create your views here.
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse

from upload.models import Document
from upload.forms import DocumentForm

def list(request):
	if request.method == 'POST':
		form = DocumentForm(request.POST, request.FILES)
		if form.is_valid():
			form.save()

			return HttpResponseRedirect(reverse('upload.views.list'))
	else:
		form = DocumentForm()

	documents = Document.objects.all()

	return render_to_response(
		'upload/list.html',
		{'documents': documents, 'form': form},
		context_instance=RequestContext(request)
	)


# -*- coding: utf8 -*-
# T-Rax - GUI program for analysis of spectroscopy data during
# diamond anvil cell experiments
# Copyright (C) 2016 Clemens Prescher (clemens.prescher@gmail.com)
# Institute for Geology and Mineralogy, University of Cologne
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from qtpy import QtWidgets, QtCore

import pyqtgraph as pg

from .BaseWidget import BaseWidget

class RamanWidget(BaseWidget, object):
    def __init__(self, parent):
        super(RamanWidget, self).__init__(parent)

        self.display_mode_gb = DisplayModeGroupBox()
        self.add_control_widget(self.display_mode_gb)

        self.create_raman_shortcuts()

    def create_raman_shortcuts(self):
        self.laser_line_txt = self.display_mode_gb._laser_line_txt
        self.reverse_cm_cb = self.display_mode_gb._reverse_cm_cb
        self.nanometer_cb = self.display_mode_gb._nanometer_cb

class DisplayModeGroupBox(QtWidgets.QGroupBox):
    def __init__(self, title='Options'):
        super(DisplayModeGroupBox, self).__init__(title)

        self.create_widgets()
        self.create_layout()
        self.style_widgets()

        self._reverse_cm_cb.setChecked(True)

    def create_widgets(self):
        self._laser_line_lbl = QtWidgets.QLabel('Laser Line:')
        self._laser_line_txt = QtWidgets.QLineEdit('532')
        self._laser_lint_unit_lbl = QtWidgets.QLabel('nm')

        self._mode_lbl = QtWidgets.QLabel('Unit:')
        self._reverse_cm_cb = QtWidgets.QRadioButton('cm-1')
        self._nanometer_cb = QtWidgets.QRadioButton('nm')

    def create_layout(self):
        self._laser_line_layout = QtWidgets.QHBoxLayout()
        self._laser_line_layout.addWidget(self._laser_line_txt)
        self._laser_line_layout.addWidget(self._laser_lint_unit_lbl)

        self._mode_layout = QtWidgets.QHBoxLayout()
        self._reverse_cm_layout = QtWidgets.QHBoxLayout()
        self._reverse_cm_layout.setSpacing(1)
        self._reverse_cm_layout.addWidget(self._reverse_cm_cb)
        self._mode_layout.addLayout(self._reverse_cm_layout)
        self._mode_layout.addWidget(self._nanometer_cb)

        self._layout = QtWidgets.QGridLayout()
        self._layout.addWidget(self._laser_line_lbl, 0, 0)
        self._layout.addLayout(self._laser_line_layout, 0, 1)
        self._layout.addWidget(self._mode_lbl, 1, 0)
        self._layout.addLayout(self._mode_layout, 1, 1)

        self.setLayout(self._layout)

    def style_widgets(self):
        self._laser_line_txt.setMaximumWidth(50)

        align = QtCore.Qt.AlignVCenter | QtCore.Qt.AlignRight
        self._laser_line_lbl.setAlignment(align)
        self._laser_line_txt.setAlignment(align)
        self._mode_lbl.setAlignment(align)

if __name__ == '__main__':
    app = QtWidgets.QApplication([])
    widget = RamanWidget(parent=None)
    widget.show()
    app.exec_()


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import rl.callbacks

class EpisodeLogger(rl.callbacks.Callback):
    def __init__(self):
        self.observations = {}
        self.rewards = {}
        self.actions = {}

    def on_episode_begin(self, episode, logs):
        self.observations[episode] = []
        self.rewards[episode] = []
        self.actions[episode] = []

    def on_step_end(self, step, logs):
        episode = logs['episode']
        self.observations[episode].append(logs['observation'])
        self.rewards[episode].append(logs['reward'])
        self.actions[episode].append(logs['action'])


"""
Solution for 1145. Binary Tree Coloring Game
https://leetcode.com/problems/binary-tree-coloring-game/
"""
# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None
from collections import deque

class Solution:
    """
    Runtime: 36 ms, faster than 100.00% of Python3 online submissions for Binary Tree Coloring Game.
    Memory Usage: 14.1 MB, less than 100.00% of Python3 online submissions for Binary Tree Coloring Game.
    """
    def btreeGameWinningMove(self, root, n, x):
        """
        Two players play a turn based game on a binary tree.  We are given the root of this binary tree, and the number of nodes n in the tree.  n is odd, and each node has a distinct value from 1 to n.

        Initially, the first player names a value x with 1 <= x <= n, and the second player names a value y with 1 <= y <= n and y != x.  The first player colors the node with value x red, and the second player colors the node with value y blue.

        Then, the players take turns starting with the first player.  In each turn, that player chooses a node of their color (red if player 1, blue if player 2) and colors an uncolored neighbor of the chosen node (either the left child, right child, or parent of the chosen node.)

        If (and only if) a player cannot choose such a node in this way, they must pass their turn.  If both players pass their turn, the game ends, and the winner is the player that colored more nodes.

        You are the second player.  If it is possible to choose such a y to ensure you win the game, return true.  If it is not possible, return false.

        Example 1:

        Input: root = [1,2,3,4,5,6,7,8,9,10,11], n = 11, x = 3
        Output: true
        Explanation: The second player can choose the node with value 2.

        Constraints:

        root is the root of a binary tree with n nodes and distinct node values from 1 to n.
        n is odd.
        1 <= x <= n <= 100
        Args:
            root: TreeNode as the root
            n: int value
            x: int value

        Returns:
            bool:
        """
        def count_nodes(node):
            """
            Counts # of nodes
            Args:
                node: TreeNode

            Returns:
                int: # of nodes
            """
            if not node:
                return 0
            return count_nodes(node.left) + count_nodes(node.right) + 1

        queue = deque([root])
        while queue:
            temp_queue = deque([])
            while queue:
                node = queue.popleft()
                if node.val == x:
                    left_count = count_nodes(node.left)
                    right_count = count_nodes(node.right)
                    parent_count = n - left_count - right_count - 1

                    if left_count > right_count + parent_count + 1:
                        return True
                    elif right_count > left_count + parent_count + 1:
                        return True
                    elif parent_count > left_count + right_count + 1:
                        return True
                    else:
                        return False

                if node.left:
                    temp_queue.append(node.left)
                if node.right:
                    temp_queue.append(node.right)
            queue = temp_queue


#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com
import logging
import importlib

from thumbor.storages import BaseStorage

class Storage(BaseStorage):
    """Uses each storage option, in order"""

    def __init__(self, context):
        BaseStorage.__init__(self, context)

        self.storages = []
        for modpath in context.config.FAILOVER_STORAGE_OPTIONS:
            logging.debug("importing " + modpath)
            mod = importlib.import_module(modpath)
            self.storages.append(mod.Storage(context))

    def put(self, path, bytes):
        for s in self.storages:
            s.put(path, bytes)
        return path

    def put_crypto(self, path):
        for s in self.storages:
            s.put_crypto(path)
        return path

    def put_detector_data(self, path, data):
        for s in self.storages:
            s.put_detector_data(path, path, data)
        return path

    def get_crypto(self, path):
        for i, s in enumerate(self.storages):
            result = s.get_crypto(path)
            if result:
                for s2 in self.storages[0:i]:
                    s2.put_crypto(path)
                return result
        return None

    def get_detector_data(self, path):
        for i, s in enumerate(self.storages):
            result = s.get_detector_data(path)
            if result:
                for s2 in self.storages[0:i]:
                    s2.put_detector_data(path, result)
                return result
        return None

    def get(self, path):
        for i, s in enumerate(self.storages):
            result = s.get(path)
            if result:
                for s2 in self.storages[0:i]:
                    s2.put_detector_data(path, result)
                return result
        return None

    def exists(self, path):
        for s in self.storages:
            result = s.exists(path)
            if result:
                return True
        return False

    def remove(self, path):
        for s in self.storages:
            s.remove(path)