import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime as dt
from datetime import timedelta as td
from datetime import timezone
import pandas as pd
from glob import glob
from skyfield.api import load, Topos, utc, Star
from skyfield.almanac import find_discrete, risings_and_settings
from scipy import signal
from scipy.optimize import curve_fit
from matplotlib.colors import LogNorm
from astropy import units as u
from astropy.coordinates import SkyCoord, EarthLocation
from astropy.time import Time
import pytz
from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)

#%matplotlib

##############################################################################
## Function definitions
#############################

## COSQM Data load function, returns data and dates in tuple
def LoadData(path,cache={}):
	#print (path)
	if path in cache:
		return cache[path]
	else:
		try:
			data = pd.read_csv(path, delimiter=' ', names=['Date', 'Time', 'Lat', 'Lon', 'Alt', 'Temp', 'Wait', 'Sqm0', 'Sqm1', 'Sqm2', 'Sqm3', 'Sqm4', 'Sbcals0', 'Sbcals1', 'Sbcals2', 'Sbcals3', 'Sbcals4'])
			data['Datetime'] = pd.to_datetime(data['Date']+'T'+data['Time'], utc=True)
			cache[path] = data
			return data
		except:
			print('********error*********', path)

##COSQM TEIDE data load function ***BUG IN COSQM DATA FROM WGET COMMAND***
def LoadDataCorrupt(path,cache={}):
	print (path)
	if path in cache:
		return cache[path]
	else:
		try:
			data_server = np.array(pd.read_csv(path,sep=' ',dtype=None, error_bad_lines=False))
			data=data_server[:,2:].astype(float)
			dates_str=data_server[:,0:2].astype(str)
			dates = np.array([ dt.strptime( dates+times, '%Y-%m-%d%H:%M:%S' ).timestamp() for dates, times in dates_str ])
			print('data', np.shape(data), 'dates', np.shape(dates))
			dates1=dates.reshape(len(dates),1)
			output = np.concatenate((data,dates1),axis=1)
			cache[path] = (data,dates)
			out=(data,dates)
			output=np.concatenate((out[0],out[1].reshape(np.shape(out[1])[0],1)),axis=1)
			return output
		except:
			print('********error*********', path, '\n')

## Cloud screening function
def Cloudslidingwindow(data_array, window_size, threshold):
	da=np.copy(data_array)
	dd = np.lib.stride_tricks.sliding_window_view(data_array, window_shape = window_size, axis=0).copy() #start and end values lost. size of array is data_array.shape[0]-2
	diffs = np.sum(np.abs(np.diff(dd, axis=2)), axis=2)/(window_size-1)
	padd = np.full([1, da.shape[1]], np.nan)
	for i in range(window_size//2):
		diffs = np.insert(diffs, 0, padd, axis=0)                     #append nan to start to get same shape as input
		diffs = np.insert(diffs, diffs.shape[0], padd, axis=0)            #append nan to end to get same shape as input
	da[diffs[:,0]>threshold/(window_size-1)] = np.nan
	return da

## Moon presence data reduction function and variables
ts = load.timescale()
planets = load('de421.bsp')
earth,moon,sun = planets['earth'],planets['moon'],planets['sun']
eph = load('de421.bsp')

## Define variables for locations
izana_loc = earth + Topos('28.300398N', '16.512252W')
santa_loc = earth + Topos('28.472412500N', '16.247361500W')

## !! Time objects from dt need a timezone (aware dt object) to work with skyfield lib
## ex.: time = dt.now(timezone.utc), with an import of datetime.timezone
def RaDecGal(dt_array, earth_location):
	sid_lon = Time(dt_array, location=earth_location).sidereal_time('mean')
	gal_coords = SkyCoord(sid_lon, loc_lat*u.degree, frame='icrs').galactic
	radec_gal = np.array((gal_coords.l.deg, gal_coords.b.deg)).T
	return radec_gal

def ObjectAngle(dt_array, object, location):
	t = ts.utc(dt_array)
	astrometric = location.at(t).observe(object)
	alt, _, _ = astrometric.apparent().altaz()
	return alt.degrees

##############################################################################
# COSQM DATA
#
# Data taken from Martin Aubé's server: http://dome.obsand.org:2080/DATA/CoSQM-Network/
#
# Data format: Date(YYYY-MM-DD), time(HH:MM:SS),
#             Latitude, Longitude, Elevation, SQM temperature, integration time,
#             C(mag), R(mag), G(mag), B(mag), Y(mag),
#             C(watt),R(watt),G(watt),B(watt),Y(watt)
#############################

#######
## Variables
path_cosqm='cosqm_santa/data/'
path_aod = 'cosqm_santa/20190601_20210131_Santa_Cruz_Tenerife.lev10'
cosqm_bands = np.array([652, 599, 532, 588])
bands = ['clear', 'red', 'green', 'blue', 'yellow']

loc = santa_loc
loc_lat = 28.472412500
loc_lon = -16.247361500
eloc = EarthLocation(lat=loc_lat, lon=loc_lon)
loc_str = 'santa'
slide_threshold = 0.1
slide_window_size = 5
mw_min_angle = 30
moon_min_angle = -2
sun_min_angle = -18
cosqm_window_size = 7

cosqm_mean_count = 1        #number of averages on cosqm data for correlation
cosqm_min_am_hour = 2       #start of dusk cosqm measurements for correlation
cosqm_max_pm_hour = 25      #end of dawn cosqm measurements for correlation
aod_mean_count = 5          #number of averages on aeronet aod data for correlation
aod_max_am_hour = 10        #end of dusk aeronet aod data for correlation
aod_min_pm_hour = 14        #start of dawn aeronet aod data for correlation

normalized_cosqm_znsb_threshold = 1.8
cosqm_aod_threshold = 0.04
ae_min = 0
ae_max = 2

#######

## find all paths of files in root directory
print('Load cosqm data')
paths_cosqm = sorted(glob(path_cosqm+"*/*/*.txt"))
files = pd.concat([LoadData(path) for path in paths_cosqm], ignore_index=True)
cosqm_santa = files[['Sqm0', 'Sqm1', 'Sqm2', 'Sqm3', 'Sqm4']].values
dt_santa_raw = files['Datetime']

#remove non datetime errors in cosqm files (NaT)
cosqm_santa = cosqm_santa[~pd.isnull(dt_santa_raw)]
dt_santa = dt_santa_raw[~pd.isnull(dt_santa_raw)]

## Remove zeros from cosqm measurements (bugs from instruments)
print('Cleaning: remove zeros from cosqm measurements')
zeros_mask = (cosqm_santa!=0).all(1)
cosqm_santa = cosqm_santa[zeros_mask]
dt_santa = dt_santa[zeros_mask]

#Clouds sliding window filter
print('Filter: clouds sliding window filter')
cosqm_santa_cloud = Cloudslidingwindow(cosqm_santa, slide_window_size, slide_threshold)
dt_santa_cloud = dt_santa[~np.isnan(cosqm_santa_cloud[:,0])]
cosqm_santa_cloud = cosqm_santa_cloud[~np.isnan(cosqm_santa_cloud[:,0])]

#milky way filter
print('Filter: milky way angles calculation')
mw_angles = RaDecGal(dt_santa_cloud, eloc)
mw_mask = mw_angles[:,1]>mw_min_angle
cosqm_santa_mw = cosqm_santa_cloud[mw_mask]
dt_santa_mw = dt_santa_cloud[mw_mask].reset_index(drop=True)

## Compute moon angles for each timestamp in COSQM data
print('Filter: moon angles calculation')
moon_angles = ObjectAngle(dt_santa_mw, moon, loc)
moon_mask = moon_angles<moon_min_angle
cosqm_santa_moon = cosqm_santa_mw[moon_mask]
dt_santa_moon = dt_santa_mw[moon_mask].reset_index(drop=True)

## Compute sun angles for each timestamp in COSQM data
print('Filter: sun_angles calculation')
sun_angles = ObjectAngle(dt_santa_moon, sun, santa_loc)
sun_mask = sun_angles<sun_min_angle
cosqm_santa_sun = cosqm_santa_moon[sun_mask]
dt_santa_sun = dt_santa_moon[sun_mask].reset_index(drop=True)

# plot filtering
band = 3
plt.figure(figsize=[7,4], dpi=150)
plt.scatter(dt_santa, cosqm_santa[:,band], s=10, label='cosqm_santa')
plt.scatter(dt_santa_mw, cosqm_santa_mw[:,band], s=10, alpha=0.5, label='milky way below '+str(mw_min_angle))
plt.scatter(dt_santa_moon, cosqm_santa_moon[:,band], s=8, alpha=0.5, label='moon below '+str(moon_min_angle))
plt.scatter(dt_santa_sun, cosqm_santa_sun[:,band], s=6, label='sun below '+str(sun_min_angle), c='k')
plt.legend(loc=[0,0])
plt.ylabel(f'CoSQM {cosqm_bands[band-1]}nm magnitude (mag)')

#hist2d#
#raw without clouds
hours_float_raw = np.array([ date.hour+                    #WATCH OUT FOR TIMEZONE HERE!
	date.minute/60+
	date.second/3600 for date in dt_santa ])
hours_float_raw[hours_float_raw>12]-=24

# plt.figure(figsize=[12,8])
# plt.hist2d(hours_float_raw, cosqm_santa[:,1], 200, cmap='inferno')
# plt.hist2d(hours_float_raw, cosqm_santa[:,1], 200, cmap='inferno', norm=LogNorm())
# plt.ylim(15,21)
# plt.title('ZNSB - no filter - clear')
# plt.xlabel('hour')
# plt.ylabel('CoSQM magnitude')

#clouds filter
hours_float_diff = np.array([ date.hour+                    #WATCH OUT FOR TIMEZONE HERE!
	date.minute/60+
	date.second/3600 for date in dt_santa ])
hours_float_diff[hours_float_diff>12]-=24

#plt.figure(figsize=[12,8])
#plt.hist2d(hours_float_diff[np.isfinite(cosqm_santa_diff)[:,0]], cosqm_santa_diff[:,0][np.isfinite(cosqm_santa_diff)[:,0]], 200, cmap='inferno')
#plt.hist2d(hours_float_diff[np.isfinite(cosqm_santa_diff)[:,0]], cosqm_santa_diff[:,0][np.isfinite(cosqm_santa_diff)[:,0]], 200, cmap='inferno', norm=LogNorm())
#plt.ylim(15,21)
#plt.title('ZNSB - filter: clouds+variance - clear')
#plt.xlabel('hour')
#plt.ylabel('CoSQM magnitude')

#moon filter
hours_float_moon = np.array([ date.hour+                    #WATCH OUT FOR TIMEZONE HERE!
	date.minute/60+
	date.second/3600 for date in dt_santa_moon ])
hours_float_moon[hours_float_moon>12]-=24

# plt.figure(figsize=[12,8])
# plt.hist2d(hours_float_moon[np.isfinite(cosqm_santa_moon)[:,0]], cosqm_santa_moon[:,0][np.isfinite(cosqm_santa_moon)[:,0]], 200, cmap='inferno')
# plt.hist2d(hours_float_moon[np.isfinite(cosqm_santa_moon)[:,0]], cosqm_santa_moon[:,0][np.isfinite(cosqm_santa_moon)[:,0]], 200, cmap='inferno', norm=LogNorm())
# plt.ylim(15,21)
# plt.title('ZNSB - filter: moon - clear')
# plt.xlabel('hour')
# plt.ylabel('CoSQM magnitude')

#sun filter
hours_float_sun = np.array([ date.hour+                    #WATCH OUT FOR TIMEZONE HERE!
	date.minute/60+
	date.second/3600 for date in dt_santa_sun ])
hours_float_sun[hours_float_sun>12]-=24

# plt.figure(figsize=[12,8])
# plt.hist2d(hours_float_sun[np.isfinite(cosqm_santa_sun)[:,1]], cosqm_santa_sun[:,1][np.isfinite(cosqm_santa_sun)[:,1]], 200, cmap='inferno')
# plt.hist2d(hours_float_sun[np.isfinite(cosqm_santa_sun)[:,1]], cosqm_santa_sun[:,1][np.isfinite(cosqm_santa_sun)[:,1]], 200, cmap='inferno', norm=LogNorm())
# plt.ylim(15,21)
# plt.title('ZNSB - filter: sun - clear')
# plt.xlabel('hour')
# plt.ylabel('CoSQM magnitude')

dts_event = np.array([dt_santa.dt.date, dt_santa_cloud.dt.date, dt_santa_mw.dt.date, dt_santa_moon.dt.date, dt_santa_sun.dt.date])

fsize=10
sizem = 1
fig,ax = plt.subplots(figsize=(7,4), dpi=150)
ax.scatter(np.unique(dt_santa), 5*np.ones(np.unique(dt_santa).shape[0]), label='Raw', s=sizem, marker='s', alpha=0.5)
ax.scatter(np.unique(dt_santa_cloud), 4*np.ones(np.unique(dt_santa_cloud).shape[0]), label='Clouds', s=sizem, marker='s', alpha=0.5)
ax.scatter(np.unique(dt_santa_mw), 3*np.ones(np.unique(dt_santa_mw).shape[0]), label='Milky Way', s=sizem, marker='s', alpha=0.5)
ax.scatter(np.unique(dt_santa_moon), 2*np.ones(np.unique(dt_santa_moon).shape[0]), label='Moon', s=sizem, marker='s', alpha=0.5)
ax.scatter(np.unique(dt_santa_sun), np.ones(np.unique(dt_santa_sun).shape[0]), label='Sun', s=sizem, marker='s', alpha=0.5)
ax.set_yticks([])
ax.set_xticks([pd.Timestamp('2019-08-27'), pd.Timestamp('2020-05-01'), pd.Timestamp('2021-01-04')])
ax.set_xticklabels(['2019-09', '2020-05', '2021-01'])
ax.xaxis.set_minor_locator(MultipleLocator(31))
ax.xaxis.grid(ls='--', lw='0.5', which='both')
ax.set_ylim(0.8, 5.5)
ax.text(pd.Timestamp('2019-09-01'), 5+0.2, 'Raw', fontsize=fsize)
ax.text(pd.Timestamp('2019-09-01'), 4+0.2, 'Clouds removed', fontsize=fsize)
ax.text(pd.Timestamp('2019-09-01'), 3+0.2, 'Milky Way removed', fontsize=fsize)
ax.text(pd.Timestamp('2019-09-01'), 2+0.2, 'Moon removed', fontsize=fsize)
ax.text(pd.Timestamp('2019-09-01'), 1+0.2, 'Sun removed', fontsize=fsize)
plt.tight_layout()
#plt.savefig('figures/filtering_eventplot.png')

# # Make eventplot to graph filtering of dates
# lineoffsets1 = np.array([3, 6, 9, 12, 15])
# linelengths1 = np.ones(5)*2

# sizem = 2
# fsize = 7
# dts = np.array([pd.Timestamp(date).timestamp() for date in dt_santa.dt.date])
# dts_cloud = np.copy(dts)
# dts_cloud[np.isnan(cosqm_santa_cloud[:,0])] = False
# dts_mw = np.copy(dts)
# dts_mw[~mw_mask] = np.nan
# dts_moon = np.copy(dts)
# dts_moon[mw_mask][~moon_mask] = np.nan
# dts_sun = np.copy(dts)
# dts_sun[mw_mask][moon_mask][~sun_mask] = np.nan

# dts_event = np.vstack((dts,dts_cloud,dts_mw,dts_moon,dts_sun))
# fig, ax = plt.subplots(figsize=(12,8), dpi=150)
# ax.eventplot(dts_event, colors=['r', 'g', 'b', 'y', 'k'], orientation='horizontal', lineoffsets=lineoffsets1, linelengths=linelengths1, linewidths=0.1)
# ax.set_xlim(pd.Timestamp('2019-08-27').timestamp()-3600*24, pd.Timestamp('2021-01-04').timestamp()+3600*24)
# ax.set_ylim(0,20)
# ax.tick_params(left=False, labelleft = False )
# ax.set_xticks([pd.Timestamp('2019-08-27').timestamp(), pd.Timestamp('2020-05-01').timestamp(), pd.Timestamp('2021-01-04').timestamp()])
# ax.set_xticklabels(['2019-09', '2020-05', '2021-01'])
# #ax.legend(loc=(pd.Timestamp('2019-08-27').timestamp(), 3.5), prop={'size': 6})
# ax.set_ylim(0.9,5.5)
# #ax.xaxis.set_minor_locator(MultipleLocator(31*24*1627660252))
# ax.xaxis.grid(linestyle = '--', linewidth = 0.5, which='both')
# ax.text(pd.Timestamp('2019-08-27').timestamp(), 5+0.2, 'Raw', fontsize=fsize )
# ax.text(pd.Timestamp('2019-08-27').timestamp(), 4+0.2, 'Clouds removed', fontsize=fsize )
# ax.text(pd.Timestamp('2019-08-27').timestamp(), 3+0.2, 'Milky Way removed', fontsize=fsize )
# ax.text(pd.Timestamp('2019-08-27').timestamp(), 2+0.2, 'Moon removed', fontsize=fsize )
# ax.text(pd.Timestamp('2019-08-27').timestamp(), 1+0.2, 'Sun removed', fontsize=fsize )
# plt.savefig('figures/filtering_eventplot.png')

## set to nan values that are color higher than clear by visual analysis, followed by clouded nights by visual analysis
print('Remove invalid cosqm data (R channel higher than C), and visual cloud screening')
dates_color_str = np.array(['2019-06-12', '2019-06-13', '2019-06-26',
					'2019-07-01', '2019-07-02', '2019-07-21', '2019-07-23', '2019-07-24', '2019-07-25', '2019-07-26', '2019-07-27', '2019-07-29',
					'2019-08-03', '2019-08-04', '2019-08-07', '2019-08-08', '2019-08-09', '2019-08-10', '2019-08-12', '2019-08-13', '2019-08-18', '2019-08-19', '2019-08-25','2019-08-26', '2019-08-30', '2019-08-31',
					'2019-09-06', '2019-09-23', '2019-09-24', '2019-09-25', '2019-09-26', '2019-09-27', '2019-09-28', '2019-09-29', '2019-09-30',
					'2019-10-01', '2019-10-02', '2019-10-03', '2019-10-04', '2019-10-05', '2019-10-07', '2019-10-25', '2019-10-26', '2019-10-30', '2019-10-31',
					'2019-11-01', '2019-11-22', '2019-11-23', '2019-11-25', '2019-11-26', '2019-11-27', '2019-11-28', '2019-11-29', '2019-11-30',
					'2019-12-01', '2019-12-02', '2019-12-04', '2019-12-17', '2019-12-19', '2019-12-20', '2019-12-21', '2019-12-23', '2019-12-24', '2019-12-25', '2019-12-26', '2019-12-28', '2019-12-29', '2019-12-30',
					'2020-01-02', '2020-01-03', '2020-01-17', '2020-01-18', '2020-01-23', '2020-01-28',
					'2020-02-12', '2020-02-13', '2020-02-14', '2020-02-17', '2020-02-18', '2020-02-19', '2020-02-20',
					'2020-03-13', '2020-03-16', '2020-03-18', '2020-03-20', '2020-03-21', '2020-03-22',
					'2020-05-20', '2020-05-29', '2020-05-30',
					'2020-06-02', '2020-06-11', '2020-06-12', '2020-06-13', '2020-06-14', '2020-06-15', '2020-06-16', '2020-06-17', '2020-06-18', '2020-06-19', '2020-06-20',  '2020-06-26',
					'2020-07-01', '2020-07-18', '2020-07-19',
					'2020-08-01', '2020-08-12', '2020-08-13', '2020-08-21', '2020-08-22', '2020-08-27',
					'2020-12-03', '2020-12-04', '2020-12-07', '2020-12-08', '2020-12-09', '2020-12-10', '2020-12-12', '2020-12-13', '2020-12-15', '2020-12-16', '2020-12-17', '2020-12-18', '2020-12-19', '2020-12-20', '2020-12-21', '2020-12-22', '2020-12-23', '2020-12-24', '2020-12-25',
					'2021-01-13', '2021-01-14', '2021-01-15', '2021-01-16', '2021-01-17'])

dates_cosqm_str = np.array([dt.strftime(date, '%Y-%m-%d') for date in dt_santa_sun])
dates_color_mask = np.ones(dates_cosqm_str.shape[0], dtype=bool)
dates_color_mask[np.isin(dates_cosqm_str, dates_color_str)] = False

dt_santa_sun = dt_santa_sun[dates_color_mask]
cosqm_santa_sun = cosqm_santa_sun[dates_color_mask]
cosqm_santa_sun_unfiltered = np.copy(cosqm_santa_sun)

d = np.copy(dt_santa_sun)

ddays_cosqm = np.array([(date.date()-d[0].date()).days for date in d])
hours = np.array([date.hour for date in d])

# Sliding window filter on all bands to smooth out rapid variations (mainly caused by R channel having too high variance since resolution of sqm)
print('Sliding window filter on all cosqm bands')
def Sliding_window_cosqm(data_array, window_size):
	da = np.copy(data_array)
	dslide = np.lib.stride_tricks.sliding_window_view(data_array, window_shape = window_size, axis=0).copy() #start and end values lost. size of array is data_array.shape[0]-2
	dd = np.nanmean(dslide, axis=2)
	padd = np.full([1, da.shape[1]], np.nan)
	for i in range(window_size//2):
		dd = np.insert(dd, 0, padd, axis=0)                     #append nan to start to get same shape as input
		dd = np.insert(dd, dd.shape[0], padd, axis=0)      	      #append nan to end to get same shape as input
	return dd

cosqm_santa_sun = Sliding_window_cosqm(cosqm_santa_sun, cosqm_window_size)
np.savetxt('cosqm_filtered_santa.csv', np.array([dt_santa_sun[~np.isnan(cosqm_santa_sun[:,0])], cosqm_santa_sun[:,0][~np.isnan(cosqm_santa_sun[:,0])], cosqm_santa_sun[:,1][~np.isnan(cosqm_santa_sun[:,0])], cosqm_santa_sun[:,2][~np.isnan(cosqm_santa_sun[:,0])], cosqm_santa_sun[:,3][~np.isnan(cosqm_santa_sun[:,0])], cosqm_santa_sun[:,4][~np.isnan(cosqm_santa_sun[:,0])]]).T, fmt="%s")

#print('remove artefact from sliding window filter ZNSB')
for day in np.unique(ddays_cosqm):
	d_mask = np.zeros(ddays_cosqm.shape[0], dtype=bool)
	d_mask[(ddays_cosqm == day) & (hours < 12)] = True
	d_mask[(ddays_cosqm == day-1) & (hours > 12)] = True
	try:
		cosqm_santa_sun[np.where(d_mask==True)[0][0]] = np.nan
	except:
		print('0', day)
		pass
	try:
		cosqm_santa_sun[np.where(d_mask==True)[0][-1]] = np.nan
	except:
		print('-1', day)
		pass

# Plot variance filter effect on data
colors=['r', 'g', 'b', 'y']
plt.figure(dpi=150, figsize=(7,4))
for i in (0,2):
	plt.scatter(dt_santa_sun, cosqm_santa_sun_unfiltered[:,i+1], c=colors[i], s=8, label=f'{cosqm_bands[i]}nm')
	plt.scatter(dt_santa_sun, cosqm_santa_sun[:,i+1], c='k', marker='+', s=8)
plt.xlabel('Time of measurement (UTC)')
plt.ylabel('CoSQM ZNSB (MPSAS)')
plt.ylim(19.6)
plt.legend()
plt.xticks(ticks=[pd.Timestamp(f'2020-01-24 0{i+1}') for i in range(6)], labels=[str(i+1) for i in range(6)])
plt.xlim(pd.Timestamp('2020-01-24 02:30'), pd.Timestamp('2020-01-24 06:33'))
plt.tight_layout()
plt.savefig('figures/sliding_window_cosqm.png')

#Find variance before and after cosqm sliding window
#inds = np.where((dt_santa_date.values == pd.Timestamp('2020-01-24')) | (dt_santa_date.values == pd.Timestamp('2020-01-25')))[0]
#cosqm_santa_sun_unfiltered_var = np.nanvar(cosqm_santa_sun_unfiltered[inds], axis=0)
#cosqm_santa_sun_var = np.nanvar(cosqm_santa_sun[inds], axis=0)

################
# Light pollution trends
################
c = np.copy(cosqm_santa_sun)
c_norm = np.copy(c)
## Every night normalized (substracted magnitude) with mean of 1am to 2am data
## correct for timechange
print('Correct for timechange (DST)')
date2019 = pd.Timestamp('2019-10-27 02', tz='UTC')
date2020p = pd.Timestamp('2020-03-29 01', tz='UTC')
date2020 = pd.Timestamp('2020-10-25 02', tz='UTC')
date2021p = pd.Timestamp('2021-03-28 01', tz='UTC')
date2021 = pd.Timestamp('2021-10-31 02', tz='UTC')
d_local = np.copy(dt_santa_sun)
td = pd.Timedelta('01h')
d_local[(d_local>date2020p) & (d_local<date2020)] += td
d_local[(d_local>date2021p) & (d_local<date2021)] += td
hours_local = np.array([date.hour for date in d_local])

months = np.array([date.month for date in d])

print('Normalize ZNSB with 1-2am mean ZNSB per night')
for day in np.unique(ddays_cosqm):
	d_mask = np.zeros(ddays_cosqm.shape[0], dtype=bool)
	d_mask[(ddays_cosqm == day) & (hours_local < 12)] = True
	d_mask[(ddays_cosqm == day-1) & (hours_local > 12)] = True
	d_mask_mean = d_mask.copy()
	d_mask_mean[hours_local != 1] = False
	c_mean = np.nanmean(c[d_mask_mean],axis=0)
	c_norm[d_mask] -= c_mean

## Threshold clip of data
print('Threshold clip of normalized cosqm ZNSB')
c_norm[c_norm > normalized_cosqm_znsb_threshold] = np.nan

# Fitting of the normalized data to correct for night trend (does not change through year, month or day of week)
# 2nd order function for fit
def third_order(x,a,b,c,d):
	return a*x**3+b*x**2+c*x+d

hours_float = np.array([ date.hour+                    #WATCH OUT FOR TIMEZONE HERE!
	date.minute/60+
	date.second/3600 for date in d_local ])
hours_float[hours_float>12]-=24

fit_params_c, _ = curve_fit(third_order, hours_float[~np.isnan(c_norm[:,0])], c_norm[~np.isnan(c_norm[:,0])][:,0])
fit_params_r, _ = curve_fit(third_order, hours_float[~np.isnan(c_norm[:,1])], c_norm[~np.isnan(c_norm[:,1])][:,1])
fit_params_g, _ = curve_fit(third_order, hours_float[~np.isnan(c_norm[:,2])], c_norm[~np.isnan(c_norm[:,2])][:,2])
fit_params_b, _ = curve_fit(third_order, hours_float[~np.isnan(c_norm[:,3])], c_norm[~np.isnan(c_norm[:,3])][:,3])
fit_params_y, _ = curve_fit(third_order, hours_float[~np.isnan(c_norm[:,4])], c_norm[~np.isnan(c_norm[:,4])][:,4])
fit_params = np.array([fit_params_c, fit_params_r, fit_params_g, fit_params_b, fit_params_y]).T

cosqm_santa_2nd_noweight = np.array([third_order(hours_float, fit_params_c[0], fit_params_c[1], fit_params_c[2], fit_params_c[3]),
	third_order(hours_float, fit_params_r[0], fit_params_r[1], fit_params_r[2], fit_params_r[3]),
	third_order(hours_float, fit_params_g[0], fit_params_g[1], fit_params_g[2], fit_params_g[3]),
	third_order(hours_float, fit_params_b[0], fit_params_b[1], fit_params_b[2], fit_params_b[3]),
	third_order(hours_float, fit_params_y[0], fit_params_y[1], fit_params_y[2], fit_params_y[3])]).T

# Add weighted fit function with weights computed from initial unweighted fit: sigma loop.
# Values bigger than one sigma are removed at each iteration until sigma converges. Once obtained, the final parameters are kept. The computation is made for each band seperately.
# std dev values for Santa Cruz ended up at R:0.078, B:0.072

cosqm_santa_2nd = np.copy(cosqm_santa_2nd_noweight)

for i in range(10):
	sigma = cosqm_santa_2nd - c_norm
	cw_mask = np.abs(sigma[:,0])<np.nanstd(sigma[:,0])
	rw_mask = np.abs(sigma[:,1])<np.nanstd(sigma[:,1])
	gw_mask = np.abs(sigma[:,2])<np.nanstd(sigma[:,2])
	bw_mask = np.abs(sigma[:,3])<np.nanstd(sigma[:,3])
	yw_mask = np.abs(sigma[:,4])<np.nanstd(sigma[:,4])
	print('Trends weighted fit')
	print(f'n_sigma:{i}, std_R:{np.nanstd(sigma[:,1])}, std_B:{np.nanstd(sigma[:,3])}')
	fit_params_weight_c, _ = curve_fit(third_order, hours_float[cw_mask], c_norm[:,0][cw_mask])
	fit_params_weight_r, _ = curve_fit(third_order, hours_float[rw_mask], c_norm[:,1][rw_mask])
	fit_params_weight_g, _ = curve_fit(third_order, hours_float[gw_mask], c_norm[:,2][gw_mask])
	fit_params_weight_b, _ = curve_fit(third_order, hours_float[bw_mask], c_norm[:,3][bw_mask])
	fit_params_weight_y, _ = curve_fit(third_order, hours_float[yw_mask], c_norm[:,4][yw_mask])
	fit_params_weight = np.array([fit_params_weight_c, fit_params_weight_r, fit_params_weight_g, fit_params_weight_b, fit_params_weight_y]).T

	cosqm_santa_2nd = np.array([third_order(hours_float, *fit_params_weight_c),
		third_order(hours_float, *fit_params_weight_r),
		third_order(hours_float, *fit_params_weight_g),
		third_order(hours_float, *fit_params_weight_b),
		third_order(hours_float, *fit_params_weight_y)]).T

# 	if i==0:
# 		xs = np.linspace(hours_float.min()-0.5, hours_float.max()+0.5, 1001)
# 		band = 1
# 		plt.figure(dpi=150,figsize=(7,4))
# 		plt.hist2d(hours_float[np.isfinite(c_norm)[:,band]], c_norm[:,band][np.isfinite(c_norm)[:,band]], 80, cmap='inferno')
# 		#plt.hist2d(hours_float[np.isfinite(c_norm)[:,band]], c_norm[:,band][np.isfinite(c_norm)[:,band]], 80, cmap='inferno', norm=LogNorm(), vmax=10)
# 		plt.plot(xs, third_order(xs, *fit_params[:, band]), label=f'third order fit {cosqm_bands[band-1]}nm', c='c')
# 		plt.xlabel('Local time from midnight (h)')
# 		plt.ylabel(f'Normalized {cosqm_bands[band-1]}nm ZNSB (MPSAS)')
# 		plt.ylim(-0.3)
# 		cbar = plt.colorbar(label='counts')
# 		#cbar.set_ticks(np.arange(1,12,1))
# 		#cbar.update_ticks()
# 		#cbar.set_ticklabels(np.arange(0,11,1))
# 		plt.tight_layout()
# 		#plt.savefig(f'figures/trend/normalized_cosqm_{cosqm_bands[band-1]}.png')
# 	plt.plot(xs, third_order(xs, *fit_params_weight[:,band]), label=f'third order weighted fit 644nm - n_sigma={i+1}', c='w', linestyle='--')
# plt.legend()

#hist 2d
weekdays_str = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday','Sunday']
months_str = ['January', 'Feburary', 'March', 'April', 'May', 'June', 'July', 'August', 'Septembre', 'October', 'November', 'December']
weekdays = np.array([ date.weekday() for date in d_local ])
markers = ['.', '1', '2', '3', '4', 'o', 's']

xs = np.linspace(hours_float.min()-0.5, hours_float.max()+0.5, 1001)
band = 1
plt.figure(dpi=150,figsize=(7,4))
plt.hist2d(hours_float[np.isfinite(c_norm)[:,band]], c_norm[:,band][np.isfinite(c_norm)[:,band]], 80, cmap='inferno')
#plt.hist2d(hours_float[np.isfinite(c_norm)[:,band]], c_norm[:,band][np.isfinite(c_norm)[:,band]], 80, cmap='inferno', norm=LogNorm(), vmax=10)
#plt.plot(xs, third_order(xs, *fit_params[:, band]), label=f'third order fit {cosqm_bands[band-1]}nm', c='c')
plt.plot(xs, third_order(xs, *fit_params_weight[:,band]), label=f'third order weighted fit {cosqm_bands[band-1]}nm', c='w', linestyle='--')
plt.xlabel('Local time from midnight (h)')
plt.ylabel(f'Normalized {cosqm_bands[band-1]}nm ZNSB (MPSAS)')
plt.ylim(-0.3)
cbar = plt.colorbar(label='counts')
#cbar.set_ticks(np.arange(1,12,1))
#cbar.update_ticks()
#cbar.set_ticklabels(np.arange(0,11,1))
plt.tight_layout()
plt.savefig(f'figures/trend/normalized_cosqm_{cosqm_bands[band-1]}.png')

xs = np.linspace(hours_float.min()-0.5, hours_float.max()+0.5, 1001)
band = 3
plt.figure(dpi=150,figsize=(7,4))
plt.hist2d(hours_float[np.isfinite(c_norm)[:,band]], c_norm[:,band][np.isfinite(c_norm)[:,band]], 80, cmap='inferno')
#plt.plot(xs, third_order(xs, fit_params[0, band], fit_params[1, band], fit_params[2, band], fit_params[3, band]), label='third order fit 644nm', c='c')
plt.plot(xs, third_order(xs, fit_params_weight[0, band], fit_params_weight[1, band], fit_params_weight[2, band], fit_params_weight[3, band]), label='third order weighted fit 644nm', c='w', linestyle='--')
plt.xlabel('Local time from midnight (h)')
plt.ylabel(f'Normalized {cosqm_bands[band-1]}nm ZNSB (MPSAS)')
plt.ylim(-0.3)
cbar = plt.colorbar(label='counts')
#cbar.set_ticks(np.arange(1,12,1))
#cbar.update_ticks()
#cbar.set_ticklabels(np.arange(0,11,1))
plt.tight_layout()
plt.savefig(f'figures/trend/normalized_cosqm_{cosqm_bands[band-1]}.png')

# Correct filtered data with fit curve
print('Correct filtered data with trend fits')
dt_santa_final = dt_santa_sun
cosqm_santa_final = np.copy(cosqm_santa_sun) - cosqm_santa_2nd
ddays_cosqm = np.array([(date.date()-d[0].date()).days for date in dt_santa_final])
dt_santa_date = dt_santa_final.dt.date
dt_santa_dt = dt_santa_final.dt.to_pydatetime()

#Plot effect of corrected trends on specific nights
band = 3
fig,ax = plt.subplots(dpi=150, figsize=(7,4))
plt.setp(ax, xticks=[pd.Timestamp('2020-05-22 23'), pd.Timestamp('2020-05-23 00'), pd.Timestamp('2020-05-23 01'), pd.Timestamp('2020-05-23 02')], xticklabels=['-1', '0', '1', '2'])
ax.scatter(d_local, cosqm_santa_sun[:,band], s=10, label=f'Filtered CoSQM {cosqm_bands[band-1]}nm ZNSB')
ax.scatter(d_local, cosqm_santa_final[:,band], s=10, label='Corrected for nightly trend')
ax.set_xlim(pd.Timestamp('2020-05-22 22h35'), pd.Timestamp('2020-05-23 03'))
ax.set_xlabel('Local time from midnight (h)')
ax.tick_params('x', labelrotation=0)
ax.set_ylim(19.40, 19.65)
ax.set_ylabel(f'{cosqm_bands[band-1]}nm CoSQM ZNSB (MPSAS)')
ax.legend()
plt.tight_layout()
#fig.savefig(f'figures/trend/correction_single_night_{cosqm_bands[band-1]}.png')

c_tot = np.array((c_norm[:,1]+np.mean(cosqm_santa_sun[:,1][np.where((hours_float ==1) | (hours_float ==2))]),
c_norm[:,2]+np.mean(cosqm_santa_sun[:,2][np.where((hours_float ==1) | (hours_float ==2))]),
c_norm[:,3]+np.mean(cosqm_santa_sun[:,3][np.where((hours_float ==1) | (hours_float ==2))]),
c_norm[:,4]+np.mean(cosqm_santa_sun[:,4][np.where((hours_float ==1) | (hours_float ==2))]))).T

x = np.arange(-2,6, 0.01)

#Plot corrected trend to show red goes darker throughout the night compared to other colors
plt.figure(dpi=200, figsize=(7,5))
plt.plot(x, third_order(x, fit_params_weight_r[0], fit_params_weight_r[1], fit_params_weight_r[2], fit_params_weight_r[3]), c='r', linestyle='solid', linewidth=2, label=f'{cosqm_bands[0]} nm')
plt.plot(x, third_order(x, fit_params_weight_g[0], fit_params_weight_g[1], fit_params_weight_g[2], fit_params_weight_g[3]), c='g', linestyle='dotted', linewidth=2, label=f'{cosqm_bands[1]} nm')
plt.plot(x, third_order(x, fit_params_weight_b[0], fit_params_weight_b[1], fit_params_weight_b[2], fit_params_weight_b[3]), c='b', linestyle='dashed', linewidth=2, label=f'{cosqm_bands[2]} nm')
plt.plot(x, third_order(x, fit_params_weight_y[0], fit_params_weight_y[1], fit_params_weight_y[2], fit_params_weight_y[3]), c='y', linestyle='dashdot', linewidth=2, label=f'{cosqm_bands[3]} nm')
plt.legend()
plt.xlabel('Local time from midnight (h)')
plt.ylabel('Normalized CoSQM ZNSB (MPSAS)')
plt.tight_layout()
#plt.savefig(f'figures/trend/trend_fits.png')

# #hist 2d
# plt.figure(figsize=[12,8])
# plt.hist2d(hours_float[np.isfinite(cosqm_santa_corr)[:,0]], cosqm_santa_corr[:,0][np.isfinite(cosqm_santa_corr)[:,0]], 80, cmap='inferno')
# #plt.hist2d(hours_float[np.isfinite(cosqm_santa_corr)[:,0]], cosqm_santa_corr[:,0][np.isfinite(cosqm_santa_corr)[:,0]], 80, cmap='inferno', norm=LogNorm())
# plt.xlabel('hour')
# plt.ylabel('CoSQM magnitude')

##########################
# AOD
##########################

#%%--------------------
#
#       AOD Values from AERONET network
#   Values are from level 1.0, meaning there are no corrections
#
#######
# read from columns 4 to 25 (aod 1640 to 340nm)

data_aod_raw = pd.read_csv(path_aod, delimiter=',', header=6)
data_angstrom_raw = data_aod_raw['440-675_Angstrom_Exponent']

data_aod_raw['Datetime'] = pd.to_datetime(data_aod_raw['Date(dd:mm:yyyy)']+'T'+data_aod_raw['Time(hh:mm:ss)'], utc=True, format='%d:%m:%YT%H:%M:%S')
dt_aod = data_aod_raw['Datetime']
dt_aod_date = data_aod_raw['Datetime'].dt.date
aod_bands = data_aod_raw.columns[(data_aod_raw.values[0]!=-999.0)]
#aod_bands = aod_bands[[column[:3] == 'AOD' for column in aod_bands]]
aod_bands = ['AOD_675nm', 'AOD_500nm', 'AOD_500nm', 'AOD_500nm']
data_aod_raw = data_aod_raw[aod_bands].values

print('filter 0 values in aeronet AOD')
mask = np.all(data_aod_raw>0, axis=1)
data_aod = data_aod_raw[mask]
data_angstrom = data_angstrom_raw[mask].values

dt_aod = dt_aod[mask]
dt_aod_date = dt_aod_date[mask]

#BANDS = [AOD_1640nm', 'AOD_1020nm', 'AOD_870nm', 'AOD_675nm', 'AOD_500nm',
#       'AOD_440nm', 'AOD_380nm']

#CoSQM bands matching:
#R:629->AOD 675 index 3
#G:546->AOD 500 index 4
#B:514->AOD 500 index 4
#Y:562->AOD 500 index 4

# Plot each band of aod measurements for total data
#plt.figure()
#for i in range(aod_bands.shape[0]):
#    plt.scatter(dt_aod, data_aod[:,i], label=aod_bands[i], s=0.2)
#plt.legend()
#plt.show()

# find same days in each instrument
print('find same days that have measurements (aeronet vs cosqm)')
mask_aod = np.isin(dt_aod_date.values, np.unique(dt_santa_date))
dt_aod = dt_aod[mask_aod]
dt_aod_date = dt_aod_date[mask_aod]
data_aod = data_aod[mask_aod]
data_angstrom = data_angstrom[mask_aod]

mask_santa = np.isin(dt_santa_date.values, np.unique(dt_aod.dt.date))
dt_santa_final = dt_santa_final[mask_santa]
dt_santa_date = dt_santa_date[mask_santa]
cosqm_santa_final = cosqm_santa_final[mask_santa]
hours_float = hours_float[mask_santa]

dt_santa_corr = dt_santa_final
cosqm_santa_corr = cosqm_santa_final[:,1:]

# Evaluation of distinction between cosqm bands -> to determine if bands are uncorrelated to extract valuable aod and AE
xs = np.arange(16,21,0.01)
fig, ax = plt.subplots(1,3, sharex=True, sharey=True, figsize=(10,7), dpi=200)
ax[0].scatter(cosqm_santa_corr[:,0], cosqm_santa_corr[:,1], s=10, c='g', label=f'{cosqm_bands[1]} nm')
ax[0].scatter(cosqm_santa_corr[:,0], cosqm_santa_corr[:,2], s=10, c='b', label=f'{cosqm_bands[2]} nm')
ax[0].scatter(cosqm_santa_corr[:,0], cosqm_santa_corr[:,3], s=10, c='y', label=f'{cosqm_bands[3]} nm')
ax[1].scatter(cosqm_santa_corr[:,2], cosqm_santa_corr[:,1], s=10, c='g', label=f'{cosqm_bands[1]} nm')
ax[1].scatter(cosqm_santa_corr[:,2], cosqm_santa_corr[:,3], s=10, c='y', label=f'{cosqm_bands[3]} nm')
ax[2].scatter(cosqm_santa_corr[:,1], cosqm_santa_corr[:,3], s=10, c='y', label=f'{cosqm_bands[3]} nm')
ax[0].set_xlabel(f'{cosqm_bands[0]} nm')
ax[1].set_xlabel(f'{cosqm_bands[2]} nm')
ax[2].set_xlabel(f'{cosqm_bands[1]} nm')
ax[0].legend()
ax[1].legend()
ax[2].legend()
fig.supxlabel('CoSQM ZNSB (MPSAS)', fontsize=10)
fig.supylabel('CoSQM ZNSB (MPSAS)', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/correlation/santa/correlation_znsb.png')

#############
# CORRELATION
#############

# Selection of filtered and corrected ZNSB values
print('selection of cosqm measurements for dusk and dawn correlation')
cosqm_am = np.zeros((np.unique(dt_santa_date).shape[0], 5))
cosqm_pm = np.zeros((np.unique(dt_santa_date).shape[0], 5))
hours_cosqm = dt_santa_final.dt.hour

for i,day in enumerate(np.unique(dt_santa_date)):
#    d_mask_am = np.zeros(ddays_cosqm.shape[0], dtype=bool)
#    d_mask_pm = np.zeros(ddays_cosqm.shape[0], dtype=bool)
	d_mask_am = (dt_santa_date == day) & (hours_cosqm >= cosqm_min_am_hour) & (hours_cosqm < 12)
	d_mask_pm = (dt_santa_date == day) & (hours_cosqm >= 12) & (hours_cosqm < cosqm_max_pm_hour)
	inds_am = np.where(d_mask_am == True)[0]
	if inds_am.shape[0]>cosqm_mean_count:
		inds_am = inds_am[-inds_am.shape[0]:]
		cosqm_am[i] = np.nanmean(cosqm_santa_final[inds_am],axis=0)
	else:
		inds_am[inds_am == True] = False
		cosqm_am[i] = np.nan
	inds_pm = np.where(d_mask_pm == True)[0]
	if inds_pm.shape[0]>cosqm_mean_count:
		inds_pm = inds_pm[:inds_pm.shape[0]]
		cosqm_pm[i] = np.nanmean(cosqm_santa_final[inds_pm],axis=0)
	else:
		inds_pm[inds_pm == True] = False
		cosqm_pm[i] = np.nan
#    cosqm_am[cosqm_am == 0] = np.nan                    # remove zeros values from sensor problem (no data?)
#    cosqm_pm[cosqm_pm == 0] = np.nan                    # remove zeros values from sensor problem (no data?)

# band = 3
# plt.figure()
# plt.scatter(np.unique(dt_santa_date), cosqm_am[:,band], label='cosqm_am', s=15)
# plt.scatter(np.unique(dt_santa_date), cosqm_pm[:,band], label='cosqm_pm', s=15)
# plt.xlabel('days from july 2019')
# plt.ylabel(f'CoSQM {cosqm_bands[band-1]}nm Magnitudes (mag)')
# plt.legend()

# Selection of AOD values
print('selection of aeronet aod measurements for dusk and dawn correlation')
aod_am = np.zeros((np.unique(dt_aod_date).shape[0], 4))
aod_pm = np.zeros((np.unique(dt_aod_date).shape[0], 4))
angstrom_am = np.zeros((np.unique(dt_aod_date).shape[0], 4))
angstrom_pm = np.zeros((np.unique(dt_aod_date).shape[0], 4))
hours_aod = dt_aod.dt.hour

for i,day in enumerate(np.unique(dt_aod_date)):
	d_mask_am = np.zeros(dt_aod_date.shape[0], dtype=bool)
	d_mask_pm = np.zeros(dt_aod_date.shape[0], dtype=bool)
	d_mask_am[(dt_aod_date == day) & (hours_aod < aod_max_am_hour)] = True
	d_mask_pm[(dt_aod_date == day) & (hours_aod >= aod_min_pm_hour)] = True
	inds_am = np.where(d_mask_am == True)[0]
	if inds_am.shape[0]>aod_mean_count:
		inds_am = inds_am[:inds_am.shape[0]]
		aod_am[i] =  np.nanmean(data_aod[inds_am], axis=0)
		angstrom_am[i] = np.nanmean(data_angstrom[inds_am], axis=0)
	else:
		inds_am[inds_am == True] = False
		aod_am[i] =  np.nan
		angstrom_am[i] = np.nan

	inds_pm = np.where(d_mask_pm == True)[0]
	if inds_pm.shape[0]>aod_mean_count:
		inds_pm = inds_pm[-inds_pm.shape[0]:]
		aod_pm[i] =  np.nanmean(data_aod[inds_pm],axis=0)
		angstrom_pm[i] = np.nanmean(data_angstrom[inds_pm], axis=0)
	else:
		inds_pm[inds_pm == True] = False
		aod_pm[i] =  np.nan
		angstrom_pm[i] = np.nan

wls1 = np.ones((aod_am.shape[0], 4))*cosqm_bands      # CoSQM center wavelenghts per filter
wls2 = np.ones((aod_am.shape[0], 4))*np.array([675, 500, 500, 500])     # AERONET daytime sunphoto center wavelenghts per filter

def aod_from_angstrom(aod2, wls1, wls2, alpha):
	return aod2*(wls1/wls2)**-alpha

def angstrom_from_aod(arr, bands, ind1, ind2):
	return -np.log(arr[:,ind1]/arr[:,ind2])/np.log(bands[ind1]/bands[ind2])

# Correct computed aod with matching band nominal wavelength
print('correct computed aod with matching band nominal wavelength')
aod_am_corr = aod_from_angstrom(aod_am, wls1, wls2, angstrom_am)
aod_pm_corr = aod_from_angstrom(aod_pm, wls1, wls2, angstrom_pm)

# band=3
# plt.figure()
# plt.scatter(np.unique(dt_santa_date), aod_am_corr[:,band], label='aod_am_corr', s=15)
# plt.scatter(np.unique(dt_santa_date), aod_pm_corr[:,band], label='aod_pm_corr', s=15)
# plt.xlabel('days from july 2019')
# plt.ylabel(f'CoSQM {cosqm_bands[band-1]}nm Magnitudes (mag)')
# plt.legend()

# Correlation fits
print('AERONET AOD-CoSQM ZNSB correlation calculations')

def fit_func(x, a,b):
	return -a*np.log(x/b)

def fit_func1(x, params):
	return fit_func(x, params[0], params[1])

#single fit functions for both dusk and dawn correlated points
param_bounds=([0,10],[15,25])
p0 = [7, 20]
cosqm = np.vstack([cosqm_am, cosqm_pm])[:,1:]
cosqm = np.array(cosqm, dtype=np.longdouble)

aod_unfiltered = np.vstack([aod_am, aod_pm])
aod_am[aod_am<0.04] = np.nan
aod_pm[aod_pm<0.04] = np.nan
aod = np.vstack([aod_am, aod_pm])
valid = ~(np.isnan(cosqm) | (np.isnan(aod)))

# make histogram of aod values
plt.figure(dpi=150, figsize=(12,8))
hist_aod_unfiltered = np.histogram(aod_unfiltered[~np.isnan(aod_unfiltered)], bins=100)
plt.scatter(hist_aod_unfiltered[1][:-1][hist_aod_unfiltered[0]>0], hist_aod_unfiltered[0][hist_aod_unfiltered[0]>0], c='b')
plt.xlim(0,1.5)
plt.ylim(0)
plt.xlabel('AOD values')
plt.ylabel('counts')

corr_fitr, _ = curve_fit(fit_func, cosqm[:,0][valid[:,0]], aod[:,0][valid[:,0]], p0=p0, bounds=param_bounds)
corr_fitg, _ = curve_fit(fit_func, cosqm[:,1][valid[:,1]], aod[:,1][valid[:,1]], p0=p0, bounds=param_bounds)
corr_fitb, _ = curve_fit(fit_func, cosqm[:,2][valid[:,2]], aod[:,2][valid[:,2]], p0=p0, bounds=param_bounds)
corr_fity, _ = curve_fit(fit_func, cosqm[:,3][valid[:,3]], aod[:,3][valid[:,3]], p0=p0, bounds=param_bounds)

corr_single_list = [corr_fitr,corr_fitg,corr_fitb,corr_fity]
corr_single_round = [np.around(corr, decimals=2) for corr in corr_single_list]

#fit functions for dusk
valid = ~(np.isnan(cosqm_am[:,1:]) | (np.isnan(aod_am)))
corr_am_fitr, _ = curve_fit(fit_func, cosqm_am[:,1][valid[:,0]], aod_am[:,0][valid[:,0]], p0=p0, bounds=param_bounds)
corr_am_fitg, _ = curve_fit(fit_func, cosqm_am[:,2][valid[:,1]], aod_am[:,1][valid[:,1]], p0=p0, bounds=param_bounds)
corr_am_fitb, _ = curve_fit(fit_func, cosqm_am[:,3][valid[:,2]], aod_am[:,2][valid[:,2]], p0=p0, bounds=param_bounds)
corr_am_fity, _ = curve_fit(fit_func, cosqm_am[:,4][valid[:,3]], aod_am[:,3][valid[:,3]], p0=p0, bounds=param_bounds)

#fit functions for dawn
valid = ~(np.isnan(cosqm_pm[:,1:]) | (np.isnan(aod_pm)))
corr_pm_fitr, _ = curve_fit(fit_func, cosqm_pm[:,1][valid[:,0]], aod_pm[:,0][valid[:,0]], p0=p0, bounds=param_bounds)
corr_pm_fitg, _ = curve_fit(fit_func, cosqm_pm[:,2][valid[:,1]], aod_pm[:,1][valid[:,1]], p0=p0, bounds=param_bounds)
corr_pm_fitb, _ = curve_fit(fit_func, cosqm_pm[:,3][valid[:,2]], aod_pm[:,2][valid[:,2]], p0=p0, bounds=param_bounds)
corr_pm_fity, _ = curve_fit(fit_func, cosqm_pm[:,4][valid[:,3]], aod_pm[:,3][valid[:,3]], p0=p0, bounds=param_bounds)

corr_list = [corr_am_fitr,corr_pm_fitr,corr_am_fitg,corr_pm_fitg,corr_am_fitb,corr_pm_fitb,corr_am_fity,corr_pm_fity]
corr_round = [np.around(corr, decimals=2) for corr in corr_list]

cosqm_aod_r = fit_func1(cosqm_santa_corr[:,0], corr_fitr)
cosqm_aod_g = fit_func1(cosqm_santa_corr[:,1], corr_fitg)
cosqm_aod_b = fit_func1(cosqm_santa_corr[:,2], corr_fitb)
cosqm_aod_y = fit_func1(cosqm_santa_corr[:,3], corr_fity)

# # Threshold clip on correlated AOD (crazy AOD values from imperfect AOD-ZNSB fit)
# print('threshold clip on correlated cosqm aod (for aod values showing 50% error)')
# cosqm_aod_r[cosqm_aod_r<cosqm_aod_threshold] = np.nan
# cosqm_aod_g[cosqm_aod_g<cosqm_aod_threshold] = np.nan
# cosqm_aod_b[cosqm_aod_b<cosqm_aod_threshold] = np.nan
# cosqm_aod_y[cosqm_aod_y<cosqm_aod_threshold] = np.nan
# cosqm_aod_all = np.array((cosqm_aod_r, cosqm_aod_g, cosqm_aod_b, cosqm_aod_y)).T

# # fit AE for each 4 points cosqm measurement
# print('compute AE fit for each cosqm 4 values measurements')
# def FitAe(wl, k, alpha):
# 	return k*wl**-alpha

# def fit_except(data_array):                         # Some cosqm aod values do not converge while fitting
# 	try:
# 		return curve_fit(FitAe, cosqm_bands, data_array, p0=(1000, 0.5))[0]
# 	except RuntimeError:
# 		print(f"Error - curve_fit failed: cosqm aod values {data_array}")
# 		return np.array((None, None), dtype=object)

# ae_fit_params = np.zeros((cosqm_aod_all.shape[0], 2))
# ae_fit_params[:] = np.nan
# cosqm_nonnans = ~np.isnan(cosqm_aod_all).any(axis=1)
# ae_fit_params[cosqm_nonnans] = np.vstack([fit_except(cosqm_aods) for cosqm_aods in cosqm_aod_all[cosqm_nonnans]])
# ae_fit_succeed = ~np.isnan(ae_fit_params[:,0])

# cosqm_ae = np.array([(fit1, fit2) for fit1, fit2 in ae_fit_params])

cosqm_ae = angstrom_from_aod(np.array([cosqm_aod_r, cosqm_aod_b]).T, cosqm_bands, 0, 2)

#Filter bad AE values (negative and higher than threshold)
ae_mask = (cosqm_ae>ae_min) & (cosqm_ae<=ae_max)

# Plot fitted AE from cosqm_aod values
# dt_santa_final_date = np.array([date.to_datetime64() for date in dt_santa_final])
# x_ae = np.arange(500,700, 0.1)
# inds=np.arange(2550, 2555, 1)    # to find specific nights: np.where(dt_santa_date == pd.to_datetime('2020-02-24'))

# for ind in inds:
#     plt.scatter(cosqm_bands, cosqm_aod_all[ind], label=f'{dt_santa_final_date[ind]}')
#     plt.plot(x_ae, FitAe(x_ae, ae_fit_params[ind][0], ae_fit_params[ind][1]))
# plt.legend()

# # plot znsb to understand why alpha goes crazy for high ZNSB and small ZNSB differences between bands
# colors=['r', 'g', 'b', 'y']
# for i in range(4):
#     plt.scatter(dt_santa_dt, cosqm_santa_final[:, i+1], c=colors[i], s=10)

#single fit function: correlation
xs = np.arange(16,21,0.01)
fig, ax = plt.subplots(2,1, sharex=True, sharey=True, figsize=(7,7), dpi=200)
ax[0].scatter(cosqm_pm[:,1], aod_pm[:,0], label='dawn')
ax[0].scatter(cosqm_am[:,1], aod_am[:,0], label='dusk')
ax[0].plot(xs, fit_func1(xs, corr_fitr), c='k', linewidth=1)
ax[1].scatter(cosqm_pm[:,3], aod_pm[:,2])
ax[1].scatter(cosqm_am[:,3], aod_am[:,2])
ax[1].plot(xs, fit_func1(xs, corr_fitb), c='k', linewidth=1)
ax[0].set_xlim(17,20.7)
ax[0].set_yscale('log')
ax[0].set_ylim(0.028,3.75)
ax[0].text(0.05,0.2, f'{cosqm_bands[0]} nm\na,b = {str(corr_single_round[0])[1:-1]}', horizontalalignment='left', verticalalignment='center', transform=ax[0].transAxes)
ax[1].text(0.05,0.2, f'{cosqm_bands[2]} nm\na,b = {str(corr_single_round[2])[1:-1]}', horizontalalignment='left', verticalalignment='center', transform=ax[1].transAxes)
ax[0].legend(loc='center left', prop={'size': 10})
fig.supxlabel('CoSQM ZNSB (MPSAS)', fontsize=10)
fig.supylabel('AERONET daytime AOD', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/correlation/santa/correlation_single_fit_santa.png')

# Relative humidity data
msize = 5
hr = pd.read_csv('hr_2020.dat').values
hr[:,1] = np.around(hr[:,1].astype(float), decimals=0)
hr[:,3] = np.around(hr[:,3].astype(float), decimals=0)
fig,ax = plt.subplots(figsize=(6,3), dpi=200)
plt.hist(hr[:,1], bins=np.unique(hr[:,1]),  label='Mean (Dawn)', alpha=0.8)
plt.hist(hr[:,3], bins=np.unique(hr[:,3]), label='Mean (Dusk)', alpha=0.8)
plt.legend(loc='upper left', prop={'size': 9})
plt.xlabel('Relative humidity (%)')
plt.ylabel('Counts')
plt.tight_layout()
#plt.savefig('figures/continuity/hr_2020.png')

#single fit function: continuity 2020-02-21
msize=1.5
fig, ax = plt.subplots(2,1, sharex=True, sharey=True, dpi=200, figsize=(8,8))
plt.setp(ax, xticks=[pd.Timestamp('2020-02-22'), pd.Timestamp('2020-02-27'), pd.Timestamp('2020-03-03')], xticklabels=['2020-02-22', '2020-02-27', '2020-03-03'])
ax[0].scatter(dt_aod, data_aod[:,0], s=msize, label='AERONET daytime')
ax[0].scatter(dt_santa_corr[ae_mask], cosqm_aod_r[ae_mask], s=msize, label='CoSQM derived AOD')
ax[1].scatter(dt_aod, data_aod[:,2], s=msize)
ax[1].scatter(dt_santa_corr[ae_mask], cosqm_aod_b[ae_mask], s=msize)
ax[0].tick_params('x', labelrotation=0)
ax[1].tick_params('x', labelrotation=0)
ax[0].set_yscale('log')
ax[0].set_xlim(pd.Timestamp('2020-02-20 12'), pd.Timestamp('2020-03-05'))
ax[0].set_ylim(2e-2,5.3)
ax[0].legend(prop={"size":6}, loc='upper right')
ax[0].text(0.35,0.1, f'{cosqm_bands[0]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[0].transAxes, fontsize=10)
ax[1].text(0.35,0.1, f'{cosqm_bands[2]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[1].transAxes, fontsize=10)
fig.supxlabel('Time (UTC)', fontsize=10)
fig.supylabel('AOD', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_aod_20200220.png')

#single fit function: continuity Angstrom 2020-02-21
fig, ax = plt.subplots(1,1, dpi=200, figsize=(8,6))
plt.setp(ax, xticks=[pd.Timestamp('2020-02-22'), pd.Timestamp('2020-02-27'), pd.Timestamp('2020-03-03')], xticklabels=['2020-02-22', '2020-02-27', '2020-03-03'])
ax.scatter(dt_aod, data_angstrom, s=1.5, label=f'AERONET daytime derived {cosqm_bands[2]}-{cosqm_bands[0]}nm AE')
ax.scatter(dt_santa_corr[ae_mask], cosqm_ae[ae_mask], s=1.5, label=f'CoSQM derived fitted AE')
ax.plot(dt_aod, np.zeros(dt_aod.shape[0]), linewidth=1, linestyle='--', color='grey')
ax.set_xlim(pd.Timestamp('2020-02-20 16'), pd.Timestamp('2020-03-04'))
ax.set_ylim(-0.2,2)
ax.set_xlabel('Time - UTC')
ax.set_ylabel('Angstrom exponent')
ax.legend(prop={"size":10})
ax.xaxis.set_minor_locator(MultipleLocator(1))
#ax.set_yscale('log')
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_angstrom_20200220.png')

# Import Lidar data from AERONET on may 2020 (Africa provided the data)
mpl_may = pd.read_csv('wetransfer_lidar_data_2021-09-06_0658/Lidar_Data/aod_SCO.dat', delimiter=' ', skiprows=2, names=('Date', 'Time', 'aod'))[:-1]
mpl_may_aod = np.array([str(val)[:-1] for val in mpl_may['aod']], dtype=float)
mpl_may_dates = np.array([pd.Timestamp(date+'T'+time) for date, time in np.array([mpl_may['Date'].values, mpl_may['Time'].values]).T])

#single fit function: continuity 2020-05-21
msize=1.5
fig, ax = plt.subplots(2,1, sharex=True, sharey=True, dpi=200, figsize=(8,8))
plt.setp(ax, xticks=[pd.Timestamp('2020-05-23'), pd.Timestamp('2020-05-27')], xticklabels=['2020-05-23', '2020-05-27'])
ax[0].scatter(dt_aod, data_aod[:,0], s=msize)
ax[0].scatter(dt_santa_corr[ae_mask], cosqm_aod_r[ae_mask], s=msize)
ax[1].scatter(mpl_may_dates, mpl_may_aod, c='k', marker='+', s=msize/2, label='MPL derived AOD (532nm)')
ax[1].scatter(dt_aod, data_aod[:,2], s=msize, label='AERONET daytime')
ax[1].scatter(dt_santa_corr[ae_mask], cosqm_aod_b[ae_mask], s=msize, label='CoSQM derived AOD')
ax[1].tick_params('x', labelrotation=0)
ax[1].tick_params('x', labelrotation=0)
ax[0].set_yscale('log')
ax[0].set_xlim(pd.Timestamp('2020-05-21 12'), pd.Timestamp('2020-05-29'))
ax[0].set_ylim(0.02,0.81)
ax[1].legend(prop={"size":10}, loc='upper left')
ax[0].text(0.35,0.10, f'{cosqm_bands[0]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[0].transAxes, fontsize=10)
ax[1].text(0.35,0.10, f'{cosqm_bands[2]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[1].transAxes, fontsize=10)
ax[1].xaxis.set_minor_locator(MultipleLocator(1))
fig.supxlabel('Time (UTC)', fontsize=10)
fig.supylabel('AOD', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_aod_20200521.png')

#single fit function: continuity Angstrom 2020-05-21
fig, ax = plt.subplots(1,1, dpi=200, figsize=(8,6))
plt.setp(ax, xticks=[pd.Timestamp('2020-05-22'), pd.Timestamp('2020-05-25'), pd.Timestamp('2020-05-28')], xticklabels=['2020-05-22', '2020-05-25', '2020-05-28'])
ax.scatter(dt_aod, data_angstrom, s=1.5, label=f'AERONET daytime derived {cosqm_bands[2]}-{cosqm_bands[0]}nm AE')
ax.scatter(dt_santa_corr[ae_mask], cosqm_ae[ae_mask], s=1.5, label='CoSQM derived fitted AE')
ax.plot(dt_aod, np.zeros(dt_aod.shape[0]), linewidth=1, linestyle='--', color='grey')
ax.set_xlim(pd.Timestamp('2020-05-21'), pd.Timestamp('2020-05-29'))
ax.set_ylim(-0.2,2)
ax.set_xlabel('Time - UTC')
ax.set_ylabel('Angstrom exponent')
ax.legend(prop={"size":10})
ax.xaxis.set_minor_locator(MultipleLocator(1))
#ax.set_yscale('log')
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_angstrom_20200521.png')

# Import Lidar data from AERONET on july 2020 (Africa provided the data)
mpl_july = pd.read_csv('wetransfer_lidar_data_2021-09-06_0658/aod_sco_july_2020.dat', delimiter=' ', skiprows=2, names=('Date', 'Time', 'aod'))[:-1]
mpl_july_aod = np.array([str(val)[:-1] for val in mpl_july['aod']], dtype=float)
mpl_july_dates = np.array([pd.Timestamp(date+'T'+time) for date, time in np.array([mpl_july['Date'].values, mpl_july['Time'].values]).T])

#single fit function: continuity 2020-07-07
msize=1.5
fig, ax = plt.subplots(2,1, sharex=True, sharey=True, dpi=200, figsize=(8,8))
plt.setp(ax, xticks=[pd.Timestamp('2020-07-08'), pd.Timestamp('2020-07-12'), pd.Timestamp('2020-07-16')], xticklabels=['2020-07-08', '2020-07-12', '2020-07-16'])
ax[0].scatter(dt_aod, data_aod[:,0], s=msize)
ax[0].scatter(dt_santa_corr[ae_mask], cosqm_aod_r[ae_mask], s=msize)
ax[1].scatter(mpl_july_dates, mpl_july_aod, c='k', marker='+', s=msize/5, label='MPL derived AOD')
ax[1].scatter(dt_aod, data_aod[:,2], s=msize, label='AERONET daytime')
ax[1].scatter(dt_santa_corr[ae_mask], cosqm_aod_b[ae_mask], s=msize, label='CoSQM derived AOD')
ax[1].tick_params('x', labelrotation=0)
ax[0].set_yscale('log')
ax[0].set_xlim(pd.Timestamp('2020-07-07'), pd.Timestamp('2020-07-18 12'))
ax[0].set_ylim(0.02,5.3)
ax[1].legend(prop={"size":6}, loc='lower left')
ax[0].text(0.1,0.3, f'{cosqm_bands[0]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[0].transAxes, fontsize=10)
ax[1].text(0.1,0.3, f'{cosqm_bands[2]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[1].transAxes, fontsize=10)
fig.supxlabel('Time (UTC)', fontsize=10)
fig.supylabel('AOD', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_aod_20200707.png')

#single fit function: continuity Angstrom 2020-07-07
fig, ax = plt.subplots(1,1, dpi=200, figsize=(8,6))
plt.setp(ax, xticks=[pd.Timestamp('2020-07-08'), pd.Timestamp('2020-07-12'), pd.Timestamp('2020-07-16')], xticklabels=['2020-07-08', '2020-07-12', '2020-07-16'])
ax.scatter(dt_aod, data_angstrom, s=1.5, label=f'AERONET daytime derived {cosqm_bands[2]}-{cosqm_bands[0]}nm AE')
ax.scatter(dt_santa_corr[ae_mask], cosqm_ae[ae_mask], s=1.5, label=f'CoSQM derived fitted AE')
ax.plot(dt_aod, np.zeros(dt_aod.shape[0]), linewidth=1, linestyle='--', color='grey')
ax.set_xlim(pd.Timestamp('2020-07-07'), pd.Timestamp('2020-07-18 12'))
ax.set_ylim(-0.2,2)
ax.set_xlabel('Time - UTC')
ax.set_ylabel('Angstrom exponent')
ax.legend(prop={"size":10})
ax.xaxis.set_minor_locator(MultipleLocator(1))
#ax.set_yscale('log')
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_angstrom_20200707.png')

msize=3
#single fit function: continuity -> variance for uncertainty
fig, ax = plt.subplots(2,1, sharex=True, sharey=True, dpi=200, figsize=(8,8))
plt.setp(ax, xticks=[pd.Timestamp('2020-05-23 17'), pd.Timestamp('2020-05-23 21'), pd.Timestamp('2020-05-24 01'), pd.Timestamp('2020-05-24 05')], xticklabels=['-7', '-3', '1', '5'])
ax[0].scatter(dt_aod, data_aod[:,0], s=msize, label='AERONET daytime')
ax[0].scatter(dt_santa_corr, cosqm_aod_r, s=msize, label='CoSQM derived AOD')
ax[1].scatter(dt_aod, data_aod[:,2], s=msize)
ax[1].scatter(dt_santa_corr, cosqm_aod_b, s=msize)
#ax[0, 0].set_yscale('log')
ax[0].set_xlim(pd.Timestamp('2020-05-23 12'), pd.Timestamp('2020-05-24 10'))
ax[0].set_ylim(0.12,0.22)
ax[0].legend(prop={"size":10}, loc='upper left')
ax[0].text(0.5,0.10, f'{cosqm_bands[0]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[0].transAxes, fontsize=10)
ax[1].text(0.5,0.10, f'{cosqm_bands[2]} nm', horizontalalignment='center', verticalalignment='center', transform=ax[1].transAxes, fontsize=10)
fig.supxlabel('Time from midnight - UTC (h)', fontsize=10)
fig.supylabel('AOD', fontsize=10)
plt.tight_layout()
#plt.savefig('figures/continuity/continuity_aod_20200524_variance.png')

#single fit function: continuity Angstrom -> variance uncertainty
fig, ax = plt.subplots(1,1, dpi=200, figsize=(8,6))
plt.setp(ax, xticks=[pd.Timestamp('2020-05-23 17'), pd.Timestamp('2020-05-23 21'), pd.Timestamp('2020-05-24 01'), pd.Timestamp('2020-05-24 05')], xticklabels=['-7', '-3', '1', '5'])
ax.scatter(dt_aod, data_angstrom, s=msize, label='AERONET daytime derived 440-679nm AE')
ax.scatter(dt_santa_corr, cosqm_ae, s=msize, label=f'CoSQM derived {cosqm_bands[0]}-{cosqm_bands[2]}nm AE')
ax.plot(dt_aod, np.zeros(dt_aod.shape[0]), linewidth=1, linestyle='--', color='grey')
ax.set_xlim(pd.Timestamp('2020-05-23 12'), pd.Timestamp('2020-05-24 10'))
ax.set_ylim(-0.73,0.54)
ax.legend(prop={"size":10})
ax.xaxis.set_minor_locator(MultipleLocator(1))
#ax.set_yscale('log')
fig.supxlabel('Time from midnight - UTC (h)', fontsize=10)
fig.supylabel('Angstrom exponent', fontsize=10)
plt.tight_layout()
plt.savefig('figures/continuity/continuity_angstrom_20200524_variance.png')

#ZNSB for the same night
fig, ax = plt.subplots(1,1, figsize=(7,5), dpi=200)
plt.setp(ax, xticks=[pd.Timestamp('2020-05-23 22'), pd.Timestamp('2020-05-23 23'), pd.Timestamp('2020-05-24 00'), pd.Timestamp('2020-05-24 01'), pd.Timestamp('2020-05-24 02')], xticklabels=['-2', '-1', '0', '1', '2'])
cs = ['r', 'g', 'b', 'y']
ax.scatter(dt_santa, cosqm_santa[:, 1], s=msize*10, c=cs[0], marker='1',alpha=0.8, label=f'{cosqm_bands[0]} nm filtered')
ax.scatter(dt_santa_final, cosqm_santa_final[:, 1], c=cs[0], s=msize, label=f'{cosqm_bands[0]} nm')
ax.scatter(dt_santa, cosqm_santa[:, 3], s=msize*10, c=cs[2], marker='1',alpha=0.8, label=f'{cosqm_bands[2]} nm filtered')
ax.scatter(dt_santa_final, cosqm_santa_final[:, 3], c=cs[2], s=msize, label=f'{cosqm_bands[2]} nm')
ax.set_xlim(pd.Timestamp('2020-05-23 21:50'), pd.Timestamp('2020-05-24 02:10'))
ax.set_ylim(19,20.35)
ax.set_xlabel('Time from midnight - UTC (h)', fontsize=10)
ax.set_ylabel('ZNSB (MPSAS)', fontsize=10)
ax.legend(prop={"size":8}, bbox_to_anchor=(0.25, 0.2))
ax.xaxis.set_minor_locator(MultipleLocator(1))
plt.tight_layout()
#plt.savefig('figures/continuity/ZNSB_20200523.png')

# COSQM spectral responses: weighted mean of response by spectral functions of local lamp technologies (HPS, LED, MH)

cosqm_responses = pd.read_csv('spectra/sensitivity-including-atm-and-spectrum.csv', header=0)
cr = np.array([cosqm_responses[band].values/np.nanmax(cosqm_responses['Y']) for band in ['R','G','B','Y']]).T
cr_led = np.array([cosqm_responses[band].values/np.nanmax(cosqm_responses['Y-LED']) for band in ['R-LED','G-LED','B-LED','Y-LED']]).T
sky = cosqm_responses['sky'].values/np.nanmax(cosqm_responses['sky'])
sky_led = cosqm_responses['Sky-LED'].values/np.nanmax(cosqm_responses['sky'])

linewidth=1
plt.figure(figsize=(6,3.5))
plt.plot(cosqm_responses['WL'], cr[:,0], c='r', linewidth=linewidth, label='red')
plt.plot(cosqm_responses['WL'], cr[:,1], c='g', linewidth=linewidth, label='green')
plt.plot(cosqm_responses['WL'], cr[:,2], c='b', linewidth=linewidth, label='blue')
plt.plot(cosqm_responses['WL'], cr[:,3], c='y', linewidth=linewidth, label='yellow')
plt.grid(b=True)
plt.legend()
plt.xlabel('Wavelenght (nm)')
plt.ylabel('Normalized spectral signal\n in 4 CoSQM bands')
plt.xlim(400,975)
plt.ylim(0)
plt.tight_layout()
#plt.savefig('figures/spectra/cosqm_spectral_response_weighted.png')

plt.figure(figsize=(6,3.5))
plt.plot(cosqm_responses['WL'], cr_led[:,0], c='r', linewidth=linewidth, label='red')
plt.plot(cosqm_responses['WL'], cr_led[:,1], c='g', linewidth=linewidth, label='green')
plt.plot(cosqm_responses['WL'], cr_led[:,2], c='b', linewidth=linewidth, label='blue')
plt.plot(cosqm_responses['WL'], cr_led[:,3], c='y', linewidth=linewidth, label='yellow')
plt.grid(b=True)
plt.legend()
plt.xlabel('Wavelenght (nm)')
plt.ylabel('Normalized spectral signal\n in 4 CoSQM bands')
plt.xlim(400,975)
plt.ylim(0)
plt.tight_layout()
#plt.savefig('figures/spectra/cosqm_spectral_response_weighted_LED.png')

plt.figure(figsize=(6,3.5))
plt.plot(cosqm_responses['WL'], sky, linewidth=linewidth, label='HPS+MH+LED')
plt.plot(cosqm_responses['WL'], sky_led, linewidth=linewidth, label='LED')
plt.grid(b=True)
plt.legend(loc='upper right')
plt.xlabel('Wavelenght (nm)')
plt.ylabel('Normalized spectral power\n distribution of the sky ')
plt.xlim(400,975)
plt.ylim(0)
plt.tight_layout()
#plt.savefig('figures/spectra/simulated_sky_spectrum.png')


from csv import reader as CSVReader
from sys import argv
from datetime import datetime

from twisted.internet import reactor
from twisted.enterprise.adbapi import ConnectionPool

def unixTime(dt):
    """
    :param dt: datetime.
    :return: Epoch time as int.
    """
    return int((dt - datetime(1970, 1, 1)).total_seconds())

def uhh(filename):
    rows = []
    with open(filename) as f:
        for i, row in enumerate(CSVReader(f)):
            if i == 0:
                continue
            row[0] = unixTime(datetime.strptime(row[0], '%Y-%m-%d'))
            if row[21] != '':
                row[21] = row[21].split('-')
            else:
                row[21] = []
            rows.append(row)

def createdb(cursor, tablename):
    schema = 'CREATE TABLE {0}' \
        '(Date INTEGER PRIMARY KEY,' \
        'MaxTemperatureF INTEGER,' \
        'MeanTemperatureF INTEGER,' \
        'MinTemperatureF INTEGER,' \
        'MaxDewPointF INTEGER,' \
        'MeanDewPointF INTEGER,' \
        'MinDewpointF INTEGER,' \
        'MaxHumidity INTEGER,' \
        'MeanHumidity INTEGER,' \
        'MinHumidity INTEGER,' \
        'MaxSeaLevelPressureIn FLOAT,' \
        'MeanSeaLevelPressureIn FLOAT,' \
        'MinSeaLevelPressureIn FLOAT,' \
        'MaxVisibilityMiles INTEGER,' \
        'MeanVisibilityMiles INTEGER,' \
        'MinVisibilityMiles INTEGER,' \
        'MaxWindSpeedMPH INTEGER,' \
        'MeanWindSpeedMPH INTEGER,' \
        'MaxGustSpeedMPH INTEGER,' \
        'PrecipitationIn FLOAT,' \
        'CloudCover INTEGER,' \
        'Events TEXT,' \
        'WindDirDegrees INTEGER)'
    cursor.execute(schema.format(tablename))

if __name__ == '__main__':
    dbpool = ConnectionPool('sqlite3', 'Weather.sqlite3')
    dbpool.runInteraction(createdb, 'DCA_2015')
    # uhh(argv[1])
    reactor.run()


from django.db import models
from django.contrib.auth.models import User
from django.core.files.storage import default_storage as storage
from maintenance.models import Rental, MaintRequest
from PIL import Image, ImageOps

# have username in uploaded image path
def upload_to(instance, filename):
    return "users/%s/%s" % (instance.user.username.lower(), filename)

class Profile(models.Model):
    """
    Extend django user model and add custom fields
    """

    # Creates relationship to user model
    user = models.OneToOneField(User, on_delete=models.CASCADE)

    # custom fields
    rental = models.ForeignKey(Rental, null=True, on_delete=models.SET_NULL)
    register_as = models.CharField(max_length=10, null=True)

    profile_image = models.ImageField(default='users/default_profile.jpg', upload_to=upload_to)

    def __str__(self):
        return f'{self.user.username} Profile'

    # resize uploaded profile img
    def save(self, *args, **kwargs):
        """
        - installed 'django-cleanup' to auto-remove old image.
        - installed 'pillow' to resize larger images.
        """
        super(Profile, self).save(*args, **kwargs)
        if self.profile_image != 'users/default_profile.jpg':
            img = Image.open(self.profile_image)
            size = 300
            thumb = (size, size)
            method = Image.ANTIALIAS
            center = (0.5, 0.5)
            extension = "png"
            # if greater than 300px on any side, then resize it to 300x300
            if img.height > size or img.width > size:
                img.thumbnail((size, size), method)
                center_img = ImageOps.fit(img, thumb, method, centering=center)
                temp = storage.open(self.profile_image.name, "w")
                center_img.save(temp, extension)
                temp.close()
                super(Profile, self).save(*args, **kwargs)


#default positional arguments
def addition (a,b):
    return a + b

if __name__ == '__main__':
    a = 2
    b = 4
    print(addition(a,b))

#key worded arguments
def multiplication (a,b ):
    return a * b

if __name__ == '__main__':
    a = 4

    print(multiplication(a,b = 2))

#*args arguments
def division(*args):
    a,b = args
    return a/b

if __name__ == '__main__':
    a = 10
    b = 5
    print(division(a,b))

def factorial(n):

    i = 0
    result = 1
    while(i != n):
        i = i + 1
        result = result * i
    return result

if __name__ == '__main__':
    n = 5
    print(factorial(n))

def substraction(a,b):
    return a - b

if __name__ == '__main__':
    print(substraction(a=8,b=4))

def even_odd(c):

    if (c % 2) == 0:
        print(c,"is even number" )
    else:
        print(c,"is odd number")

if __name__ == '__main__':
    c = 4
    even_odd(c)


from .parser import OrmParser
import traceback

class OrmHandler:

    def __init__(self):
        self.error_list = []
        self.affected_object = []
        self.parser = OrmParser()

    # Filtering all type of fields(one to one, one to many,many to one)

    def delete_functionality(self, object_to_delete, deletion_set, parent_set_dict, exception_list):
        try:
            # Filtering one to many fields
            all_related_fields = self.parser._get_all_object_field(object_to_delete)
            field_list = self.parser._get_one_to_many_field(all_related_fields)

            for i in range(len(field_list)):
                to_be_called_name = field_list[i].name
                if to_be_called_name[-1] == '+':
                    to_be_called_name = to_be_called_name[0:-1]
                to_be_called_name_passed = ''.join(e for e in to_be_called_name if e.isalnum()).lower()
                field_to_test = to_be_called_name.lower() + '_set'
                if hasattr(object_to_delete, field_to_test):
                    to_be_called = getattr(object_to_delete, field_to_test)
                    data = to_be_called.count()
                    if data > 0:
                        deletion_set, parent_set_dict = self.parser._process_operation(data, object_to_delete,
                                                                                to_be_called_name_passed,
                                                                                deletion_set, parent_set_dict)
                else:
                    if (field_list[i].__dict__['related_name']):
                        to_be_called_name = field_list[i].__dict__['related_model'].__name__
                        to_be_called_name_passed = ''.join(e for e in to_be_called_name if e.isalnum()).lower()
                        if hasattr(object_to_delete, field_list[i].__dict__['related_name']):
                            to_be_called = getattr(object_to_delete, field_list[i].__dict__['related_name'])
                            if hasattr(to_be_called, 'count'):
                                data = to_be_called.count()
                                if data > 0:
                                    deletion_set, parent_set_dict = self.parser._process_operation(data,
                                                                                object_to_delete,
                                                                                to_be_called_name_passed,
                                                                                deletion_set,
                                                                                parent_set_dict)
                            else:
                                print("to-be called count check %s" % to_be_called)
                                continue
                        else:
                            if (to_be_called_name == ''):
                                continue
                            exception_list.append({
                                                      "Error message": "Data from " + to_be_called_name + " with dependencies on this will be deleted"})
                            continue
                if (to_be_called_name.lower() == object_to_delete._meta.model.__name__.lower()):
                    continue
                to_be_iterated = to_be_called.model.objects.filter(**to_be_called.core_filters)

                # Filtering many to many fields
                manytomanylist = object_to_delete._meta.__dict__['local_many_to_many']
                for i in manytomanylist:
                    to_be_called_name_passed = i.model.__name__ + '_' + i.__dict__['name']
                    to_be_called_name = i.__dict__['name']
                    if (hasattr(object_to_delete, to_be_called_name)):
                        to_be_called = getattr(object_to_delete, to_be_called_name)
                        data = to_be_called.count()
                        if data > 0:
                            if (to_be_called_name_passed in deletion_set):
                                deletion_set[to_be_called_name_passed] += data
                            else:
                                deletion_set[to_be_called_name_passed] = data

                for item_obj in to_be_iterated:
                    deletion_set, parent_set_dict = self.delete_functionality(item_obj, deletion_set, parent_set_dict,
                                                                              exception_list)

            # Filtering one to one fields
            iter_all_fields = object_to_delete._meta.fields
            one_to_one_list = []
            for i in iter_all_fields:
                if i.one_to_one:
                    if hasattr(i, 'rel') and i.rel.on_delete.__name__ != 'DO_NOTHING':
                        one_to_one_list.append(i)
                    elif hasattr(i, 'remote_field') and i.remote_field.on_delete.__name__ != 'DO_NOTHING':
                        one_to_one_list.append(i)

            for i in one_to_one_list:
                primary_id = getattr(object_to_delete, i.__dict__['column'])
                to_be_called_name = i.related_model.__name__.lower()
                if to_be_called_name in deletion_set:
                    deletion_set[to_be_called_name] += 1
                else:
                    deletion_set[to_be_called_name] = 1
                item_obj = i.__dict__['related_model'].objects.get(pk=primary_id)
                deletion_set, parent_set_dict = self.delete_functionality(item_obj, deletion_set,
                                                                          parent_set_dict,
                                                                          exception_list)
        except Exception as e:
            self.error_list.append({'Error message': str(traceback.format_exc())})
        return deletion_set, parent_set_dict

    # Building exception list of effected objects

    def get_exception_list(self, deletion_set, parent_set_dict, exception_list):

        for key, val in list(deletion_set.items()):
            model_display_name = key
            message = "This object has been used in " + key + " " + str(
                val) + " times."
            parent_models = ''
            if (key in parent_set_dict):

                def get_parents(parent_set_dict, key):
                    if key in parent_set_dict:
                        return parent_set_dict[key]
                    return ''

                def get_list_for_list(parent_set_dict, key_list):
                    list_a = []
                    for i in key_list:
                        list_b = get_parents(parent_set_dict, i)
                        list_a = list(set(list_a).union((set(list_b) - set(list_a))))
                    return list_a

                def string_a_list(list_a):
                    parent_str = ''
                    for i in list_a:
                        if (parent_str == ''):
                            parent_str = i
                        else:
                            parent_str = parent_str + ',' + i
                    return parent_str

                parent_models = string_a_list(parent_set_dict[key])
                other_list = []
                query_list = parent_set_dict[key]
                while (1):
                    list_to_append = get_list_for_list(parent_set_dict, query_list)
                    if (other_list == []):
                        other_list.append(list_to_append)
                        query_list = list_to_append
                    elif (set(list_to_append) - set(other_list[-1])):
                        other_list.append(list_to_append)
                        query_list = list_to_append

                    else:
                        break
                for i in other_list:
                    if (string_a_list(i) != ''):
                        parent_models = parent_models + ' which is under ' + string_a_list(i)

            exception_list.append(
                {"message": message, "model_name": model_display_name, "Parent models": parent_models, "count": val})

        return exception_list

    # Deletion of one to one objects

    def delete_one_to_one(self, delete_element, id):
        element_fields = delete_element._meta.__dict__['fields']
        for field in element_fields:
            if field.one_to_one:
                parent_model = field.__dict__['related_model']
                parent_id = getattr(delete_element, field.column)
                parent_element = parent_model.objects.filter(id=parent_id).first()
                parent_element.delete()

    # Processing deletion based on parameters passed

    def delete_service(self, data, handler, exception_list):
        try:
            deletion_set = {}
            parent_set_dict = {}
            id = data['id'] if 'id' in data else None
            force_delete = data['force_delete'] if 'force_delete' in data else False
            object_to_delete = handler.objects.filter(id=id)
            delete_element = object_to_delete.first()
            if not delete_element:
                raise Exception(str(delete_element) + ' element not found with id %s' % (id))
            element_fields = delete_element._meta.__dict__['fields']
            flag = 0
            for field in element_fields:
                if field.one_to_one:
                    if hasattr(i, 'rel') and i.rel.on_delete.__name__ != 'DO_NOTHING':
                        flag = 1
                    elif hasattr(i, 'remote_field') and i.remote_field.on_delete.__name__ != 'DO_NOTHING':
                        flag = 1
            if force_delete:
                if flag == 1:
                    self.delete_one_to_one(delete_element, id)
                else:
                    object_to_delete.delete()
            else:
                deletion_set, parent_set_dict = self.delete_functionality(delete_element, deletion_set, parent_set_dict,
                                                                          exception_list)
                self.affected_object = self.get_exception_list(deletion_set, parent_set_dict, exception_list)
        except Exception as e:
            self.error_list.append({'Error message': str(e)})
        return self.error_list, self.affected_object

    # Updating foreign/Many-to-Many field dependencies from one to another object

    def update_dependencies(self, source, destination):
        try:
            all_related_fields = self.parser._get_all_object_field(source)
            field_list = self.parser._get_one_to_many_field(all_related_fields)
            for iter in range(0, len(field_list)):
                to_be_called_name = field_list[iter].name
                if to_be_called_name[-1] == '+':
                    to_be_called_name = to_be_called_name[0:-1]
                field_to_test = to_be_called_name.lower() + '_set'
                field_name = None
                if hasattr(source, field_to_test):
                    reference_objs = getattr(source, field_to_test)
                    ids = reference_objs.values_list('id')
                    for iter_in in reference_objs.model._meta._get_fields():
                        if hasattr(iter_in, 'related_model') and iter_in.related_model == source.__class__:
                            field_name = iter_in.column
                    for iter_in in ids:
                        reference_objs.model.objects.filter(id=iter_in[0]).update(**{field_name: destination.id})
                else:
                    if field_list[iter].__dict__['related_name']:
                        if hasattr(source, field_list[iter].__dict__['related_name']):
                            reference_objs = getattr(source, field_list[iter].__dict__['related_name'])
                            ids = reference_objs.values_list('id')
                            for iter_in in reference_objs.model._meta._get_fields():
                                if hasattr(iter_in, 'related_model') and (
                                        (hasattr(iter_in.related_model, 'model') and
                                         iter_in.related_model.model == source.__class__) or
                                        (hasattr(iter_in.related_model, '_meta') and
                                         hasattr(iter_in.related_model._meta, 'model')
                                         and iter_in.related_model._meta.model == source.__class__)):
                                    field_name = iter_in.column
                            for iter_in in ids:
                                reference_objs.model.objects.filter(id=iter_in[0]).update(
                                    **{field_name: destination.id})
                    else:
                        continue
                if (to_be_called_name.lower() == source._meta.model.__name__.lower()):
                    continue

                to_be_iterated = reference_objs.model.objects.filter(**reference_objs.core_filters)
                manytomany = source._meta.__dict__['local_many_to_many']
                for iter in manytomany:
                    to_be_called_name = iter.__dict__['name']
                    if hasattr(source, to_be_called_name):
                        reference_objs = getattr(source, to_be_called_name)
                        field_name = source._meta.model_name + '_id'
                        ids = reference_objs.through.objects.filter(**{field_name: source.id}).values_list('id')
                        for iter_in in ids:
                            reference_objs.through.objects.filter(id=iter_in[0]).update(**{field_name: destination.id})

                for item_obj in to_be_iterated:
                    self.update_dependencies(item_obj, destination)
        except Exception as e:
            self.error_list.append({'Error message': str(e)})
        return self.error_list


from statistics import mean, stdev
import json

def handle_result(results):
  for alg in results:
    alg_exec_times = []

    for seq_len in results[alg]:
      seq_len_times = []

      for seq_type in results[alg][seq_len]:
        seq_type_times = []

        for exec_res in results[alg][seq_len][seq_type]['all_results']:
          if isinstance(exec_res, list):
            for add_params_res in exec_res:
              exec_time = add_params_res['exec_time']
              alg_exec_times.append(exec_time)
              seq_len_times.append(exec_time)
              seq_type_times.append(exec_time)
          else:
            exec_time = exec_res['exec_time']
            alg_exec_times.append(exec_time)
            seq_len_times.append(exec_time)
            seq_type_times.append(exec_time)

        results[alg][seq_len][seq_type]['exec_times'] = seq_type_times
        results[alg][seq_len][seq_type]['average_time'] = mean(seq_type_times)
        results[alg][seq_len][seq_type]['std_dev'] = stdev(seq_type_times)

      results[alg][seq_len]['exec_times'] = seq_len_times
      results[alg][seq_len]['average_time'] = mean(seq_len_times)
      results[alg][seq_len]['std_dev'] = stdev(seq_len_times)

    results[alg]['exec_times'] = alg_exec_times
    results[alg]['average_time'] = mean(alg_exec_times)
    results[alg]['std_dev'] = stdev(alg_exec_times)

  print("Writing result to simulation_results.json")

  with open('simulation_results.json', 'w') as outfile:
    json.dump(results, outfile, indent=2)


import numpy as np

from .base import MultivariateOutlierGenerator

class MultivariateShiftOutlierGenerator(MultivariateOutlierGenerator):
    def __init__(self, timestamps=None, factor_range=(8, 10)):
        if factor_range[0] >= factor_range[1]:
            raise ValueError('First value must be less than second value')
        timestamps = timestamps or []
        self.timestamps = timestamps
        self.min_factor = factor_range[0]
        self.factor_width = factor_range[1] - factor_range[0]

    def add_outliers(self, timeseries):
        additional_values = np.zeros(timeseries.size)
        for start, end in self.timestamps:
            local_std = timeseries.iloc[max(0, start - 10):end + 10].std()
            random_factor = np.random.random() * self.factor_width + self.min_factor
            additional_values[list(range(start, end))] += np.random.choice([-1, 1]) * random_factor * local_std
        return additional_values


from django.conf.urls import url
from . import views

urlpatterns= [
    url(r'^dashboard$', views.dashboard, name="dashboard"),
    url(r'^wish_items/(?P<id>\d+)$', views.show_item, name="show_item"),
    url(r'^wish_items/create$', views.create, name="create"),
    url(r'^logout$', views.logout),
    url(r'^add$', views.validate_item),
    url(r'^addtomy$', views.addtomy),
    url(r'^del$', views.delete),
    url(r'^search$', views.search)
]


from django.conf.urls import url

from . import views

app_name = 'portfolio'
urlpatterns = [
    # ex: /
    url(r'^$', views.IndexView.as_view(), name='index'),
    # ex: /document/5/
    url(r'^document/(?P<pk>[0-9]+)/$', views.DocumentView.as_view(), name='document'),
    # ex: /audio/5/
    url(r'^audio/(?P<pk>[0-9]+)/$', views.AudioView.as_view(), name='audio'),
]


import requests
from bs4 import BeautifulSoup
import re
import json
from SQL import save_mysql

class TianMaoMall:
	def __init__(self,url,id,soup,referer):
		self.platform="淘宝"
		self.id=id
		self.address=url+id
		self.headers={
			"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
			"referer":referer}

		self.pageSoup = soup
		self.description=None
		self.sql = save_mysql()

	#获取标题
	def __getDescription(self):
		if self.pageSoup is not None:
			title=self.pageSoup.find("div",class_="tb-detail-hd").find("h1").string.strip()
			return title
		else:
			print("获取标题出错了，soup不应该为空")
			return None

	#获取商品图片链接
	def __getImages(self):
		if self.pageSoup is not None:
			results=[]
			imgs=self.pageSoup.find("ul",attrs={"id":"J_UlThumb"}).find_all("li")
			num=0
			for img in imgs:
				results.append("https:"+img.find('img')['src'])
			while num < 5:
				results.append("null")
				num+=1
			return results
		else:
			print("获取图片出错了，soup不应该为空")
			return None

	#获取产品名称、颜色分类、保质期、功效、适合肤质、产地
	def __getProductDetail(self):
		if self.pageSoup is not None:
			params=["产品名称","颜色分类","保质期","功效","适合肤质","产地"]
			results={}
			for param in params :
				results[param]="null"
			try:
				attributes = self.pageSoup.find('ul',attrs={"id":"J_AttrUL"}).find_all("li")
				for attr in attributes:
					if '\xa0' in attr:
						attr=attr.replace(u'\xa0',u'')
					attr = attr.string.strip()
					for i in range(len(params)):
						strIndex = re.search(params[i],attr)
						if strIndex is not None:
							end = strIndex.span()[1]  #因可能出现“化妆品保质期：36个月”这种情况，所以需要知道“保质期”的最后匹配位置，然后截取，才能得到干净的字段
							#results[ params[i] ] = attr[len(params[i])+1:]
							results[ params[i] ] = attr[end+1:]
							del params[i]
							break
			except AttributeError as e:
				print(e)

			return results
		else :
			return None

	#获取每一种型号的价格
	def __getPrices(self):
		url="https://mdskip.taobao.com/core/initItemDetail.htm?&itemId={}".format(self.id)
		header={
			'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
			'Referer':"https://detail.tmall.com/item.htm?id={}".format(self.id)  #这里应对阿里云服务器的反爬虫策略
		}
		html=requests.get(url,headers=header).text
		#print(html)
		try:
			datas=json.loads(html.strip())
		except(JSONDecodeError,Exception):
			print("JSONDecodeError")
			return None
		try :
			prices = datas['defaultModel']['itemPriceResultDO']['priceInfo']
			keys = list(prices.keys())
			result = None
			if keys is None:
				raise Exception('NullException')
			key = keys[0]
			results=prices[key]["promotionList"][0]['price']
			return results
		except (KeyError,Exception):
			prices = datas['defaultModel']['itemPriceResultDO']['priceInfo']
			keys = list(prices.keys())
			results=None
			key=keys[0]
			results=prices[key]['price']
			return results

	#获取商品评价总数和好评率
	def __getComments(self):
		comments={}
		url="https://dsr-rate.tmall.com/list_dsr_info.htm?itemId={}".format(self.id)
		html=requests.get(url).text
		tmp = re.search("\(.*\)",html).span()
		html = html[tmp[0]:tmp[1]][1:-1]
		datas=json.loads(html)
		comments['评论总数']=datas['dsr']['rateTotal']
		comments['好评率'] = round(float(datas['dsr']['gradeAvg'])/5*100,2)
		print(comments)
		return comments

	def __saveInDataBase(self,images,address,price,description,number,name,produce_address,comment_count,good_for_who,expiration_date,color,result_effectiveness,good_comment_percentage):
		table = "product_lipstick"
		sqlString = "insert into "+table+" ("+"first_category,second_category,third_category,"+"img1_address,img2_address,img3_address,img4_address,img5_address,"+"address,price,description,number,name,produce_address,"+"comment_count,good_for_who,expiration_date,color,"+"result_effectiveness,category,good_comment_percentage,who_handly,platform"+") values('美妆/个人护理','彩妆','唇彩','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','唇彩','{}','林鹏珊','天猫')".format(images[0],images[1],images[2],images[3],images[4],address,price,description,number,name,produce_address,comment_count,good_for_who,expiration_date,color,result_effectiveness,good_comment_percentage)
		self.sql.save_product(sqlString)

	def main(self):
		images=self.__getImages()
		address = self.address
		price=float(self.__getPrices())
		description=self.__getDescription()
		number = self.id
		detail=self.__getProductDetail()
		name = detail["产品名称"]
		produce_address = detail["产地"]
		good_for_who = detail["适合肤质"]
		expiration_date = detail["保质期"]
		color = detail["颜色分类"]
		result_effectiveness = detail["功效"]
		comments=self.__getComments()
		comment_count = comments['评论总数']
		good_comment_percentage = comments["好评率"]

		self.__saveInDataBase(images,address,price,description,number,name,produce_address,comment_count,good_for_who,expiration_date,color,result_effectiveness,good_comment_percentage)
		'''
		print(description,end="\n\n")
		print(images,end='\n\n')
		print(detail,end='\n\n')
		print(price,end='\n\n')
		print(comments,end='\n\n')
		'''


name = ["刘备", "关羽", "张飞"]
name.insert(1, "阿斗")  # 在这个索引之前插入
print(name)

# name.insert(3, ("关平", "关索"))  # 将对象整体插入到指定位置
# name[3:3] = ["关平", "关索"]  # 将对象的元素，读取出来，插入到空切片锁定的位置
name[3:3] = ("关平", "关索")  # 将对象的元素，读取出来，插入到空切片锁定的位置
# print(name)

name.append([["张苞","张绍"],["x","y"]])
print(name)


#! -*- coding: utf-8 -*-
import numpy as np
import pystan

np.random.seed(123)
N1 = 30
N2 = 20
Y1 = np.random.normal(loc=0.0, scale=5.0, size=N1)
Y2 = np.random.normal(loc=1.0, scale=4.0, size=N2)

data = {'N1': N1, 'N2': N2, 'Y1': Y1, 'Y2': Y2}

stanmodel = pystan.StanModel(file='ex5.stan')
fit = stanmodel.sampling(data=data, seed=1234)
ms = fit.extract()

print(np.mean(ms['mu1'] < ms['mu2']))
# library(rstan)

# source('generate-data.R')

# data <- list(N1=N1, N2=N2, Y1=Y1, Y2=Y2)
# fit <- stan(file='ex5.stan', data=data, seed=1234)

# ms <- extract(fit)
# prob <- mean(ms$mu1 < ms$mu2)  #=> 0.9457


from django.db import models
from django.utils import timezone

import os
import uuid
from PIL.Image import Image
from hackathon.settings import MEDIA_ROOT

# Create your models here.

def get_image_path(self, filename):
    """
    カスタマイズした画像pathを取得する
    UUIDにして画像ファイル名をhash化
    :param self: インスタンス(model)
    :param filename:  元の画像ファイル
    :return: カスタマイズしたファイル名を含む画像パス
    """

    prefix = 'images/'
    name = str(uuid.uuid4()).replace('-', '')
    extension = os.path.splitext(filename)[-1]

    return prefix + name + extension

def delete_previous_file(function):
    """
    不要となる古い画像ファイルを削除する為のデコレータ実装
    :param function: メイン関数
    :return: wrapper
    """

    def wrapper(*args, **kwargs):
        self = args[0]
        #保存前の画像ファイル名取得
        result = Post.objects.filter(pk=self.pk)
        previous = result[0] if len(result) else None
        super(Post, self).save()
        #関数実行
        result = function(*args, **kwargs)
        #保存前のファイルがあったら削除
        if previous:
            os.remove(MEDIA_ROOT + '/' + previous.image.name)
        return result
    return wrapper

class Post(models.Model):

    GENDER_CHOICES = (
        (u'M', u'Men'),  #(データベースに保存される値, 管理インターフェイスで表示される値)
        (u'W', u'Women'),
    )
    CATEGORY_CHOICES = (
        (u'outdoor', u'アウトドア'),
        (u'amekaji', u'アメカジ'),
        (u'kuranji', u'クランジ'),
        (u'surf', u'サーフ'),
        (u'salon', u'サロン'),
        (u'street', u'ストリート'),
        (u'sports', u'スポーツMIX'),
        (u'toratto', u'トラット'),
        (u'noomukoa', u'ノームコア'),
        (u'business', u'ビジネス'),
        (u'hiphop', u'ヒップホップ'),
        (u'host', u'ホスト'),
        (u'mode', u'モード'),
        (u'rock', u'ロック'),
        (u'onee', u'お姉'),
        (u'girlish', u'ガーリッシュ'),
        (u'gyaru', u'ギャル'),
        (u'kireime', u'きれいめカジュアル'),
        (u'gosurori', u'ゴスロリ'),
        (u'marine', u'マリン'),
        (u'harajuku', u'原宿'),
        (u'morigirl', u'森ガール'),
    )
    TEMP_CHOICES = (
        (u'~5', u'~5℃'),
        (u'6~10', u'6~10℃'),
        (u'11~15', u'11~15℃'),
        (u'16~20', u'16~20℃'),
        (u'21~25', u'21~25℃'),
        (u'26~30', u'26~30℃'),
        (u'31~', u'31℃~'),
    )
    PURPOSE_CHOICES = (
        (u'date', u'デート'),
        (u'party', u'パーティ'),
        (u'outdoor', u'アウトドア'),
        (u'business', u'ビジネス'),
        (u'nochoice', u'選択なし'),
    )
    created_date = models.DateTimeField(default = timezone.now)
    gender = models.CharField(max_length = 2, choices = GENDER_CHOICES)
    brand = models.CharField(max_length = 30)
    category = models.CharField(max_length = 10, choices = CATEGORY_CHOICES)
    temp = models.CharField(max_length = 8, choices = TEMP_CHOICES)
    purpose = models.CharField(max_length = 10, choices = PURPOSE_CHOICES)
    image = models.ImageField(upload_to=get_image_path)
    comment = models.TextField(blank = True)

    og_image = image.upload_to

    @delete_previous_file
    def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
        super(Post, self).save()

    @delete_previous_file
    def delete(self, using=None, keep_parents=False):
        super(Post, self).delete()

    def __str__(self):
        return u'Post Object' + str(self.id)

    def admin_og_image(self):
        if self.og_image:
            return '<img src="{}" style="width:100px;height:auto;">'.format(self.og_image)
        else:
            return 'no image'

    admin_og_image.allow_tags = True


from __future__ import print_function
from objplot import plotsave
import numpy as np
import theano
import theano.tensor as T
import theano.sandbox.cuda.basic_ops as sbcuda
import lasagne

import sys
import os

from neat import nn, population, statistics

sys.path.append(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(os.path.join(sys.path[0],'VRN'))

from utils import checkpoints
import imp

np.set_printoptions(threshold=np.nan)

# Define the testing functions
def make_testing_functions():
    # Input Array
    X = T.TensorType('float32', [False] * 5)('X')
    # # Shared Variable for input array
    # X_shared = lasagne.utils.shared_empty(5, dtype='float32')
    # Output layer
    l_out = model['l_out']
    # Get output
    y_hat_deterministic = lasagne.layers.get_output(l_out, X, deterministic=True)
    prob = T.sum(y_hat_deterministic, axis=0)
    # Compile Functions
    obj_fun = theano.function([X], [prob])
    return obj_fun

# Load config module
basepath=str(os.path.dirname(os.path.realpath(__file__)))
configure_path = basepath+'/VRN/Discriminative/VRN.py'
data_path = basepath+'/VRN/datasets/CPPNGenerated'
config_module = imp.load_source('config', configure_path)
cfg = config_module.cfg
# Find weights file
weights_fname = str(configure_path)[:-3] + '.npz'
# Get Model
model = config_module.get_model()
# Load weights
metadata = checkpoints.load_weights(weights_fname, model['l_out']) # model weights changed after this...inside load_weights, another model file is read from weights_fname

# Compile functions
print('Compiling theano functions...')
obj_fun = make_testing_functions()

#CPPN
inp=[0]*(32*32*32)
n=0
for i in range(0, 32):
    for j in range(0, 32):
        for k in range(0, 32):
            inp[n]=(i,j,k)
            n=n+1

def get_fitness(g, inp, CLASS):
    net = nn.create_feed_forward_phenotype(g)
    outputarray = [0]*32*32*32
    for inputs in inp:
        output = net.serial_activate(inputs)
        outputarray[inputs[0]+inputs[1]*32+inputs[2]*32*32] = output[0]
    outputarray = np.reshape(outputarray,(32,32,32))
    threshold=.5
    outputarray[outputarray<threshold]=-1
    outputarray[outputarray>=threshold]=3
    temp = np.zeros((1,1,32,32,32),dtype=np.float32)
    temp[0,0,:,:,:] = outputarray
    pred = obj_fun(temp)
    pred = pred[0]
    pred[:CLASS]=np.absolute(pred[:CLASS])          #make all but CLASS positive (yes? no?)
    pred[(CLASS+1):]=np.absolute(pred[(CLASS+1):])  #???
    fitness = (pred[CLASS]-min(pred))/(np.sum(pred-min(pred)))
    if ((fitness>.2) and (os.path.isfile(os.path.join(data_path, "class_{0}_fitness_{1:.4f}".format(CLASS, fitness))))==False):
        filename = "class{0}_fitness{1:.4f}".format(CLASS, fitness)
        #Save data
        np.savez(os.path.join(data_path,filename+'.npz'),**{'features': temp, 'targets': [CLASS]})
        #Save object image
        plotsave(outputarray,os.path.join(data_path,filename+'.png'))
        #Save Prediction array as text file
        file = open(os.path.join(data_path,filename+'.txt'),'w')
        np.savetxt(file, pred)
        file.close
    return fitness

def eval_fitness(genomes):
    for g in genomes:
       g.fitness = get_fitness(g, inp, 0)

local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'main_config')
pop = population.Population(config_path)
pop.run(eval_fitness, 10000)


try:
    from celery.utils import fun_takes_kwargs
except:
    fun_takes_kwargs = None
try:
    from functools import wraps
except:
    from django.utils.functional import wraps

from johnny.cache import get_backend, local

def johnny_task_wrapper(f):
    """
    Provides a task wrapper for celery that sets up cache and ensures
    that the local store is cleared after completion
    """
    if fun_takes_kwargs is None:
        return f
    @wraps(f)
    def newf(*args, **kwargs):
        backend = get_backend()
        was_patched = backend._patched
        get_backend().patch()
        #since this function takes all keyword arguments,
        #we will pass only the ones the function below accepts, just as celery does
        supported_keys = fun_takes_kwargs(f, kwargs)
        new_kwargs = dict((key, val) for key, val in kwargs.items()
                                if key in supported_keys)

        try:
            ret = f(*args, **new_kwargs)
        finally:
            local.clear()
        if not was_patched:
            get_backend().unpatch()
        return ret
    return newf