# -*- coding: utf-8 -*-
"""
@author: slawler
"""
import pandas as pd
import h5py
import requests
import json
from datetime import datetime
from collections import OrderedDict, defaultdict
import string
import requests
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
import os
from pathlib import Path
from scipy.integrate import trapz, cumtrapz, simps
import numpy as np
import os
from glob import glob
from IPython.display import Markdown, display

#-----------------------------------------------------------------------------#
#--Notebook Functions
#-----------------------------------------------------------------------------#

def printbold(string):
    # Jupyter Notebook function to print bold
    display(Markdown('**'+string+'**'))

#-----------------------------------------------------------------------------#
#--Data Initialization & Manipulation Functions
#-----------------------------------------------------------------------------#

def initialize_test_case(plot=False):
    #--Read in gage data for the Michigan Test Case
    data_dir = os.path.join(str(Path(os.getcwd()).parents[0]),'sample_data')
    tsvs = glob(os.path.join(data_dir, '04119000*'))
    printbold("Read in USGS Gage Records")
    df = MergeDatsets(tsvs)
    return df, data_dir

def initialize(plot=False):
    #--Read in gage data for this region (gages = 015*)
    data_dir = os.path.join(str(Path(os.getcwd()).parents[0]),'sample_data')
    tsvs = glob(os.path.join(data_dir, '015*'))
    metadata = os.path.join(data_dir ,'gage_metadata.tsv')
    df = MergeDatsets(tsvs)
    return df, data_dir

def MergeDatsets(tsvs):
    #--Read in all tsvs and create a single dataframe (pass a list of tsvs)
    f = tsvs[0]           # Used the first data file to initialize the dataframe
    flow_type = f[-9:].split('.tsv')[0]
    print(os.path.basename(f))

    if flow_type == '60_dv':
        df = read_usgs_60_dv(f)

    elif flow_type == '60_iv':
        df = read_usgs_60_iv(f)

    elif flow_type == '65_iv':
        df = read_usgs_65_iv(f)

    else:
        print("Error")

    for f in tsvs[1:]:
        print(os.path.basename(f))
        data = f[-9:].split('.tsv')[0]
        gage =os.path.basename(f).split('.')[0]

        if data == '60_dv':
            df1 = read_usgs_60_dv(f)
            df = pd.merge(df, df1, how='outer', left_index=True, right_index=True)

        elif data == '60_iv':
            df1 = read_usgs_60_iv(f)
            df = pd.merge(df, df1, how='outer', left_index=True, right_index=True)

        elif data == '65_iv':
            df1 = read_usgs_65_iv(f)
            df = pd.merge(df, df1, how='outer', left_index=True, right_index=True)

        else:
            continue

    return df

def read_usgs_60_dv(f):
    #--Read in to dataframe, and reformat daily flow records
    gage = os.path.basename(f).split('.')[0]
    df = pd.read_csv(f, sep='\t')
    df = df.set_index(pd.DatetimeIndex(df['Date'])+ pd.Timedelta('12 hours'))
    df.rename(columns = {'Flow':gage}, inplace=True)
    df.drop(labels=['Date', 'agency_cd', 'site_no', 'Flow_cd'], axis = 1, inplace=True)
    return df

def read_usgs_60_iv(f):
    #--Read in to dataframe, and reformat instantaneous flow records
    gage = os.path.basename(f).split('.')[0]
    df = pd.read_csv(f, sep='\t')
    df = df.set_index(pd.DatetimeIndex(df['dateTime']))
    df.rename(columns = {'Flow_Inst':gage}, inplace=True)
    df.drop(axis=1,labels =['dateTime','agency_cd','site_no', 'Flow_Inst_cd', 'tz_cd'], inplace=True)
    return df

def read_usgs_65_iv(f):
    #--Read in to dataframe, and reformat instantaneous stage records
    gage = os.path.basename(f).split('.')[0]
    df = pd.read_csv(f, sep='\t')
    df = df.set_index(pd.DatetimeIndex(df['dateTime']))
    df.rename(columns = {'GH_Inst':gage}, inplace=True)
    df.drop(axis=1,labels =['dateTime','agency_cd','site_no', 'GH_Inst_cd', 'tz_cd'], inplace=True)
    return df

#-----------------------------------------------------------------------------#
#--USGS API-like Functions
#-----------------------------------------------------------------------------#

def Get_USGS_Peaks(gage):
    #--Get peaks for a gage, may be sensitive to skiprows value. (check if superceded by GetPKFQ below)
    url = 'https://nwis.waterdata.usgs.gov/ny/nwis/peak?site_no={}&agency_cd=USGS&format=rdb'.format(gage)
    df = pd.read_csv(url, skiprows=64, sep = '\t')
    df.drop(0, axis=0, inplace=True)
    y = df['peak_va'].astype(float)
    x = df['peak_dt']
    x = pd.to_datetime(x, format= '%Y-%m-%d',  errors='coerce')
    y.index = x
    return y

def GetPKFQ(gage):
    #--Need to test this function (see Get_USGS_Peaks)
    url = 'https://nwis.waterdata.usgs.gov/ny/nwis/peak?site_no={}&agency_cd=USGS&format=hn2'.format(gage)
    pkf = pd.read_csv(url)
    pkf.to_csv('return_periods\\{}.pkf'.format(gage), sep='\t', index=False)
    print('{} Data Saved in return_periods'.format(gage))

def GotoUSGS(state):
    #--Pass state initials (e.g. 'NY'), function opens gage catalogue in browser
    #--URL stirng broken into parts for ease of reading in text editor
    part1 = 'https://waterdata.usgs.gov/nwis/uv?referred_module=sw&state_cd={}'.format(state)
    part2 = '&site_tp_cd=OC&site_tp_cd=OC-CO&site_tp_cd=ES&site_tp_cd=LK&site_tp_cd=ST&'
    part3 = 'site_tp_cd=ST-CA&site_tp_cd=ST-DCH&site_tp_cd=ST-TS&format=station_list'
    url = part1 + part2 + part3
    print("\nCLICK HERE FOR USGS GAGES: \n", url)
    print("\nCLICK HERE FOR MAP: \n", 'https://maps.waterdata.usgs.gov/mapper/index.html')

#-----------------------------------------------------------------------------#
#--Math Functions
#-----------------------------------------------------------------------------#

def Q_to_Stage(usgs_gage_id, poly_order = 3):
    #--Use USGS rating curve to convert flow to stage (where available)
    url = r'https://waterdata.usgs.gov/nwisweb/get_ratings?site_no={}&file_type=exsa'.format(usgs_gage_id)
    cols = ['INDEP', 'SHIFT','DEP','STOR']
    usgs_data = pd.read_csv(url, skiprows = 38, sep = '\t', names = cols)

    xs = np.array(usgs_data['DEP'])
    ys = np.array(usgs_data['INDEP'])

    coefs    = np.polyfit(xs,ys,poly_order)
    polynomial = np.poly1d(coefs)

    return polynomial

def IntegrateHydrograph(timeseries, da_sqft, method = 'trapezoid'):
    from scipy import integrate
    #--Integrate the volume under a hydrograph
    '''Flow should be given in units of cfs, index should be time series'''
    data = timeseries.resample('1S').asfreq().interpolate()
    da_acres = 640*da_sqft

    if method == 'simpson':
        volume_acre_inches = volume*0.00027548209085905
        inches = volume_acre_inches/da_acres
        volume = integrate.simps(np.array(data.values), x=None, dx=1, axis=0, even='avg')
    else:
        volume = trapz(np.array(data.values), x=None, dx=1.0, axis=0)
        volume_acre_inches = volume*0.00027548209085905
        inches = volume_acre_inches/da_acres

    return round(float(inches), 2)

def GetBreachFlow(base_hydrograph, site, rasdata, station, breach_point, breach_elev, data_dir, date_int=6, poly_order = 3):
    #--Calls ComputeWeirFlow to calculate weir flow & plot (would benefit from partitioning into simpler functions)
    df = GetRasData(rasdata, station)

    # Plot a Rating Curve using Stage & Flow data
    StageDischargePlot(df, figsize=(5,3))
    polyfit  = np.polyfit(df['flow'],df['stage'],poly_order)
    polyfitline = np.poly1d(polyfit)
    hydro =pd.DataFrame(base_hydrograph)
    colname = hydro.columns
    hydro.rename(columns ={colname[0]:'Base Hydrograph'}, inplace=True)
    stage = plotcomp(hydro, polyfitline,breach_elev)
    df_weir = ComputeWeirFlow(stage, breach_elev, date_int)
    output_csv = os.path.join(data_dir, '{}_BreachData_{}_location_{}.tsv'.format(site, station, breach_point))
    df_weir.to_csv(output_csv, sep = '\t')

    printbold('\nInflow Data for Breach Location: ')
    print(output_csv)

def ComputeWeirFlow(df, breach_height, date_int, weir_coeff=2.0, breach_length=250, plot=True):
    #--Compute & plot weir flow at given HEC-RAS cross section, return a plot-series
    import matplotlib.dates as mdates
    min_stage = float(df.stage.min())
    df['head'] = df['stage']-breach_height
    df = df.query('head > 0 or head == 0')
    df['weir_flow'] = weir_coeff*breach_length*df['head']**(2/3)
    if plot:
        f, (ax1, ax2, ax3) = plt.subplots(1,3, figsize = (12,5))

        ax1.plot(df['stage'],color = 'green',label = 'River Stage (ft) at Breach Location')
        ax1.grid()
        ax1.legend(loc= 'best',  fontsize='x-small')

        ax2.set_title('Normalized Hydrographs based on Breach Elevation \n')
        ax2.plot(df['head'],color = 'green',label = 'Head (ft) at Breach Location')
        ax2.grid()
        ax2.legend(loc= 'best',  fontsize='x-small')

        ax3.plot(df['weir_flow'],color = 'blue',label = 'Weir Flow (cfs)')
        ax3.grid()
        ax3.legend(loc= 'best',  fontsize='small')

        days = mdates.HourLocator(interval = date_int)   # every year
        dateFmt = mdates.DateFormatter('%Y-%m-%d :%H:%M')
        ax1.xaxis.set_major_formatter(dateFmt)
        ax1.xaxis.set_major_locator(days)
        ax2.xaxis.set_major_formatter(dateFmt)
        ax2.xaxis.set_major_locator(days)
        ax3.xaxis.set_major_formatter(dateFmt)
        ax3.xaxis.set_major_locator(days)

        f.autofmt_xdate()

    return pd.DataFrame(df)

#-----------------------------------------------------------------------------#
#--HEC-RAS UTILS
#-----------------------------------------------------------------------------#

def GetRasData(hdf_plan_file, station):
    #--Read HEC-RAS hdf plan file, return dataframe with flow & calculated water surface elevation at a given station
    with h5py.File(hdf_plan_file ,'r') as hf:
        #--Paths should be the same in any hdf plan file created by HEC-RAS up to 5.0.3
        xs = hf.get('/Results/Steady/Output/Geometry Info/Cross Section Only')[:]
        xs = pd.DataFrame([x.astype(str).split(' ') for x in xs], columns=['River', 'Reach', 'Station'])

        flows = hf.get('/Results/Steady/Output/Output Blocks/Base Output/Steady Profiles/Cross Sections/Flow')[:]
        flows = pd.DataFrame(flows)

        stages = hf.get('/Results/Steady/Output/Output Blocks/Base Output/Steady Profiles/Cross Sections/Water Surface')[:]
        stages = pd.DataFrame(stages)

    idx = xs[xs['Station'] == str(station)].index[0]
    q = flows[idx]
    s = stages[idx]

    df = pd.DataFrame({'flow':q, 'stage':s})
    df.name = station
    df.sort_values(by='flow', inplace=True)
    printbold('RAS Data for XS {}'.format(df.name))
    return df

def GetRasUnsteadyFlow(hdf_plan_file):
    #--Read in unsteady flow data from HEC-RAS plan to dataframe
    with h5py.File(hdf_plan_file ,'r') as hf:
        qs = hf.get('/Event Conditions/Unsteady/Boundary Conditions/Flow Hydrographs/')
        df=pd.DataFrame()
        for q in qs:
            flow_name = q.split(': ')[-1]
            print(flow_name)
            ats =  qs[q].attrs
            items = ats.items
            for item in items():

                if item[0] == 'Start Date':
                    start = item[1]
                    start = start.astype(str)
                    if start.split(' ')[-1][:4] == '2400': #HEC-RAS likes to end the day with 2400...confuses python!
                        s = start.replace('2400', '0000')
                    start = datetime.strptime(s, '%d%b%Y %H%M')
                    #print('start', s)

                elif item[0] == 'End Date' :
                    end= item[1]
                    end = end.astype(str)
                    if end.split(' ')[-1][:4] == '2400':
                        e = end.replace('2400', '0000')
                    end = datetime.strptime(e, '%d%b%Y %H%M')
                    #print('end',e )
                else:
                    continue

            flow = qs[q][:]
            flow = flow[:,1]
            steps = len(flow) -1
            freq = (end-start)/steps
            idx = pd.date_range(start,end, freq=freq)
            df[flow_name] = flow
        df = df.set_index(pd.DatetimeIndex(idx))
        return df

def GetRasWSE(hdf_plan_file, profile):
    with h5py.File(hdf_plan_file ,'r') as hf:
        xs = hf.get('/Results/Steady/Output/Geometry Info/Cross Section Only')[:]
        xs = pd.DataFrame([x.astype(str).split(' ') for x in xs], columns=['River', 'Reach', 'Station'])

        profiles = hf.get('/Results/Steady/Output/Output Blocks/Base Output/Steady Profiles/Profile Names')[:]
        profiles = pd.DataFrame({'p': profiles[:]})
        profiles['p'] = profiles['p'].str.decode("utf-8")
        idx = profiles[profiles['p'] == profile].index[0]

        stages = hf.get('/Results/Steady/Output/Output Blocks/Base Output/Steady Profiles/Cross Sections/Water Surface')[:]
        stages = pd.DataFrame(stages)
        stages = pd.DataFrame({profile: stages.iloc[0]})

    df = pd.merge(xs, stages, left_index=True, right_index=True, how = 'inner')
    return df

#-----------------------------------------------------------------------------#
#--Plotting Functions
#-----------------------------------------------------------------------------#

def PlotPeak(df, peaks, rank, gage, spline=False, days=10):
    epsilon = pd.Timedelta('{} day'.format(days))
    peak = peaks.index[rank]
    start = peak - epsilon
    stop = peak + epsilon
    start = start.strftime(format = '%Y-%m-%d')
    stop = stop.strftime(format = '%Y-%m-%d')
    if spline:
        data = df[gage][start:stop].interpolate(method='spline', order=2)
    else:
        data = df[gage][start:stop].interpolate()
    plot = data.plot(figsize=(10,2), grid='on', title = gage, marker='o', markersize= 1.5)
    return plot

def PlotPeaks(df, gage):
    ax1 = plt.subplot(111)
    df.sort_values(axis=0, ascending=False, inplace=True)
    plt.scatter(df.index , df, color = 'b', marker = 'o', facecolors='none', s=20)
    plt.title(gage.split('_')[0] + ' Peak Flows')
    plt.grid()

def PlotMultiPeak(df, peaks, rank, gage, spline=False, days=10):
    epsilon = pd.Timedelta('{} day'.format(days))
    peak = peaks.index[rank]
    start = peak - epsilon
    stop = peak + epsilon
    start = start.strftime(format = '%Y-%m-%d')
    stop = stop.strftime(format = '%Y-%m-%d')
    if spline:
        data = df[gage][start:stop].interpolate(method='spline', order=2)
    else:
        data = df[gage][start:stop].interpolate()
    plot = data.plot(figsize=(10,2), grid='on', title = gage, marker='o', markersize= 1.5)
    plot.set_xlim(pd.datetime(1900, 1, 1) , pd.datetime(2020, 1, 1))
    return plot

def StageDischargePlot(df, figsize = (2,3)):
    x, y = df['flow'], df['stage']
    f, (ax) = plt.subplots()
    ax.plot(x,y)
    ax.grid()
    ax.set_ylabel('Stage (ft)')
    ax.set_xlabel('Flow (cfs)')
    ax.set_xlim(0, x.max()*1.1)
    ax.set_title('Rating Curve for XS {}'.format(df.name))
    f.set_size_inches(figsize)
    f.autofmt_xdate()
    plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)

'''
!NEED TO UPDATE FUNCTION USING IntegrateHydrograph Approach
def PlotCumIntegral(instantaneous, daily, units = 60):

    #--Plot Cumlative flow (integrated)
    f, ax = plt.subplots(figsize=(10,2))
    ax.set_title('Cumulative Trapezoid')

    x = np.array(daily)*60
    ct = cumtrapz(x)
    ax.plot(ct, label = 'daily')

    x = np.array(instantaneous)*60
    ct = cumtrapz(x)
    ax.plot(ct, label = 'inst')

    handles, labels = ax.get_legend_handles_labels()
    ax.legend(handles, labels)
    #plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.grid()
'''

#-----------------------------------------------------------------------------#
#--REGION SPECIFIC FUNCTIONS (copied from ny_clean_nb.py, version control cleanup)
#
#--Functions created on the fly with a specific set of variables (i.e. gage, flow type, date, etc.)
#--TODO: Clean up functions so that they can be used universally.
#-----------------------------------------------------------------------------#

def Daily_vs_Instant(df):
    print('\n')
    printbold("Interpolated daily flows result in a lower peak flow, but a greater total volume of water:")

    # Daily Plot
    gage = '01509000_00060_dv'
    start, stop = '2005-03-15','2005-04-10'
    adj_start, adj_stop = '2005-03-29','2005-04-10'

    daily_peak= df[gage][start:stop].copy()
    daily_peak= daily_peak[adj_start:adj_stop].resample('60S').asfreq()
    daily_peak= daily_peak.interpolate(method = 'spline', order = 2)
    daily_peak = daily_peak['2005-04-02':'2005-04-05 12:00']
    daily_peak.name = 'Daily Flow'
    daily_peak.plot(figsize=(10,2), grid='on')

    # Instantaneous Plot
    gage = '01509000_00060_iv'
    inst_peak= df[gage][start:stop].copy()
    inst_peak= inst_peak[adj_start:adj_stop].resample('60S').asfreq()

    inst_peak= inst_peak.interpolate(method = 'spline', order = 3)
    inst_peak = inst_peak['2005-04-02':'2005-04-05 12:00']
    inst_peak.name = 'Instantaneous Flow'
    inst_peak.plot(figsize=(10,2), grid='on')

    plt.legend()
    plt.title('USGS Gage {}'.format(gage.split('_')[0]))
    print('Maximum Daily Flow = {} cfs'.format(round(daily_peak.max(),0)))
    print('Maximum Inst Flow = {} cfs'.format(round(inst_peak.max(),0)))

    return daily_peak, inst_peak

def Stretched_Daily_vs_Instant(df):
    print('\n')
    printbold('Results from Stretched Daily flows compared with Instantaneous Records')
    # Daily Plot
    gage = '01509000_00060_dv'
    start, stop = '2005-03-15','2005-04-10'
    adj_start, adj_stop = '2005-03-29','2005-04-10'

    #--Replace the daily mean on the day of peak flow, with the peak from the instantaneous record
    daily_peak= df[gage][start:stop].copy()
    daily_peak['2005-04-03 12:00:00'] = 14200.0
    daily_peak= daily_peak[adj_start:adj_stop].resample('60S').asfreq()
    daily_peak= daily_peak.interpolate(method = 'spline', order = 2)
    daily_peak = daily_peak['2005-04-02':'2005-04-05 12:00']
    daily_peak.name = 'Daily Flow, Stretched to Peak'
    daily_peak.plot(figsize=(10,2), grid='on')

    # Instantaneous Plot
    gage = '01509000_00060_iv'
    inst_peak= df[gage][start:stop].copy()
    inst_peak= inst_peak[adj_start:adj_stop].resample('60S').asfreq()
    inst_peak= inst_peak.interpolate(method = 'spline', order = 3)
    inst_peak = inst_peak['2005-04-02':'2005-04-05 12:00']
    inst_peak.name = 'InstantaneousFlow'
    inst_peak.plot(figsize=(10,2), grid='on')

    plt.title('{}'.format(gage.split('_')[0]))
    plt.legend()

    print('Maximum Daily Flow (stretched) = {} cfs'.format(round(daily_peak.max(),0)))
    print('Maximum Inst Flow = {} cfs'.format(round(inst_peak.max(),0)))
    return daily_peak, inst_peak

def Stretched_Daily_100yr(df, plot = False):
    print('\n')

    # Daily Plot
    gage = '01509000_00060_dv'
    start, stop = '2005-04-01','2005-04-07'

    #--Replace the daily mean on the day of peak flow, with the peak from the instantaneous record
    daily_peak= df[gage][start:stop].copy()
    daily_peak['2005-04-03 12:00:00'] = 20960.0
    daily_peak= daily_peak.resample('15T').asfreq().interpolate(method = 'spline', order = 2)
    daily_peak.name = '1 Pct'
    daily_peak = daily_peak['2005-04-02':'2005-04-05 12:00']

    # Instantaneous Plot
    gage = '01509000_00060_iv'
    inst_peak= df[gage][start:stop].copy().interpolate(method = 'spline', order =2)
    inst_peak.name = 'Base Hydrograph'
    inst_peak = inst_peak['2005-04-02':'2005-04-05 12:00']
    if plot == True:
        daily_peak.plot(figsize=(8,6), grid='on',color = 'blue')
        inst_peak.plot(figsize=(8,6), grid='on', color = 'green')

        plt.title('{}'.format(gage.split('_')[0]))
        plt.legend(loc=0)
        printbold("Hydrograph properties:")
        print('1 Percent Peak Flow = {} cfs'.format(round(daily_peak.max(),0)))
        print('Maximum Inst Flow = {} cfs'.format(round(inst_peak.max(),0)))
    return daily_peak, inst_peak

#FINAL HYDROGRAPH:
def Hydrograph_100yr(df, peak):
    print('\n')
    printbold('Historic Hydrograph Adapted to 100yr Peak Flow')
    # Daily Plot
    gage = '01509000_00060_dv'
    start, stop = '2005-03-15','2005-04-10'
    adj_start, adj_stop = '2005-03-29','2005-04-10'

    #--Replace the daily mean on the day of peak flow, with the peak from the instantaneous record
    daily_peak= df[gage][start:stop].copy()
    daily_peak['2005-04-03 12:00:00'] = peak
    daily_peak= daily_peak[adj_start:adj_stop].resample('60S').asfreq()
    daily_peak= daily_peak.interpolate(method = 'spline', order = 2)
    daily_peak = daily_peak[3600:-3600]
    daily_peak.name = 'Daily Values, Stretched to Inst. Peak'
    daily_peak.plot(figsize=(12,3), grid='on')

    plt.title('{}'.format(gage.split('_')[0]))
    plt.legend(location = 0)

    return pd.DataFrame(daily_peak)

def plotcomp(hydro, rawfitline, breach_stage=0):
    stage = rawfitline(hydro['Base Hydrograph'])
    f, (ax, ax1) = plt.subplots(1,2, figsize = (10,3))
    ax.plot(hydro.index, hydro['Base Hydrograph'], label = '1 Pct Flow (cfs)')
    ax.legend(loc='best',  fontsize='small')
    ax.grid()

    ax1.plot(hydro.index, stage, label = 'Flood Stage (ft)')
    ax1.axhline(breach_stage, color = 'red', label = 'Breach Height')
    ax1.legend(loc='best',  fontsize='small')
    ax1.grid()

    #--Format the Plots
    import matplotlib.dates as mdates
    ax.set_title('Flow Hydrograph \n (input from smoothed storm )')
    days = mdates.DayLocator()   # every year
    dateFmt = mdates.DateFormatter('%Y-%m-%d')
    ax.xaxis.set_major_formatter(dateFmt)
    ax.xaxis.set_major_locator(days)

    ax1.set_title('Stage Hydrograph \n(output using rating curve above)')
    ax1.xaxis.set_major_locator(days)
    ax1.xaxis.set_major_formatter(dateFmt)
    f.autofmt_xdate()

    stage = pd.DataFrame( stage, index = hydro.index)
    stage.rename(columns = {0: 'stage'}, inplace=True)
    return stage

def init_base_hydro(gage_data, peak=14209.0, pct_1=20960.0):
    pct_1_peak, inst_peak  = Stretched_Daily_100yr(gage_data, plot = False)
    stretch_1pct = pct_1/peak
    smooth_storm_resample = inst_peak.resample('30T').mean()
    smooth_storm_1pct = smooth_storm_resample*stretch_1pct
    #print('Factor: \n','\t1 Percent\t{}'.format(stretch_1pct))

    f, ax = plt.subplots()
    ax.plot(smooth_storm_1pct,color = 'green',label = 'Base Hydrograph from Gage, stretched to 1-Pct')
    ax.grid()
    ax.legend(loc='best')
    title = '{} - {}'.format(smooth_storm_1pct.index[0],smooth_storm_1pct.index[-1])
    ax.set_title(title)

    f.set_size_inches(6,4)
    f.autofmt_xdate()
    return smooth_storm_1pct

def smooth_base_hydro(smooth_storm_1pct):
    # 1. Make a copy of the unsmoothed hydrograph:
    final_hydrograph  = smooth_storm_1pct.copy()

    #2. Replace the dates of the portions needing to be smoothed with NaNs
    final_hydrograph['2005-04-02 00:00':'2005-04-03 00:00'] = np.nan #

    #3. Set the first data point to the desired starting value
    # Note: Typically the starting and ending values will be the same, for this example the min works
    final_hydrograph['2005-04-02 00:00':'2005-04-02 00:00'] = float(final_hydrograph.min())

    #4. Add more points as needed to smooth the curve
    # Note: This is an iterative process, using a (factor*min) appproach is used here for speed of iteration
    final_hydrograph['2005-04-02 06:00':'2005-04-02 07:00'] = float(final_hydrograph.min()*1.05)
    final_hydrograph['2005-04-02 14:00':'2005-04-02 15:00'] = float(final_hydrograph.min()*1.25)

    #5. Smooth the iterated points
    smooth_storm = final_hydrograph.interpolate(how = 'polynomial', order=3)

    f, ax = plt.subplots()
    ax.plot(smooth_storm,color = 'green',label = 'Smoothed Storm')
    ax.grid()
    ax.legend(loc= 'best',  fontsize='small')

    f.set_size_inches(6,4)
    f.autofmt_xdate()
    return smooth_storm


from TOOLS.request_fetch import FETCH
from TOOLS.redissave import Redisclient

# 爬虫逻辑
import requests
import re
from lxml import etree
from .address import find_address
from .tessert_atb import get_phone
from TOOLS.md5encode import phone_encrypte
from TOOLS.mongosave import MongoDB
from .ippool import proxies, get_ip
from TOOLS.request_fetch import FETCH

class Atb_spider():
    def __init__(self):
        self.starturl = 'http://bj.atobo.com/'
        # 数据类型模板
        self.data_demo = {'_id': '', 'category_name': '', 'company_name': '', 'company_phone': '',
                          'company_address': ''}
        self.f = FETCH()
        self.m = MongoDB('mongodb://10.2.1.121:17017,10.2.1.122:17017,10.2.1.123:17017', 'clues_resources',
                         'ZZ_20201214_rollback')

    def get_html(self, url):
        html = self.f.fetch(url)
        return html

    def parse_category_html(self, html):
        # 解析种类页面获取种类名称和种类url
        response = etree.HTML(html)
        category_list = response.xpath('//div[@class="sidebar-category"]/ul//li/p[@class="pcategory_son"]/a')
        for category in category_list:
            category_name = category.xpath('./text()').get()
            category_url = category.xpath('./@href').get()
            category_url = 'http:' + category_url

            # yield category_url
            yield self.parse_more_html(self.get_html(category_url), category_name=category_name)

    def parse_more_html(self, html, category_name):
        # 获取更多页选项的url
        response = etree.HTML(html)
        more_company_url = response.xpath('//div[@class="product-list-more"]/a/@href').get()

        yield self.parse_allpage(self.get_html(more_company_url), category_name=category_name)

    def parse_allpage(self, html, category_name):
        # 获取所有翻页url
        # 调用parse_one_url解析每个企业url yield出去，一页结束进入下方代码解析下一页url，再解析每个企业url
        self.parse_one_url(html, category_name=category_name)

        response = etree.HTML(html)
        one_page_url = response.xpath('//a[contains(text(),"下一页")]/@href').extract()
        if one_page_url:
            one_page_url = 'http://www.atobo.com' + one_page_url[0]
            if one_page_url != 'http://www.atobo.com#':
                yield self.parse_allpage(self.get_html(one_page_url), category_name=category_name)

    def parse_one_url(self, html, category_name):
        # 获取一页当中每个企业的url list
        response = etree.HTML(html)
        one_url_list = response.xpath(
            '//li[@class="product_box"]//li[@class="pp_name"]//a[@class="CompanyName"]/@href').extract()
        for one_url in one_url_list:
            one_url = "http://www.atobo.com/" + one_url

            yield self.parse_data(self.get_html(one_url), category_name=category_name)

    def parse_data(self, html, category_name):
        # 获取最终数据
        response = etree.HTML(html)
        try:
            # 图片规则
            company_name = response.xpath('//div[@class="company-intro"]//tr[1]/td/text()').get()
            company_address = response.xpath('//div[@class="company-intro"]//table//tr[2]').get()
            company_address = find_address(company_address)
            if not company_name:
                # 非图片规则
                company_name = response.xpath('//div[@class="company-banner"]//p[@class="t-companyname"]/text()').get()
                company_address = response.xpath('//div[@class="card-context"]//ul[2]//li[2]').get()
                company_address = find_address(company_address)
        except Exception:
            company_name = None
            company_address = None

        try:
            # 有手机号的情况
            company_phone = response.xpath('//div[@class="company-intro"]//tr/td/img/@src').extract()[1]
            company_phone = 'http:' + company_phone
            phone2 = get_phone(picurl=company_phone)
            phone = ''.join(phone2)

            if not company_phone:
                phone1 = response.xpath('//ul[4]//li[2]').get()
                pattern = re.compile("\d+", re.S)
                phone = pattern.findall(phone1)
                if not phone:
                    phone3 = response.xpath('//div[@class="company-intro"]//tr/td/img/@src').extract()[0]
                    try:
                        phone4 = 'http:' + phone3
                        phone5 = get_phone(picurl=phone4)
                        phone = ''.join(phone5)
                    except:
                        phone4 = 'http:' + phone3
                        phone5 = get_phone(picurl=phone4)
                        phone = phone5[0]

        except Exception:
            phone = None

        if company_name and phone:
            # mongo储存数据
            id = phone_encrypte(phone)
            data = self.data_demo
            data["_id"] = id
            data["categoryname"] = category_name
            data["companyname"] = company_name
            data["companyaddress"] = company_address
            data["companyphone"] = phone
            self.m.save_data(datadict=data)
            yield

    def runspider(self):
        html = self.get_html(self.starturl)
        self.parse_category_html(html)


from flask import Flask
from gevent.pywsgi import WSGIServer
import yaml
import requests
from datetime import datetime
import socket

class LocationException(Exception):
  def __init__(self, message):
    self.message = message

def at_time():
    now = datetime.now()
    return now.strftime("%Y-%m-%d %H:%M:%S")

app=Flask(__name__)

def read_config(file_name):
    with open(file_name, 'r') as file:
        load_config = yaml.safe_load(file)
        url=load_config['url']
        loc=load_config["location"]
    return url,loc

def check_rainfall(url,loc):
    resp=requests.get(url)
    status=""
    if resp.status_code == 200:
        timestamp=resp.json()["items"][0]["timestamp"]
        time=timestamp.split('T')[1][0:5]
        station_readings=resp.json()["items"][0]["readings"]
        reading_unit=resp.json()["metadata"]["reading_unit"]
        for station in resp.json()["metadata"]["stations"]:
            if station['name'] == loc:
                for reading in station_readings:
                    if reading["station_id"]==station['id']:
                        status="Raining" if reading["value"] >0 else "Not Raining"
                        output=station['name']+', '+time+', '+str(reading["value"])+reading_unit+', '+status
                        break
                status="found"
                break
        if status=="":
           raise LocationException("Location "+loc+" doesn't exist")
    else:
        print(at_time()+" - URL=> "+url+" is not correct")
        output="There is some internal error. Please contact the admin"
    return output

@app.route('/',methods=['GET'])
def query_rainfall():
    try:
        url,loc=read_config("conf/config.yaml")
        return check_rainfall(url,loc)
    except Exception as e:
        print(at_time()+" - "+str(e))
        return "There is some internal error. Please contact the admin"

if __name__=='__main__':
    http_server = WSGIServer(('',8080),app)
    print(at_time()+" - Rainfall Service started on "+ socket.gethostbyname(socket.gethostname())+':8080')
    http_server.serve_forever()


# 程序创建@李宏宇
# 创建时间 ：2023/3/14

"""
1.使用场景：数据库链接、Socket(套接字)
2.

"""
from random import randint

def Singleton(cls):
    instance = {}

    def _singleton_wrapper(*args, **kargs):
        if cls not in instance:
            instance[cls] = cls(*args, **kargs)
        return instance[cls]

    return _singleton_wrapper

@Singleton
class SingletonTest(object):
    def __init__(self, name):
        self.name = name

slt_1 = SingletonTest('第1次创建')
print(slt_1.name)
slt_2 = SingletonTest('第2次创建')
print(slt_1.name, slt_2.name)

print(slt_1 is slt_2)

from threading import RLock

class Singleton(object):
    single_lock = RLock()  # 上锁

    def __init__(self, name):
        self.name = name

    def __new__(cls, *args, **kwargs):
        with Singleton.single_lock:
            if not hasattr(Singleton, "_instance"):
                Singleton._instance = object.__new__(cls)

        return Singleton._instance

single_1 = Singleton('第1次创建')
single_2 = Singleton('第2次创建')

print(single_1.name, single_2.name)  # 第2次

py基础语法/Py设计模式/1.单例模式.py


from CRABClient.UserUtilities import config, getUsernameFromSiteDB
config = config()

config.General.requestName = 'ZpM3000W30_test'
config.General.workArea = 'crab_projects'
config.General.transferOutputs = True
config.General.transferLogs = True

config.JobType.pluginName = 'Analysis'
config.JobType.psetName = '/afs/desy.de/user/u/usaiem/xxl-af-cms/code/cmssw/CMSSW_7_4_3/src/UHH2/core/python/ntuplewriter.py'
config.JobType.outputFiles = ["Ntuple.root"]

config.Data.inputDataset = '/ZprimeToTT_M-3000_W-30_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/MINIAODSIM'
config.Data.inputDBS = 'global'
config.Data.splitting = 'LumiBased'
config.Data.unitsPerJob = 10
config.Data.outLFNDirBase = '/store/user/eusai/' # % (getUsernameFromSiteDB())
config.Data.publication = False
#config.Data.publishDataName = 'CRAB3_tutorial_May2015_MC_analysis'

config.Site.storageSite = 'T2_DE_DESY'


"""
Testing the behaviour of tsfresh feature extraction function.

Trying make_forecasting_frame and extract_features to transform a univariate
time series in a n_measurements x n_features matrix suitable for machine
learning algorithms.
"""
import time

import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import classification_report
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import OneClassSVM
from tsfresh import (  # extract_relevant_features,
    extract_features, select_features
)
from tsfresh.utilities.dataframe_functions import (  # , roll_time_series
    impute, make_forecasting_frame
)

from ddm_project.readers.nab_dataset_reader import NABReader

# import statsmodels.api as sm
# import statsmodels.tsa.api as tsa
# from sklearn.preprocessing import (
#     MaxAbsScaler, MinMaxScaler, RobustScaler, StandardScaler
# )

reader = NABReader()
reader.load_data()
reader.load_labels()

dataset_name = "iio_us-east-1_i-a2eb1cd9_NetworkIn.csv"

df = reader.data.get(dataset_name)
labels = reader.labels.get(dataset_name)

# add_lags = False
# if add_lags:
#    max_lags = 10
#    lagged = pd.concat(
#        [df] + [df.shift(i).rename(columns=lambda c: "{}_lag_{}".format(c, i))
#                for i in range(1, max_lags)], axis=1).dropna()

# rolled = roll_time_series(df,
#                          column_id='id',
#                          column_sort='timestamp',
#                          column_kind='kind',
#                          rolling_direction=1,
#                          max_timeshift=2)
scaled_value = MinMaxScaler().fit_transform(
    df.value.values.reshape(-1, 1))[:, 0]
scaled_value = pd.Series(scaled_value, index=df.index, name=df.value.name)

df_shift, y = make_forecasting_frame(scaled_value,
                                     kind="kind",
                                     max_timeshift=10,
                                     rolling_direction=1)

if False:
    extract_start = time.time()
    X = extract_features(df_shift,
                         column_id="id",
                         column_sort="time",
                         column_value="value",
                         impute_function=impute,
                         n_jobs=8,
                         show_warnings=False)
    extract_end = time.time()
    print("Extraction time: {}".format(extract_end - extract_start))
    raw_feat_num = X.shape[1]
    print("Extracted {} features.".format(raw_feat_num))
    X = X.loc[:, X.apply(pd.Series.nunique) != 1]
    print("Dropped {} constant features. Remaining features: {}".format(
        raw_feat_num - X.shape[1], X.shape[1]))
    select_start = time.time()
    X = select_features(X, y, ml_task='regression')
    select_end = time.time()
    print("Selection time: {}".format(select_end - select_start))
    print("Final filtered features: {}".format(X.shape[1]))
    X.to_pickle("X.pkl")
else:
    print("Reading features from file")
    X = pd.read_pickle("X.pkl")

tdf = df[1:]
nu = 0.0015
gamma = 0.02
# X = StandardScaler().fit_transform(X)
predictions = OneClassSVM(nu=nu, gamma='auto').fit_predict(X)
anomalies = tdf.value.mask(predictions == -1)
plt.plot(tdf.index, tdf.value)
plt.plot(anomalies, 'x')
plt.show()
y_gt = pd.Series(1, tdf.index)
y_gt.loc[labels] = -1
print(classification_report(y_gt, predictions, labels=[-1]))


import os
import sys
import yaml
import json

BinPath = os.path.split(os.path.realpath(__file__))[0]

def get_dict_aminoacids():
    """
    Retrieve dictionary from the data.yml file to translate between 1-letter and 3-letter aminoacid codes
    :return:	Dictionary of 1-letter to 3-letter aminoacid codes.
    """
    f = BinPath + "/data/data.yml"
    entry_name = "aminoacids"
    with open(f, "r") as infile:
        try:
            data = yaml.safe_load(infile)
        except yaml.YAMLError as exc:
            print(exc)
            sys.exit(1)
    # Sanity check that expected entry is contained in the yml file
    if entry_name not in data.keys():
        raise ValueError("Please provide a dictionary of one-letter to three-letter aminoacid codes via the '%s' entry in %s!" %(entry_name,f))
    dict_codes = data[entry_name]

    return dict_codes

def get_dict_support():
    """
    Retrieve dictionary from the data.yml file to translate between CIViC evidence directions and clinical significances and their corresponding drug support (assigned by user)
    :return:	Dictionary of evidence to drug support.
    """
    f = BinPath + "/data/data.yml"
    entry_name = "drug_support"
    with open(f, "r") as infile:
        try:
            data = yaml.safe_load(infile)
        except yaml.YAMLError as exc:
            print(exc)
            sys.exit(1)
    # Sanity check that expected entry is contained in the yml file
    if entry_name not in data.keys():
        raise ValueError("Please provide a custom dictionary of drug support for CIViC evidences via the '%s' entry in %s!" %(entry_name,f))
    support_dict = data[entry_name]

    return support_dict

def check_header_field(name, header_split, is_required=True):
    """
    Retrieve the position for a given column name from a list of tab-split header fields. Throw error when a required column is not found.
    :param name:		Column name to check in the header to retrieve its position.
    :param header_split:	List of columns from splitting a tab-separated header.
    :param is_required:		Boolean indicating if the given column to check is required. If required, throw an error when column is not found.
    :return:			Either numeric position of the column in the header or None (when non-required column is not found).
    """
    pos = None
    if name in header_split:
        pos = header_split.index(name)
    else:
        if is_required:
            raise ValueError("Required column '%s' could not be found in header '%s'" %(name," ".join(header_split)))

    return pos

def process_snv_header(header_split, gene_name, variant_dna_name, variant_prot_name, variant_impact_name, variant_exon_name):
    """
    Retrieve the required column names expected for the header of an input SNV file. Assume column names are: Gene, Variant_dna, Variant_prot, Variant_impact and Variant_impact. Only the last two are not required.
    :param header_split:	List of columns from splitting a tab-separated header.
    :return:			Tuple of column positions for the gene, cHGVS, pHGVS, impact and exon, in that order. Only the last two can be None.
    """
    gene_pos = check_header_field(gene_name, header_split, is_required=True)
    c_pos = check_header_field(variant_dna_name, header_split, is_required=True)
    p_pos = check_header_field(variant_prot_name, header_split, is_required=True)
    impact_pos = check_header_field(variant_impact_name, header_split, is_required=False)
    exon_pos = check_header_field(variant_exon_name, header_split, is_required=False)

    return (gene_pos, c_pos, p_pos, impact_pos, exon_pos)

def read_in_snvs(infile, expected_gene_name="Gene", expected_variant_dna_name="Variant_dna", expected_variant_prot_name="Variant_prot", expected_variant_impact_name="Variant_impact", expected_variant_exon_name="Variant_exon"):
    """
    Read-in input file of SNV data and process it into structured dictionaries. Assumes header and that relevant info is contained in the following columns: Gene, Variant_dna, Variant_prot. Optional columns:  Variant_impact, Variant_exon.
    :param infile:    Path to the input SNV file to read in.
    :return:          Tuple of 3 elements: dictionary containing original rows and fields from input file (raw_data), dictionary of SNV data to be used to match in CIViC (snv_data) and list of additional columns provided in the input file (which are not required but should be written to output).
    """
    # dict n_line -> [gene, dna, prot, (impact, exon)]
    raw_data = {}

    # dict gene -> variant (dna|prot|impact|exon|n_line) -> null
    snv_data = {}

    in_file = open(infile, "r")
    header = in_file.readline().strip()
    header_split = header.strip().split("\t")
    (gene_pos, c_pos, p_pos, impact_pos, exon_pos) = process_snv_header(header_split, gene_name=expected_gene_name, variant_dna_name=expected_variant_dna_name, variant_prot_name=expected_variant_prot_name, variant_impact_name=expected_variant_impact_name, variant_exon_name=expected_variant_exon_name)

    extra_header = []
    if impact_pos:
        extra_header.append(expected_variant_impact_name)
    if exon_pos:
        extra_header.append(expected_variant_exon_name)

    extra_pos = []
    for pos,x in enumerate(header_split):
        if (pos == gene_pos) or (pos == c_pos) or (pos == p_pos):
            continue
        if impact_pos:
            if pos == impact_pos:
                continue
        if exon_pos:
            if pos == exon_pos:
                continue
        extra_header.append(x)
        extra_pos.append(pos)

    for n_line,line in enumerate(in_file):
        line_split = line.strip().split("\t")
        c_var = line_split[c_pos].strip()
        p_var = line_split[p_pos].strip()
        gene = line_split[gene_pos].strip()
        impact = ""
        if impact_pos:
            impact = line_split[impact_pos].strip()
        exon = ""
        if exon_pos:
            exon = line_split[exon_pos].strip()
        raw_data[str(n_line)] = [gene, c_var, p_var, impact, exon]
        for p in extra_pos:
            raw_data[str(n_line)].append(line_split[p].strip())

        # Process raw_data to have gene-centered dict
        # Returns dict of gene -> [var1,var2,..,varN], where a given var="dna|prot|impact|exon|n_line"
        if gene not in snv_data.keys():
            snv_data[gene] = {}

        # Collapse variant info separated with "|"
        # Keep track of what line each variant comes from
        variant = c_var + "|" + p_var + "|" + impact + "|" + exon + "|" + str(n_line)
        # NOTE: Variants can never be duplicated because of the different row numbers assigned
        # if variant in snv_data[gene].keys():
        #     print("Found duplicated variant '%s|%s' for gene '%s' in line '%s'!" %(c_var, p_var, gene, str(n_line)))
        #     sys.exit(1)
        snv_data[gene][variant] = None
    in_file.close()

    return (raw_data, snv_data, extra_header)

def process_cnv_header(header_split, gene_name, variant_cnv_name):
    """
    Retrieve the required column names expected for the header of an input CNV file. Assume column names are: Gene, Variant_cnv (both are required). The type of CNV variant should be one of the following terms: 'GAIN', 'DUPLICATION', 'DUP', 'AMPLIFICATION' or 'AMP' (synonyms for AMPLIFICATION), and 'DELETION', 'DEL' or 'LOSS' (synonyms for DELETION).
    :param header_split:	List of columns from splitting a tab-separated header.
    :return:			Tuple of column positions for the gene and CNV type.
    """
    gene_pos = check_header_field(gene_name, header_split, is_required=True)
    cnv_pos = check_header_field(variant_cnv_name, header_split, is_required=True)

    return (gene_pos, cnv_pos)

def read_in_cnvs(infile, expected_gene_name="Gene", expected_variant_cnv_name="Variant_cnv"):
    """
    Read-in input file of CNV data and process it into structured dictionaries. Assumes header and that relevant info is contained in the following columns: Gene, Variant_cnv.
    :param infile:    Path to the input CNV file to read in.
    :return:          Tuple of 3 elements: dictionary containing original rows and fields from input file (raw_data), dictionary of CNV data to be used to match in CIViC (cnv_data) and list of additional columns provided in the input file (which are not required but should be written to output).
    """
    # dict n_line -> [gene, cnv]
    raw_data = {}
    # dict gene -> variant (cnv|n_line) -> null
    cnv_data = {}

    in_file = open(infile, "r")
    header = in_file.readline().strip()
    header_split = header.strip().split("\t")
    (gene_pos, cnv_pos) = process_cnv_header(header_split, gene_name=expected_gene_name, variant_cnv_name=expected_variant_cnv_name)

    extra_header = []
    extra_pos = []
    for pos,x in enumerate(header_split):
        if (pos == gene_pos) or (pos == cnv_pos):
            continue
        extra_header.append(x)
        extra_pos.append(pos)

    for n_line,line in enumerate(in_file):
        line_split = line.strip().split("\t")
        gene = line_split[gene_pos].strip()
        cnv = line_split[cnv_pos].strip()
        raw_data[str(n_line)] = [gene,cnv]
        for p in extra_pos:
            raw_data[str(n_line)].append(line_split[p].strip())
        # Process raw_data to have gene-centered dict
        # Returns dict of gene -> [var1,var2,..,varN], where a given var="cnv|n_line"
        if gene not in cnv_data.keys():
            cnv_data[gene] = {}

        # Collapse variant info separated with "|"
        # Keep track of what line each variant comes from
        variant = cnv + "|" + str(n_line)
        # NOTE: Variants can never be duplicated because of the different row numbers assigned
        # if variant in cnv_data[gene].keys():
        #     print("Found duplicated variant '%s' for gene '%s' in line '%s'!" %(cnv,gene,str(n_line)))
        #     sys.exit(1)
        cnv_data[gene][variant] = None
    in_file.close()

    return (raw_data, cnv_data, extra_header)

def process_expr_header(header_split, gene_name, logfc_name):
    """
    Retrieve the required column names expected for the header of an input Expression file. Assume column names are: Gene, logFC (both are required). The log fold-change value of a differentially expressed gene should be numeric and different from zero (to query CIViC, it will be translated into 'OVEREXPRESSION', if positive, and 'UNDEREXPRESSION', if negative).
    :param header_split:	List of columns from splitting a tab-separated header.
    :return:			Tuple of column positions for the gene and log fold-change.
    """
    gene_pos = check_header_field(gene_name, header_split, is_required=True)
    logfc_pos = check_header_field(logfc_name, header_split, is_required=True)

    return (gene_pos, logfc_pos)

def read_in_expr(infile, expected_gene_name="Gene", expected_logFC_name="logFC"):
    """
    Read-in input file of differentially expressed data and process it into structured dictionaries. Assumes header and that relevant info is contained in the following columns: Gene, logFC.
    :param infile:    Path to the input expression file to read in.
    :return:          Tuple of 3 elements: dictionary containing original rows and fields from input file (raw_data), dictionary of EXPR data to be used to match in CIViC (expr_data) and list of additional columns provided in the input file (which are not required but should be written to output).
    """
    # dict n_line -> [gene, logfc]
    raw_data = {}
    # dict gene -> expression (logfc|n_line) -> null
    expr_data = {}

    in_file = open(infile, "r")
    header = in_file.readline().strip()
    header_split = header.strip().split("\t")
    (gene_pos, logfc_pos) = process_expr_header(header_split, gene_name=expected_gene_name, logfc_name=expected_logFC_name)

    extra_header = []
    extra_pos = []
    for pos,x in enumerate(header_split):
        if (pos == gene_pos) or (pos == logfc_pos):
            continue
        extra_header.append(x)
        extra_pos.append(pos)

    for n_line,line in enumerate(in_file):
        line_split = line.strip().split("\t")
        gene = line_split[gene_pos].strip()
        logfc = line_split[logfc_pos].strip()

        raw_data[str(n_line)] = [gene, logfc]
        for p in extra_pos:
            raw_data[str(n_line)].append(line_split[p].strip())
        # Process raw_data to have gene-centered dict
        # Returns dict of gene -> [var1,var2,..,varN], where a given var="logfc|n_line"
        if gene not in expr_data.keys():
            expr_data[gene] = {}

        # Collapse expression info separated with "|"
        # Keep track of what line each variant comes from
        expression = str(logfc) + "|" + str(n_line)
        # NOTE: Expression values can never be duplicated because of the different row numbers assigned
        # if expression in expr_data[gene].keys():
        #     print("Found duplicated expression value '%s' for gene '%s' in line '%s'!" %(logfc,gene,str(n_line)))
        #     sys.exit(1)
        expr_data[gene][expression] = None
    in_file.close()

    return (raw_data, expr_data, extra_header)

def write_to_json(in_dict, outfile, indent=1):
    """
    Write a nested dictionary into a new output file (in JSON format).
    :param in_dict:	Object of type 'dict' (can be nested).
    :param outfile:	Path to the output JSON file to write into.
    :param indent:	Value of indentation to use as margin.
    :return:		None
    """
    with open(outfile, "w") as f:
        json.dump(in_dict, f, ensure_ascii=False, indent=indent)

    return None

def write_to_yaml(in_dict, outfile):
    """
    Write a nested dictionary into a new output file (in YAML format).
    :param in_dict:	Object of type 'dict' (can be nested).
    :param outfile:	Path to the output YAML file to write into.
    :return:		None
    """
    with open(outfile, "w") as f:
        # Preserve the original order of the entries in the manifest template
        yaml.dump(in_dict, f, default_flow_style=False, sort_keys=False)

    return None

def write_header_line(data_type, header, write_support, expected_gene_name, expected_logFC_name, expected_variant_cnv_name, expected_variant_dna_name, expected_variant_prot_name, expected_variant_impact_name, expected_variant_exon_name):
    """
    Given a list of sorted column names (from splitting the input header), process it to generate the corresponding output header.
    :param data_type:		['SNV', 'CNV', 'EXPR']
				SNV:   Expects a file of genomic single nucleotide variants and insertions/deletions
				CNV:   Expects a file of genomic copy number alterations
				EXPR:  Expects a file of differential gene expression data
				Data type of the corresponding input file (string).
    :param header:		List of input column names (in order) from splitting the tab-separated header.
    :param write_support:	Boolean indicating if processed drug support from CIViC should be written to output.
    :return:			Tuple of 4 elements (last 3 are only relevant when 'data_type=SNV'): string containing complete header ready to be written to output, list of column names excluding 'Variant_impact' and 'Variant_exon' (if they were present), boolean indicating if column 'Variant_impact' was present, boolean indicating if column '\tVariant_exon' was present.
    """
    sorted_evidence_types = ["PREDICTIVE", "DIAGNOSTIC", "PROGNOSTIC", "PREDISPOSING"]

    # Variables only relevant for data_type="SNV"
    write_impact = False
    write_exon = False

    if data_type == "SNV":
        main_header = f"{expected_gene_name}\t{expected_variant_dna_name}\t{expected_variant_prot_name}"
        if expected_variant_impact_name in header:
            main_header += f"\t{expected_variant_impact_name}"
            write_impact = True
        if expected_variant_exon_name in header:
            main_header += f"\t{expected_variant_exon_namef}"
            write_exon = True

    if data_type == "CNV":
        main_header = f"{expected_gene_name}\t{expected_variant_cnv_name}"

    if data_type == "EXPR":
        main_header = f"{expected_gene_name}\t{expected_logFC_name}"

    clean_header = []
    if header:
        for tmp in header:
            if (tmp != expected_variant_impact_name and tmp != expected_variant_exon_name):
                clean_header.append(tmp)
    if clean_header:
        main_header += "\t%s" %("\t".join(clean_header))

    if write_support:
        out_header = "%s\tCIViC_Tier\tCIViC_Score\tCIViC_VariantType\tCIViC_Drug_Support\t%s" %(main_header,"\t".join(["CIViC_" + x for x in sorted_evidence_types]))
    else:
        out_header = "%s\tCIViC_Tier\tCIViC_Score\tCIViC_VariantType\t%s" %(main_header,"\t".join(["CIViC_" + x for x in sorted_evidence_types]))

    return (out_header, clean_header, write_impact, write_exon)

def write_output_line(tier, main_line, gene_scores, gene_var_types, drug_support, result_map, write_support):
    """
    Process CIViC information available for a given input line, and generate the corresponding output line. Empty fields are indicated with '.'.
    :param tier:		String with format 'tier_[N]', where N can be: '1', '1b', '2', '3' or '4'. They indicate different tier cases of the variant matching.
    :param main_line:		String already containing the required columns to be written to output. This will be extended with the available CIViC information to generate the complete output line.
    :param gene_scores:		List containing the CIViC scores of the matched variants for the given 'tier'. Already formatted as required i.e. 'GENE:VARIANT_NAME:SCORE'.
    :param gene_var_types:	List containing the variant types of the matched variants in CIViC. Already formatted as required i.e. 'GENE:VARIANT_NAME:VAR_TYPE1,..'.
    :param drug_support:	List containing the strings of processed drug support for the matched variants. Already formatted as required i.e. 'DRUG:CT_TAG:CIVIC_CONSENSUS_PREDICTION'.
    :param result_map:		Nested dictionary containing the CIViC evidences associated with each evidence type, already formatted for writing to output (assumes a particular structure).
    :param write_support:	Boolean indicating if processed drug support from CIViC should be written to output.
    :return:			String containing complete output line ready to be written to output.
    """
    sorted_evidence_types = ["PREDICTIVE", "DIAGNOSTIC", "PROGNOSTIC", "PREDISPOSING"]

    # Remove the "tier" tag from the assined tier
    if tier.startswith("tier_"):
        tier = tier.replace("tier_", "")

    if gene_scores:
        out_line = main_line + "\t" + tier + "\t" + ";".join(gene_scores)
    else:
        out_line = main_line + "\t" + tier + "\t."

    if gene_var_types:
        out_line += "\t" + ";".join(gene_var_types)
    else:
        out_line += "\t."

    if write_support:
        if drug_support:
            out_line += "\t" + ";".join(drug_support)
        else:
            out_line += "\t."

    for evidence_type in sorted_evidence_types:
        # if evidence_type not in result_map.keys():
        #     raise ValueError("Evidence type '%s' is not found in provided 'result_map'!" %(evidence_type))
        if evidence_type in result_map.keys():
            if result_map[evidence_type]:
                out_line += "\t" + ";".join(result_map[evidence_type])
            else:
                out_line += "\t."
        else:
            out_line += "\t."

    return out_line

def write_evidences(item, write_drug=False, write_ct=None, write_complete=False):
    """
    Process CIViC information available for a given evidence type, and reformat into structured strings for reporting to output.
    Format: DISEASE[|DRUG1,DRUG2..](direction, significance(level(PMID,..,PMID),level(..)));
    :param item:		Nested dictionary containing CIViC evidences associated to a single evidence type (assumes a particular structure).
    :param write_drug:		Boolean indicating if the current type being evaluated corresponds to 'PREDICTIVE' (i.e. these are the only evidences associated to any drug names in CIViC).
    :param write_ct:		Boolean indicating if the available disease specificity annotations should be included in the output table.
    :param write_complete:	Boolean indicating if the complete information string of each CIViC evidence should be written to output. When 'write_complete=False', only the ids of the associated publications will be reported instead, already formatted as 'SOURCE_ID' (e.g. 'PUBMED_12345' or 'ASCO_12345').
    :return:			List of complete evidence strings to be written under the current evidence type (one per column).
    """
    evidences = []
    # For each disease found in the provided item
    for disease in item.keys():
        # For each drug associated with the given cancer type
        for drug in item[disease].keys():
            # For each evidence associated with the given drug
            # Evidences are simplified by using the combined form "direction:significance"
            for evidence in item[disease][drug].keys():
                # If drug=True, write drug information, i.e. DISEASE|DRUG(..)
                if write_drug:
                    # Always one drug (single or combination with "+")
                    if write_ct:
                        out_string = disease + "|" + write_ct.upper() + "|" + drug + "("
                    else:
                        out_string = disease + "|" + drug + "("
                else:
                    if write_ct:
                        out_string = disease + "|" + write_ct.upper() + "("
                    else:
                        out_string = disease + "("

                # Split the evidence direction and clinical significance
                evidence_list = evidence.strip().split(":")
                if (len(evidence_list) != 2):
                    raise ValueError("Unexpected format of evidence '%s'! Please provide string as 'EVIDENCE_DIRECTION:CLINICAL_SIGNIFICANCE'." %(evidence))
                direction = evidence_list[0]
                clin_signf = evidence_list[1]
                out_string += direction + "," + clin_signf + "("
                # There may be several levels grouped per evidence
                levels = []
                for level in item[disease][drug][evidence].keys():
                    pmids = []
                    # There may be several publications (i.e. PMIDs) grouped per level
                    for z in item[disease][drug][evidence][level]:
                        if write_complete:
                            pmids.append(z)
                        else:
                            z_split = z.strip().split(":")
                            if (len(z_split) != 5):
                                raise ValueError("Unexpected format of evidence item '%s'! Please provide string as 'EVIDENCE_ID:EVIDENCE_STATUS:SOURCE_STATUS:VARIANT_ORIGIN:RATING'." %(z))
                            pub_id = z_split[0].strip()
                            pmids.append(pub_id)
                    levels.append(level + "(" + ",".join(pmids) + ")")
                out_string += ",".join(levels) + "))"
                evidences.append(out_string)

    return evidences

def write_match(match_map, var_map, raw_map, header, data_type, outfile, has_support=True, has_ct=True, write_ct=False, write_support=True, write_complete=False, expected_gene_name="Gene", expected_logFC_name="logFC", expected_variant_cnv_name="Variant_cnv", expected_variant_dna_name="Variant_dna", expected_variant_prot_name="Variant_prot", expected_variant_impact_name="Variant_impact", expected_variant_exon_name="Variant_exon"):
    """
    Process input data and matched CIViC information, and reformat to write into a tab-separated table.
    :param match_map:		Dictionary containing data matched in CIViC (there must be a correspondance of 'match_map' and the variant data in 'var_map'). See README for more details about the specific structure of dictionary 'match_map'.
    :param var_map:		Nested dictionary of results from querying genes in CIViC (there must be a correspondance of 'var_map' and the variants matched in 'match_map'). See README for more details about the specific structure of dictionary 'var_map'.
    :param raw_map:		Dictionary containing the original rows and fields from the processed input file (n_line -> [field1,field2,..]).
    :param header:		List of input column names (in order) from splitting the tab-separated header.
    :param data_type:		['SNV', 'CNV', 'EXPR']
                             	SNV:   Expects a file of genomic single nucleotide variants and insertions/deletions
                             	CNV:   Expects a file of genomic copy number alterations
                             	EXPR:  Expects a file of differential gene expression data
                             	String data type of the corresponding input file.
    :param outfile:          	Path to the output file to write the matched CIViC evidences into (tab-separated table with header).
    :param has_support:       	Boolean indicating if the provided 'match_map' is annotated for drug support.
    :param has_ct:            	Boolean indicating if the provided 'var_map' is annotated for disease specificity.
    :param write_ct:          	Boolean indicating if the available disease specificity annotations should be included in the output table. To use this option, 'has_ct' must be True.
    :param write_support:     	Boolean indicating if the available drug support annotations should be included in the output table. To use this option, 'has_support' must be True.
    :param write_complete:    	Boolean indicating if the complete information string of each CIViC evidence should be written to output. When 'write_complete=False', only the ids of the associated publications will be reported instead, already formatted as 'SOURCE_ID' (e.g. 'PUBMED_12345' or 'ASCO_12345').
    :return:                 	None
    """
    # NOTE: uppercase is critical for matching!
    sorted_evidence_types = ["PREDICTIVE", "DIAGNOSTIC", "PROGNOSTIC", "PREDISPOSING"]
    evidence_type = "PREDICTIVE"
    special_cases = ["NON_SNV_MATCH_ONLY", "NON_CNV_MATCH_ONLY", "NON_EXPR_MATCH_ONLY"]
    sorted_cts = ["ct", "gt", "nct"]
    varmap_entries_variant = ["name", "hgvs", "types"]

    from civicutils.utils import check_match_before_writing, check_keys, check_keys_not, check_data_type, check_dict_entry
    check_match_before_writing(match_map, var_map, raw_map, has_support, has_ct, write_ct, write_support, write_complete)
    check_data_type(data_type)
    outfile = open(outfile, "w")

    # Retrieve the output header given the argument selection
    (out_header, clean_header, write_impact, write_exon) = write_header_line(data_type, header, write_support, expected_gene_name, expected_logFC_name, expected_variant_cnv_name, expected_variant_dna_name, expected_variant_prot_name, expected_variant_impact_name, expected_variant_exon_name)
    outfile.write(out_header + "\n")

    for n_line in raw_map.keys():
        line_list = raw_map[n_line]
        extra_line = []
        if data_type == "SNV":
            if (len(line_list) < 5):
                raise ValueError("Must provide at least 5 elements to describe a SNV variant (even if some can be empty): 'gene,dna,[prot],[impact],[exon],..'")
            gene = line_list[0]
            c_var = line_list[1]
            p_var = line_list[2]
            impact = line_list[3]
            exon = line_list[4]
            comb_id = c_var + "|" + p_var + "|" + impact + "|" + exon + "|" + str(n_line)
            # Extract any additional fields that might be present for this line
            for pos in range(5, len(line_list)):
                extra_line.append(line_list[pos])
            # Build line string for writing to output
            main_line = gene + "\t" + c_var + "\t" + p_var
            if write_impact:
                main_line += "\t" + impact
            if write_exon:
                main_line += "\t" + exon

        if data_type == "CNV":
            if (len(line_list) < 2):
                raise ValueError("Must provide at least 2 elements to describe a CNV variant: 'gene,cnv,..'")
            gene = line_list[0]
            cnv = line_list[1]
            comb_id = cnv + "|" + str(n_line)
            # Extract any additional fields that might be present for this line
            for pos in range(2, len(line_list)):
                extra_line.append(line_list[pos])
            # Build line string for writing to output
            main_line = gene + "\t" + cnv

        if data_type == "EXPR":
            if (len(line_list) < 2):
                raise ValueError("Must provide at least 2 elements to describe a EXPR variant: 'gene,logFC,..'")
            gene = line_list[0]
            logfc = line_list[1]
            comb_id = str(logfc) + "|" + str(n_line)
            # Extract any additional fields that might be present for this line
            for pos in range(2, len(line_list)):
                extra_line.append(line_list[pos])
            # Build line string for writing to output
            main_line = gene + "\t" + str(logfc)

        # Sanity check that as many data fields were provided as in the header
        if len(extra_line) != len(clean_header):
            raise ValueError("Number of fields available does not match provided header!")
        # Add extra fields to the current line build
        for extra in extra_line:
            main_line += "\t" + extra

        # Check if match_map contains the provided input variants
        if gene not in match_map.keys():
            raise ValueError("Provided gene '%s' is not contained in 'match_map'." %(gene))
        if comb_id not in match_map[gene].keys():
            raise ValueError("Provided variant '%s' is not contained in 'match_map' of gene '%s'." %(comb_id,gene))
        for tier in match_map[gene][comb_id].keys():
            gene_scores = []
            gene_var_types = []
            drug_support = []
            result_map = {}
            write_line = False
            if tier != "tier_4":
                all_variants = []
                if has_support:
                    this_drug_support = match_map[gene][comb_id][tier]["drug_support"]
                    for tmp_var in match_map[gene][comb_id][tier]["matched"]:
                        all_variants.append(tmp_var)
                    if write_support:
                        for i in this_drug_support:
                            drug_support.append(i.upper())
                else:
                    if write_support:
                        raise ValueError("Option 'write_support' cannot be selected when 'has_support'=False!")
                    for tmp_var in match_map[gene][comb_id][tier]:
                        all_variants.append(tmp_var)

                for var_id in all_variants:
                    # NOTE: check for special case when tier3 but no matching variant returned for the given data type
                    # This is a dummy tag and not an actual variant record from CIViC, so skip checking in var_map
                    if var_id.upper() in special_cases:
                        # In this case, current line will be associated with tier3, but all columns will be empty with "."
                        for evidence_type in sorted_evidence_types:
                            if evidence_type not in result_map.keys():
                                result_map[evidence_type] = []
                        continue

                    variant = var_map[gene][var_id]["name"]

                    gene_var_types.append(gene + ":" + variant + ":" + ",".join(var_map[gene][var_id]["types"]))
                    molecular_profile_ids = set(list(var_map[gene][var_id].keys())) ^ set(varmap_entries_variant)
                    for molecular_profil_id in molecular_profile_ids:
                        gene_scores.append(gene + ":" + variant + ":" + molecular_profil_id + ":" + str(var_map[gene][var_id][molecular_profil_id]["civic_score"]))
                        for evidence_type in sorted_evidence_types:
                            if evidence_type in var_map[gene][var_id][molecular_profil_id]["evidence_items"].keys():
                                if evidence_type not in result_map.keys():
                                    result_map[evidence_type] = []
                                write_drug = False
                                if evidence_type == evidence_type:
                                    write_drug=True
                                if has_ct:
                                    check_keys(list(var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type].keys()), "var_map", sorted_cts, matches_all=True)
                                    for ct in var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type].keys():
                                        if write_ct:
                                            results = write_evidences(var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type][ct], write_drug=write_drug, write_ct=ct, write_complete=write_complete)
                                        else:
                                            results = write_evidences(var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type][ct], write_drug=write_drug, write_ct=None, write_complete=write_complete)
                                        for x in results:
                                            result_map[evidence_type].append(gene + ":" + variant + ":" + molecular_profil_id + ":" + x)
                                else:
                                    check_keys_not(list(var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type].keys()), "var_map", sorted_cts)
                                    if write_ct:
                                        raise ValueError("Option 'write_ct' cannot be selected when 'has_ct'=False!")
                                    results = write_evidences(var_map[gene][var_id][molecular_profil_id]["evidence_items"][evidence_type], write_drug=write_drug, write_ct=None, write_complete=write_complete)
                                    for x in results:
                                        result_map[evidence_type].append(gene + ":" + variant + ":" + molecular_profil_id + ":" + x)

                # Only write line current tier when there was at least one match for it
                if all_variants:
                    write_line = True

            else:
                if has_support:
                    if match_map[gene][comb_id][tier]["matched"]:
                        write_line = True
                else:
                    if write_support:
                        raise ValueError("Option 'write_support' cannot be selected when 'has_support'=False!")
                    if match_map[gene][comb_id][tier]:
                        write_line = True

            if write_line:
                out_line = write_output_line(tier, main_line, gene_scores, gene_var_types, drug_support, result_map, write_support)
                outfile.write(out_line + "\n")
    outfile.close()

    return None

def write_drug_targets(drug_targets, raw_map, data_type, outfile_drug_targets):
    """
    Process drug targets data to write into a tab-separated table.
    :param drug_targets:		Dictionary containing data matched in CIViC (there must be a correspondance of 'match_map' and the variant data in 'var_map'). See README for more details about the specific structure of dictionary 'match_map'.
    :param raw_map:		Dictionary containing the original rows and fields from the processed input file (n_line -> [field1,field2,..]).
    :param data_type:		['SNV', 'CNV', 'EXPR']
                             	SNV:   Expects a file of genomic single nucleotide variants and insertions/deletions
                             	CNV:   Expects a file of genomic copy number alterations
                             	EXPR:  Expects a file of differential gene expression data
                             	String data type of the corresponding input file.
    :param outfile_drug_targets:          	Path to the output file to write the drug targets into (tab-separated table).
    :return:                 	None
    """

    # Count the frequency of each drug in the dictionary
    drug_frequency = {}
    for drugs in drug_targets:
        drug_frequency[drugs] = len(drug_targets[drugs].keys())

    # Sort the dictionary based on the frequency of the drug (column one)
    sorted_drug_targets = sorted(drug_targets.items(), key=lambda x: drug_frequency[x[0]], reverse=True)

    if data_type == "SNV":
        with open(outfile_drug_targets, 'w') as file:
            # Write the header
            file.write("Drug\tGene\tObserved_DNA_Variant\tObserved_Prot_Variant\tCivic_Variant\tTier\tEvidence_type\tct\tDisease\tEvidence\n")

            # Iterate through the dictionary and write each row
            for drugs, targets_info in sorted_drug_targets:
                targets = targets_info.keys()
                for target in targets:
                    info = targets_info[target]
                    keys_with_search_item = [key for key, values in raw_map.items() if target in values]
                    Observed_data = raw_map[keys_with_search_item[0]]
                    file.write(f"{drugs}\t{target}\t{Observed_data[1]}\t{Observed_data[2]}\t{info[0]}\t{info[1]}\t{info[2]}\t{info[3]}\t{info[4]}\t{info[5]}\n")

    if data_type == "CNV":
        amplification_term = ["AMPLIFICATION", "AMP", "GAIN", "DUPLICATION", "DUP"]
        deletion_term = ["DELETION", "DEL", "LOSS"]

        with open(outfile_drug_targets, 'w') as file:
            # Write the header
            file.write("Drug\tGene\tObserved_CNV_Variant\tCivic_Variant\tTier\tEvidence_type\tct\tDisease\tEvidence\n")
            # Iterate through the dictionary and write each row
            for drugs, targets_info in sorted_drug_targets:
                targets = targets_info.keys()
                for target in targets:
                    info = targets_info[target]
                    keys_with_search_item = [key for key, values in raw_map.items() if target in values]
                    if len(keys_with_search_item) > 1:
                        Observed_data=[]
                        for variant_cnv in keys_with_search_item:
                            var_annot = raw_map[variant_cnv][1]
                            if (any(term in var_annot for term in amplification_term)) and (info[0] == "AMPLIFICATION"):
                                Observed_data = raw_map[variant_cnv]
                            elif (any(term in var_annot for term in deletion_term)) and (info[0] == "DELETION"):
                                Observed_data = raw_map[variant_cnv]
                    else:
                        Observed_data = raw_map[keys_with_search_item[0]]
                    file.write(f"{drugs}\t{target}\t{Observed_data[1]}\t{info[0]}\t{info[1]}\t{info[2]}\t{info[3]}\t{info[4]}\t{info[5]}\n")

    if data_type == "EXPR":
        with open(outfile_drug_targets, 'w') as file:
            # Write the header
            file.write("Drug\tGene\tObserved_logFC\tCivic_Variant\tTier\tEvidence_type\tct\tDisease\tEvidence\n")

            # Iterate through the dictionary and write each row
            for drugs, targets_info in sorted_drug_targets:
                targets = targets_info.keys()
                for target in targets:
                    info = targets_info[target]
                    keys_with_search_item = [key for key, values in raw_map.items() if target in values]
                    Observed_data = raw_map[keys_with_search_item[0]]
                    file.write(f"{drugs}\t{target}\t{Observed_data[1]}\t{info[0]}\t{info[1]}\t{info[2]}\t{info[3]}\t{info[4]}\t{info[5]}\n")

    return None


                 #This code is done by Thiruselvan E

# This program prints the sum of two values without using '+' operator

# The approach of the program is using for loop from range 1 to second value until second value is reached N1 is incremented by 1

N1 = int(input("Enter the first number : "))

N2 = int(input("Enter the second number : "))

for i in range(1,N2+1):
    N1 += 1

print(N1)


from unittest import TestCase
from unittest.mock import patch, call, MagicMock

from django.test import RequestFactory

from learn.infrastructure.database import Database
from learn.views.wrong_answer import exercise_wrong_answer

class ExerciseWrongAnswerTests(TestCase):
    @patch("learn.views.wrong_answer.render")
    def test_shouldRenderWrongAnswer_WithDictionaryPK_AndTranslationPK_AndTranslation(self, render_mock):
        # Given
        database = MagicMock(Database)
        factory = RequestFactory()
        request = factory.get('fake-url')

        # When
        exercise_wrong_answer(request, 23, 42, database=database)

        # Then
        expected_args = call(request, 'learn/exercise_wrong_answer.html', context={
            'dictionary_pk': 23,
            'translation_pk': 42,
            'translation': database.get_translation()
        })
        self.assertEqual(render_mock.call_args_list[0], expected_args)


import codecademylib3_seaborn
import pandas as pd
import numpy as np
from weather_data import london_data

# print(london_data.head())
#print(london_data.iloc[100:200])
print(len(london_data))

temp = london_data["TemperatureC"]
#print(temp.head())
average_temp = np.mean(temp)
print(average_temp)
temperature_var = np.var(temp)
print(temperature_var)
temperature_standard_deviation = np.std(temp)
print(temperature_standard_deviation)

#print(london_data.head())
#print(london_data.tail())

june = london_data.loc[london_data["month"] == 6]["TemperatureC"]
#print(june.head())
july = london_data.loc[london_data["month"] == 7]["TemperatureC"]
#print(july.head())

june_mean_temp = np.mean(june)
print(june_mean_temp)
july_mean_temp = np.mean(july)
print(july_mean_temp)

june_std_temp = np.std(june)
print(june_std_temp)
july_std_temp = np.std(july)
print(july_std_temp)

# temperature
# for i in range(1, 13):
#   month = london_data.loc[london_data["month"] == i]["TemperatureC"]
#   print("The mean temperature in month "+str(i) +" is "+ str(np.mean(month)))
#   print("The standard deviation of temperature in month "+str(i) +" is "+ str(np.std(month)) +"\n")

# humidity
# for i in range(1, 13):
#   month = london_data.loc[london_data["month"] == i]["Humidity"]
#   print("The mean humidity in month "+str(i) +" is "+ str(np.mean(month)))
#   print("The standard deviation of humidity in month "+str(i) +" is "+ str(np.std(month)) +"\n")

# print(london_data.tail())
# hour
for i in range(1, 23+1):
  hour = london_data.loc[london_data["hour"] == i]["TemperatureC"]
  print("The mean temperature in hour "+str(i) +" is "+ str(np.mean(hour)))
  print("The standard deviation of Temperature in hour "+str(i) +" is "+ str(np.std(hour)) +"\n")


# By submitting this assignment, I agree to the following:
#  “Aggies do not lie, cheat, or steal, or tolerate those who do”
#  “I have not given or received any unauthorized aid on this assignment”
#
# Name: Matthew McCabe
# Section: 102 538
# Assignment: Lab 10b Act1
# Date: 10/31/2022
import random

print("Guess the secret integer between 1 and 100")
r = random.randint(1, 100)

def user_guess():
    g = int(input("What is your guess? "))
    return g

def is_less():
    if i < r:
        print("Too low, try again")
        user_guess()

def is_high():
    if i > r:
        print("Too high, try again")
        user_guess()

i = user_guess()
c = 1

while i != r:
    is_less()
    is_high()
    i = user_guess()
    c += 1

print(f'You found it in {c} guess(es)... PARTY TIME!!!')


import operator
from typing import Any, Callable, List, Optional, overload, Set, Tuple, Union

from optcom.utils.utilities_helpers.terminal_display_helpers \
    import warning_terminal

def check_attr_type(attr: Any, attr_name: str, *types: Any) -> None:
    """Check if the provided attribute is of the provided type."""
    res = False
    for type in types:
        if (type is None):    # Allow attr to be None
            if (attr is None):
                res = True
        else:
            res = res or isinstance(attr, type)   # Keep the True
    if (not res):
        types_name = locals()['types']
        raise TypeError("{} must be one of the following type: {}"
                        .format(attr_name, types_name))

def check_attr_range(attr: Any, attr_name: str, min_attr: Union[int, float],
                     max_attr: Union[int, float], strict_left: bool = False,
                     strict_right: bool = False) -> None:
    """Check if the provided attribute is in the range provided."""
    if (strict_left):
        op_left = operator.lt
    else:
        op_left = operator.le
    if (strict_right):
        op_right = operator.gt
    else:
        op_right = operator.ge

    if (not(op_left(min_attr, attr) and op_right(max_attr, attr))):
        raise IndexError("Attribute {} out of range, must be in interval "
                         "[{},{}]"
                         .format(attr_name, min_attr, max_attr))

def check_attr_value(attr: Any, attr_values: List[Any], attr_default: Any
                     ) -> Any:
    """Check if the provided attribute is one of the allowed values
    provided. If not, return a provided default value.
    """
    if (attr not in attr_values):
        warning_terminal("The attribute specified '{}' is not "
            "supported, default attribute '{}' set instead."
            .format(attr, attr_default))

        return attr_default
    else:

        return attr

Quantum Cryptography/ML Kalman Filter/ml_kalman/Lib/site-packages/optcom/utils/utilities_helpers/attr_helpers.py


# Importing all necessary libraries
import cv2
import os

# Read the video from specified path
cam = cv2.VideoCapture("E:\\电影\\搭错车 1983.avi")

try:

    # creating a folder named data
    if not os.path.exists('data'):
        os.makedirs('data')

# if not created then raise error
except OSError:
    print('Error: Creating directory of data')

# frame
currentframe = 0

# reading from frame

Correctdata = [];
with open("timeSeries", "r", encoding='UTF-8') as f:  # 打开文件
    data = f.readlines()  # 读取文件
    for x in data:
        temp = x.replace('\n','');
        tempList = temp.split(":")
        if len(tempList) == 2:
            Correctdata.append(int(tempList[0]) * 60 + int(tempList[1]))
        if len(tempList) == 3:
            Correctdata.append(int(tempList[0]) * 3600 + int(tempList[1]) * 60 + int(tempList[2]))
print(Correctdata)

for x in Correctdata:
    cam.set(cv2.CAP_PROP_POS_MSEC, x * 1000)
    ret, frame = cam.read()
    if ret:
        name = './data/frame' + str(currentframe) + '.jpg'
        print('Creating...' + name)

        # writing the extracted images
        cv2.imwrite(name, frame)
        currentframe += 1

cam.release()
cv2.destroyAllWindows()


from django.urls import path
from onlineapp.views import *
from rest_framework.authtoken import views
from rest_framework_jwt.views import obtain_jwt_token

# from onlineapp.views import CollegeListView,CollegeDetailsView,CreateCollegeView,CreateStudentView,
# EditCollege,DeleteCollege,EditStudent,DeleteStudent,CreateUser,LoginForm

app_name = 'onlineapp'

urlpatterns = [
	# path('', hello_world),
	# path('collegelist/', list_colleges),
	# path('sample/', sample_index),
	# path('student_list/', student_details),
	# path('student_list/<int:id>/', student_details2),
	# path('college_summary/<str:acronym>/', college_student_list),
	# path('test_session/', test_sessions),

	path('colleges/', CollegeListView.as_view(), name='colleges_html'),
	path('colleges/<int:id>/', CollegeDetailsView.as_view(), name='college_details_id'),
	path('colleges/<str:acronym>/', CollegeDetailsView.as_view(), name='college_details_ac'),
	path('college/add/', CreateCollegeView.as_view(), name='add_college'),
	path('colleges/<int:college_id>/add/',CreateStudentView.as_view(),name='add_student'),
	path('colleges/<int:pk>/edit',EditCollege.as_view(),name='edit_college'),
	path('colleges/<int:pk>/delete',DeleteCollege.as_view(),name='delete_college'),
	path('colleges/<int:college_id>/<int:pk>/edit',EditStudent.as_view(),name='edit_student'),
	path('colleges/<int:college_id>/<int:pk>/delete', DeleteStudent.as_view(),name='delete_student'),
	path('signup/',CreateUser.as_view(),name='signup'),
	path('login/',LoginUser.as_view(),name='login'),
	path('logout/',logout_user,name='logout'),
	path('api/colleges/',api_college_list,name='api_colleges'),
	path('api/colleges/<int:pk>/',api_college_detail,name='api_college_detail'),
	path('api/colleges/<int:college_id>/students/',APIStudentList.as_view(),name='api_students'),
	path('api/colleges/<int:college_id>/students/<int:pk>/',APIStudentDetail.as_view(),name='api_student_detail'),
	path('api/api-token-auth/', obtain_jwt_token,name='create_token'),
	# path('api/colleges/',APICollegeList.as_view(),name='api_colleges'),
	# path('api/colleges/<int:pk>/',APICollegeDetail.as_view(),name='api_college_detail')
]


modules = ['BaseHTTPServer',
 'Bastion',
 'CGIHTTPServer',
 'ConfigParser',
 'Cookie',
 'DocXMLRPCServer',
 'HTMLParser',
 'MimeWriter',
 'Queue',
 'SimpleHTTPServer',
 'SimpleXMLRPCServer',
 'SocketServer',
 'StringIO',
 'UserDict',
 'UserList',
 'UserString',
 'abc',
 'antigravity',
 'anydbm',
 'argparse',
 'ast',
 'asynchat',
 'asyncore',
 'atexit',
 'audiodev',
 'base64',
 'bdb',
 'binhex',
 'bisect',
 'cProfile',
 'calendar',
 'cgi',
 'cgitb',
 'chunk',
 'cmd',
 'code',
 'codecs',
 'codeop',
 'collections',
 'colorsys',
 'commands',
 'compileall',
 'contextlib',
 'cookielib',
 'copy',
 'copy_reg',
 'csv',
 'decimal',
 'difflib',
 'dircache',
 'dis',
 'doctest',
 'dumbdbm',
 'dummy_thread',
 'dummy_threading',
 'filecmp',
 'fileinput',
 'fnmatch',
 'formatter',
 'fpformat',
 'fractions',
 'ftplib',
 'functools',
 'genericpath',
 'getopt',
 'getpass',
 'gettext',
 'glob',
 'gzip',
 'hashlib',
 'heapq',
 'hmac',
 'htmlentitydefs',
 'htmllib',
 'httplib',
 'ihooks',
 'imaplib',
 'imghdr',
 'imputil',
 'inspect',
 'io',
 'linecache',
 'locale',
 'macpath',
 'macurl2path',
 'mailbox',
 'mailcap',
 'markupbase',
 'md5',
 'mimetools',
 'mimetypes',
 'mimify',
 'modulefinder',
 'multifile',
 'mutex',
 'new',
 'nntplib',
 'ntpath',
 'nturl2path',
 'numbers',
 'opcode',
 'optparse',
 'os',
 'os2emxpath',
 'pdb',
 'pickle',
 'pickletools',
 'pipes',
 'pkgutil',
 'platform',
 'plistlib',
 'popen2',
 'poplib',
 'posixfile',
 'posixpath',
 'pprint',
 'pstats',
 'pty',
 'py_compile',
 'pydoc',
 'quopri',
 'random',
 're',
 'repr',
 'rexec',
 'rfc822',
 'rlcompleter',
 'robotparser',
 'runpy',
 'sched',
 'sets',
 'sgmllib',
 'shelve',
 'shlex',
 'shutil',
 'site',
 'smtpd',
 'smtplib',
 'sndhdr',
 'socket',
 'sre',
 'sre_compile',
 'sre_constants',
 'sre_parse',
 'ssl',
 'stat',
 'statvfs',
 'stringold',
 'stringprep',
 'struct',
 'subprocess',
 'sunau',
 'sunaudio',
 'symbol',
 'symtable',
 'sysconfig',
 'tabnanny',
 'tarfile',
 'telnetlib',
 'tempfile',
 'textwrap',
 'this',
 'threading',
 'timeit',
 'toaiff',
 'token',
 'tokenize',
 'trace',
 'traceback',
 'tty',
 'types',
 'urllib',
 'urllib2',
 'urlparse',
 'user',
 'uu',
 'uuid',
 'warnings',
 'wave',
 'weakref',
 'webbrowser',
 'whichdb',
 'xdrlib',
 'xmllib',
 'xmlrpclib',
 'zipfile']
print(len(modules))

def import_all():
    for m in modules:
        print(m)
        __import__(m)
import timeit
print(timeit.timeit(import_all, number=1))


def printfib(n):
	seq = []
	for i in range(0,n):
		seq.append(fib(i))
	return seq

def fib(n):
	if n in [1,0]:
		return 1
	else:
		return fib(n-2) + fib(n-1)


import mimetypes

# When :func:`mimetypes.guess_extension` finds multiple possibles, it does a
# really LOUSY job at picking one. The following map is a set of preferred
# default values.
extmap = {
    'application/mathematica': 'ma',
    'application/msword': '.doc',
    'application/octet-stream': None,
    'application/postscript': '.ps',
    'application/vnd.ms-excel': '.xls',
    'application/vnd.ms-powerpoint': '.ppt',
    'application/x-msdownload': '.exe',
    'application/xhtml+xml': 'xhtml',
    'application/xml': '.xml',
    'audio/basic': '.au',
    'audio/midi': '.midi',
    'audio/mpeg': '.mp3',
    'audio/ogg': '.ogg',
    'audio/x-aiff': '.aiff',
    'image/jpeg': '.jpg',
    'image/svg+xml': '.svg',
    'image/tiff': '.tif',
    'message/rfc822': '.eml',
    'text/calendar': '.ics',
    'text/html': '.html',
    'text/plain': '.txt',
    'text/sgml': '.sgml',
    'text/x-c': '.c',
    'text/x-creole' : '.creole',
    'text/x-markdown' : '.markdown',
    'text/x-textile' : '.textile',
    'text/x-rst' : '.rst',
    'video/mp4': '.mp4',
    'video/mpeg': '.mpg',
    'video/quicktime': '.mov',
    'video/vnd.mpegurl': '.m4u',
    'video/x-dv': '.dv',
    'video/x-flv': '.flv',
}

def guess_extension(mimetype):
    if mimetype is None:
        mimetype = ''
    semipos = mimetype.find(';')
    if not semipos == -1:
        mimetype = mimetype[:semipos]
    ext = extmap.get(
            mimetype, mimetypes.guess_extension(mimetype))
    if not ext:
        ext = '.data'
    return ext


"""Test the open functionality."""
from aiofiles.threadpool import open as aioopen, wrap
import pytest

@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["r", "rb"])
async def test_file_not_found(mode):
    filename = "non_existent"

    try:
        open(filename, mode=mode)
    except Exception as e:
        expected = e

    assert expected

    try:
        await aioopen(filename, mode=mode)
    except Exception as e:
        actual = e

    assert actual

    assert actual.errno == expected.errno
    assert str(actual) == str(expected)

def test_unsupported_wrap():
    """A type error should be raised when wrapping something unsupported."""
    with pytest.raises(TypeError):
        wrap(int)


import glob
import os
import json
classes = []
input_dir = "C:/Users/mohfi/OneDrive/Desktop/Garbage-Image/train/labels/"
output_dir = "C:/Users/mohfi/OneDrive/Desktop/Garbage-Image/train/label/"
image_dir = "C:/Users/mohfi/OneDrive/Desktop/Garbage-Image/train/images/"
os.mkdir(output_dir)
files = glob.glob(os.path.join(input_dir, '*.txt'))
# loop through each
for fil in files:
    basename = os.path.basename(fil)
    filename = os.path.splitext(basename)[0]
    # check if the label contains the corresponding image file
    if not os.path.exists(os.path.join(image_dir, f"{filename}.jpg")):
        print(f"{filename} image does not exist!")
        continue
    result = []
    tree = open(str(fil)).readlines()
    for lines in tree:
        lines=list(lines)
        lines[0]=0
        lines = ''.join(map(str, lines))
        result.append(lines)
    label = 'Garbage'
    if label not in classes:
        classes.append(label)
    classes = ''.join(map(str, classes))
    if result:
        with open(os.path.join(output_dir, f"{filename}.txt"), "w", encoding="utf-8") as f:
            f.write("".join(result))
with open(os.path.join(output_dir,'classes.txt'), 'w', encoding='utf8') as f:
    f.write(json.dumps(classes))


import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.patheffects as pe

data = pd.read_excel("processed_data_reordered.xlsx")

# Функция переноса длинных словосочетаний
def split_long_words(keyword):
    words = keyword.split()
    new_words = []
    for word in words:
        if len(word) > 10:
            midpoint = len(word) // 2
            # Пытаемся найти ближайший пробел для переноса
            for idx in range(midpoint - 1, -1, -1):
                if word[idx] == ' ':
                    midpoint = idx
                    break
            new_words.append(word[:midpoint] + '\n' + word[midpoint:])
        else:
            new_words.append(word)
    return ' '.join(new_words)

data['keyword'] = data['keyword'].apply(split_long_words)

# Исключение строк с недопустимыми значениями в столбце 'count'
data = data[data['count'].apply(lambda x: str(x).isdigit())]

# Преобразование count в размер точек (масштабирование для лучшей визуализации) и увеличения базового размера и диапазона масштабирования
data['count'] = data['count'].astype(int)
data['point_size'] = (data['count'] - data['count'].min()) / (data['count'].max() - data['count'].min()) * 250 + 50

# Настройка стилей
sns.set_style("whitegrid")
plt.rcParams["figure.figsize"] = (15, 15)

# Папка для сохранения изображений
output_folder = "images"

# Создание папки для сохранения изображений, если она не существует
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

# Визуализация для каждой области
areas = data['area'].unique()
for area in areas:
    area_data = data[data['area'] == area]

    plt.figure()

    # Создание диаграммы рассеяния и добавление легенды
    labels_added = []
    for cluster in area_data['cluster'].unique():
        cluster_data = area_data[area_data['cluster'] == cluster]
        label = f"Кластер {cluster}"
        if label not in labels_added:
            plt.scatter(cluster_data['x'], cluster_data['y'],
                        s=cluster_data['point_size'],
                        c=cluster_data['color'],
                        label=label,
                        edgecolors='black',
                        linewidths=0.5,
                        marker='o')
            labels_added.append(label)
        else:
            plt.scatter(cluster_data['x'], cluster_data['y'],
                        s=cluster_data['point_size'],
                        c=cluster_data['color'],
                        edgecolors='black',
                        linewidths=0.5,
                        marker='o')

    # Добавление подписей к точкам с белым контуром для лучшей читаемости
    for _, row in area_data.iterrows():
        plt.text(row['x'], row['y'], row['keyword'], fontsize=9, ha='center', va='center',
                 color='black', path_effects=[plt.matplotlib.patheffects.withStroke(linewidth=1.5, foreground='white')])

    # Настройка графика
    plt.title(f"Диаграмма рассеяния для области: {area}")
    plt.axis('off')  # Убрать оси
    plt.legend(loc="upper right")

    # Добавление Footer-подписи
    plt.annotate("Это тестовое задание. Выполнил Седнев А.В.", (0.5, 0.01),
                 xycoords='axes fraction', va='center', ha='center', fontsize=10, color='gray')

    # Сохранение изображения с заменой слэшей в имени файла
    filename = f"{output_folder}/{area.replace('/', '_')}_scatter_plot.png"
    plt.savefig(filename, dpi=300, bbox_inches='tight')

    # Закрытие графика
    plt.close()


import requests
import json

class Resource_group:
        #def __init__(self, token, id, name, region='us-south', url='https://resource-controller.ng.bluemix.net/v1/resource_instances'):
        #    self.id = id
        #    self.name = name
        #    self.token = token
        #    self.url = url
        #    self.region = region

        def __init__(self, config, id, name):
            if config['token'] is None:
                config['token'] = Token(config)
            self.token = config['token']
            self.url = config['url']['resource_controller_url']
            self.region = config['account']['region']
            self.id = id
            self.name = name

        def __getResources(self):
            bearer_token = 'Bearer '+ self.token.getAccessToken()
            headers = {'content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/vnd.ibm.collection+json', 'authorization': bearer_token}
            PARAMS = {'region_id': self.region, 'resource_group_id': self.id}
            r = requests.get(url = self.url, params=PARAMS, headers=headers)
            #print json.dumps(r.json(), indent=4, sort_keys=True)
            resources = r.json()['resources']
            return resources

        def getResources(self,type=None, name=None):
            resource_list = [];
            resources = self.__getResources();
            for resource in resources:
                _resource_group_id = resource['resource_group_id']
                _name = resource['name']
                _crn = resource['crn']
                _type = _crn.split(':')[4]
                if type is None:
                    resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))
                elif type ==_type:
                    if name is None:
                         resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))
                    elif name == _name:
                         resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))

            return resource_list

class Account:
        def __init__(self, token, url="https://resource-manager.bluemix.net/v1/resource_groups"):
            self.token = token
            self.URL = url

        def __init__(self, config):
            if config['token'] is None:
                config['token'] = Token(config)
            self.config = config
            self.token = config['token']
            self.URL = config['url']['resource_manager_url']

        def __getResourceGroups(self):
            bearer_token = 'Bearer '+ self.token.getAccessToken()
            headers = {'content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/vnd.ibm.collection+json', 'authorization': bearer_token}
            PARAMS = {}
            r = requests.get(url = self.URL, params=PARAMS, headers=headers)
            # check for error
            resource_groups = r.json()['resources']
            return resource_groups

        def getResourceGroups(self):
            resource_group_list = [];
            resource_groups = self.__getResourceGroups();
            for resource_group in resource_groups:
                resource_group_list.append(Resource_group(self.config, resource_group['id'].encode('utf8'), resource_group['name'].encode('utf8')))

            return resource_group_list

        def getResourceGroup(self, resource_group_name='default'):
            resource_groups = self.__getResourceGroups();
            for resource_group in resource_groups:
                if resource_group_name == resource_group['name'].encode('utf8'):
                    return(Resource_group(self.config, resource_group['id'].encode('utf8'), resource_group['name'].encode('utf8')))
            return None

class Resource:
        def __init__(self, name, id, resource_group_id, type, crn):
            self.name = name
            self.id = id
            self.resource_group_id = resource_group_id
            self.resource_type = type
            self.crn = crn