from os.path import join as pjoin

import matplotlib.pyplot as plt
import numpy as np
import obspy
from matplotlib.patches import Rectangle
from obspy import UTCDateTime as utct
from tqdm import tqdm

from mqs_reports.catalog import Catalog
from mqs_reports.event import EVENT_TYPES_PRINT
from mqs_reports.utils import create_ZNE_HG, remove_sensitivity_stable, solify

SECONDS_PER_DAY = 86400.

class Noise:
    def __init__(self,
                 data: dict = None,
                 sc3_dir: str = None,
                 starttime: obspy.UTCDateTime = None,
                 endtime: obspy.UTCDateTime = None,
                 inv: obspy.Inventory = None,
                 winlen_sec: float = None,
                 fmin_LF: float = 1. / 6.,
                 fmax_LF: float = 1. / 1.5,
                 fmin_HF: float = 2.2,
                 fmax_HF: float = 2.6,
                 fmin_HF_broad: float = 1.2,
                 fmax_HF_broad: float = 3.0,
                 fmin_press: float = 1. / 12.,
                 fmax_press: float = 1. / 1.5
                 ):
        self.sols_quant = None
        if data is None:
            self.sc3_dir = sc3_dir
            self.winlen_sec = winlen_sec

            self.stds_HF_broad = 0.
            self.stds_HF = 0.
            self.stds_LF = 0.
            self.stds_press = 0.
            self.times = 0.
            self.times_LMST = 0.
            self.sol = 0.
            self.fmin_LF = fmin_LF
            self.fmax_LF = fmax_LF
            self.fmin_HF = fmin_HF
            self.fmax_HF = fmax_HF
            self.fmin_HF_broad = fmin_HF_broad
            self.fmax_HF_broad = fmax_HF_broad
            self.fmin_press = fmin_press
            self.fmax_press = fmax_press
            self._add_data(starttime=starttime,
                           endtime=endtime,
                           inv=inv)

        else:
            self.stds_HF = np.asarray(data['stds_HF'])
            self.stds_LF = np.asarray(data['stds_LF'])
            self.stds_HF_broad = np.asarray(data['stds_HF_broad'])
            self.stds_press = np.asarray(data['stds_press'])
            self.times = np.asarray(data['times'])
            self.times_LMST = np.asarray(data['times_LMST'])
            self.sol = np.asarray(data['sol'])
            self.fmin_LF = data['freqs'][0]
            self.fmax_LF = data['freqs'][1]
            self.fmin_HF = data['freqs'][2]
            self.fmax_HF = data['freqs'][3]
            self.fmin_press = data['freqs'][4]
            self.fmax_press = data['freqs'][5]
            self.fmin_HF_broad = 1.2  # data['freqs'][6]
            self.fmax_HF_broad = 3.0  # data['freqs'][7]

    def __str__(self):
        fmt = 'Noise from %s to %s, time windows: %d(HF), %d(LF)'
        return fmt % (self.times[0].date, self.times[-1].date,
                      len(self.stds_LF), len(self.stds_HF))

    def _add_data(self,
                  starttime: obspy.UTCDateTime,
                  endtime: obspy.UTCDateTime,
                  inv: obspy.Inventory,
                  ):

        dirnam = pjoin(self.sc3_dir, 'op/data/waveform/%d/XB/ELYSE/B[LH]?.D')
        filenam_VBB_HG = 'XB.ELYSE.0[237].B[LH]?.D.%d.%03d'

        dirnam_pressure = pjoin(self.sc3_dir,
                                'op/data/waveform/%d/XB/ELYSE/?DO.D')

        jday_start = starttime.julday
        jday_end = int(float(endtime - starttime) / SECONDS_PER_DAY
                       + jday_start)
        year_start = starttime.year

        stds_HF = list()
        stds_LF = list()
        stds_press = list()
        stds_HF_broad = list()
        times = list()
        times_LMST = list()
        sol = list()
        print('reading noise seismic data from %s (%d-%d)' % (self.sc3_dir,
                                                              jday_start,
                                                              jday_end))
        for jday in tqdm(range(jday_start, jday_end)):
            year = year_start + (jday // 365)
            try:
                # TODO: This will fail in leap years
                fnam = pjoin(dirnam % year,
                             filenam_VBB_HG % (year, jday % 365))
                st = obspy.read(fnam)
            except Exception:
                print('did not find %s' % fnam)
                st = obspy.Stream()
            try:
                # Before switching to L2 continuous data
                if year == 2019 and jday < 120:
                    filenam_pressure = 'XB.ELYSE.02.MDO.D.%d.%03d'
                else:
                    filenam_pressure = 'XB.ELYSE.03.BDO.D.%d.%03d'
                # TODO: This will fail in leap years
                fnam_press = pjoin(dirnam_pressure % year,
                                   filenam_pressure % (year, jday % 365))
                st_press = obspy.read(fnam_press)
                if year > 2019 or jday > 120:
                    st_press.decimate(5)

            except Exception:
                print('did not find %s' % fnam)
                st_press = obspy.Stream()
            st.merge()
            if len(st.select(location='03')) == 3:
                st = st.select(location='03')
            elif len(st.select(location='02')) == 3:
                st = st.select(location='02')
            else:
                st = st.select(location='07')
            if len(st) == 3:
                for tr in st:
                    remove_sensitivity_stable(tr, inv)
                st = create_ZNE_HG(st, inv=inv)
                st = st.select(channel='B[HL]Z')
                st.filter('highpass', freq=1. / 10., corners=8)
                st.integrate()
                st.filter('highpass', freq=1. / 10., corners=8)

                st_filt_HF = st.copy()
                st_filt_HF.filter('highpass', freq=self.fmin_HF, corners=16)
                st_filt_HF.filter('lowpass', freq=self.fmax_HF, corners=16)

                st_filt_HF_broad = st.copy()
                st_filt_HF_broad.filter('highpass', freq=self.fmin_HF_broad,
                                        corners=16)
                st_filt_HF_broad.filter('lowpass', freq=self.fmax_HF_broad,
                                        corners=16)

                st_filt_LF = st.copy()
                st_filt_LF.filter('highpass', freq=self.fmin_LF, corners=16)
                st_filt_LF.filter('lowpass', freq=self.fmax_LF, corners=16)

                for tr in st_press:
                    remove_sensitivity_stable(tr, inv=inv)
                st_press.detrend()
                st_press.taper(max_length=10., max_percentage=1.)
                st_press.merge(method=0, fill_value='interpolate')
                st_press.filter('highpass', freq=self.fmin_press, corners=16)
                st_press.filter('highpass', freq=self.fmin_press, corners=16)
                st_press.filter('lowpass', freq=self.fmax_press, corners=16)

                for t in np.arange(0, SECONDS_PER_DAY, self.winlen_sec):
                    t0 = utct('%04d%03d' % (year, jday % 365)) + t
                    t1 = t0 + self.winlen_sec
                    st_win = st.slice(starttime=t0, endtime=t1)
                    st_filt_HF_broad_win = st_filt_HF_broad.slice(
                                                      starttime=t0,
                                                      endtime=t1)
                    st_filt_HF_win = st_filt_HF.slice(starttime=t0,
                                                      endtime=t1)
                    st_filt_LF_win = st_filt_LF.slice(starttime=t0,
                                                      endtime=t1)

                    st_press_win = st_press.slice(starttime=t0, endtime=t1)
                    if len(st_win) > 0 and st_win[0].stats.npts > 10:
                        std_HF_broad = st_filt_HF_broad_win[0].std()
                        std_HF = st_filt_HF_win[0].std()
                        std_LF = st_filt_LF_win[0].std()

                        if len(st_press_win) > 0 and \
                                st_press_win[0].stats.npts > 10:
                            std_press = st_press_win[0].std()
                        else:
                            std_press = 0.

                        stds_press.append(std_press)
                        stds_HF_broad.append(std_HF_broad)
                        stds_HF.append(std_HF)
                        stds_LF.append(std_LF)
                        t0_lmst = solify(t0)

                        times.append(t0)
                        times_LMST.append(float(t0_lmst) % SECONDS_PER_DAY)
                        sol.append(1 + int(float(t0_lmst) / SECONDS_PER_DAY))

        self.stds_HF_broad = np.asarray(stds_HF_broad)
        self.stds_HF = np.asarray(stds_HF)
        self.stds_LF = np.asarray(stds_LF)
        self.stds_press = np.asarray(stds_press)
        self.times = np.asarray(times)
        self.times_LMST = np.asarray(times_LMST)
        self.sol = np.asarray(sol)

    def select(self,
               starttime: utct = None,
               endtime: utct = None,
               ):
        """
        Return new Catalog object only with the events that match the given
        criteria (e.g. all with name=="S026?a").
        Criteria can either be given as string with wildcards or as tuple of
        allowed values.
        :param starttime: minimum origin time (in UTC)
        :param endtime: maximum origin time (in UTC)
        :return:
        """
        import copy

        self_new = copy.deepcopy(self)

        bol = np.array((self.times > utct(starttime),
                        self.times < utct(endtime))
                       ).all(axis=0)
        self_new.stds_HF_broad = self.stds_HF_broad[bol]
        self_new.stds_HF = self.stds_HF[bol]
        self_new.stds_LF = self.stds_LF[bol]
        self_new.stds_press = self.stds_press[bol]
        self_new.times_LMST = self.times_LMST[bol]
        self_new.sol = self.sol[bol]
        self_new.times = self.times[bol]

        return self_new

    def save(self, fnam):
        np.savez(fnam,
                 stds_HF_broad=self.stds_HF_broad,
                 stds_HF=self.stds_HF,
                 stds_LF=self.stds_LF,
                 stds_press=self.stds_press,
                 times=self.times,
                 times_LMST=self.times_LMST,
                 sol=self.sol,
                 freqs=[self.fmin_LF, self.fmax_LF,
                        self.fmin_HF, self.fmax_HF,
                        self.fmin_press, self.fmax_press,
                        self.fmin_HF_broad, self.fmax_HF_broad],
                 winlen_sec=self.winlen_sec)

    def save_ascii(self, fnam):
        with open(fnam, 'w') as fid:
            fid.write(f"{'time (UTC)':>20s}, {'time (LMST)':>11s}, "
                      f"{'std_LF':>10s}, {'std_HF':>10s}, "
                      f"{'std_broad':>10s}, {'std_press':>10s}"
                      f"\n")
            for time, sol, time_LMST, std_LF, std_HF, std_press, std_broad in \
                    zip(self.times, self.sol, self.times_LMST, self.stds_LF,
                        self.stds_HF, self.stds_press, self.stds_HF_broad):
                t = utct(time)
                t_LMST = utct(time_LMST)
                for s in (std_LF, std_HF, std_press, std_broad):
                    if s < 1e-15:
                        s = 0.0

                fid.write(f"{t.strftime('%Y-%m-%dT:%H:%M:%S'):20s}, "
                          f"{sol:03d}M{t_LMST.strftime('%H:%M:%S'):8s}, "
                          f"{std_LF:10.4e}, {std_HF:10.4e}, "
                          f"{std_broad:10.4e}, {std_press:10.4e}\n")

    def plot_noise_stats(self, sol_start=80, sol_end=None,
                         ax=None, show=True):
        power_bins, p_LF, p_HF = self.calc_noise_stats(sol_end,
                                                       sol_start)

        if ax is None:
            fig, ax = plt.subplots(1, 1)

        ax.plot(power_bins, p_LF,
                label='LF, 1.5 - 6 seconds')
        ax.plot(power_bins, p_HF,
                label='HF, 2 - 3 Hz')
        plt.legend()

        if show:
            plt.show()

    def calc_noise_stats(self, sol_end=None, sol_start=80):
        if sol_end is None:
            # Now
            sol_end = float(solify(utct())) // SECONDS_PER_DAY
        binwidth = 2.
        bins = np.arange(-260, -120, binwidth)
        bol_LF = np.array([np.isfinite(self.stds_LF),
                           self.sol > sol_start,
                           self.sol < sol_end]).all(axis=0)
        power_LF, bins_tmp = np.histogram(
            20 * np.log10(self.stds_LF[bol_LF]),
            bins=bins, density=True)
        bol_HF = np.array([np.isfinite(self.stds_HF),
                           self.sol > sol_start,
                           self.sol < sol_end]).all(axis=0)
        power_HF, bins_tmp = np.histogram(
            20 * np.log10(self.stds_HF[bol_HF]),
            bins=bins, density=True)
        bins = bins[0:-1] + binwidth / 2.
        p_LF = np.cumsum(power_LF) * binwidth
        p_HF = np.cumsum(power_HF) * binwidth
        return bins, p_LF, p_HF

    def plot_daystats(self, cat: Catalog = None,
                      sol_start: int = 80, sol_end: int = 500, data_apss=False,
                      fnam_out='noise_vs_eventamplitudes.png', extra_data=None,
                      grouping='quantiles', color_scheme='standard',
                      cmap_dist='plasma_r', tau_data=None, metal=False):

        def make_patch_spines_invisible(ax):
            ax.set_frame_on(True)
            ax.patch.set_visible(False)
            for sp in ax.spines.values():
                sp.set_visible(False)

        if self.sols_quant is None:
            if grouping == 'quantiles':
                self.calc_quantiles(sol_end, sol_start,
                                    qs=(0.10, 0.33, 0.67, 0.9))
            elif grouping == 'timewindows':
                self.calc_time_windows(sol_end, sol_start)
                if metal:
                    labels = ['quiet\n(17-22 LMST)',
                              'NÖCTURNAL WAVEGÜIDE\n(22-5 LMST)',
                              'SÜNRÏSE\n(5-9 LMST)',
                              'SËISMIC NÖISË\n(9-17 LMST)']
                else:
                    labels = ['quiet\n(17-22 LMST)',
                              'noisy night\n(22-5 LMST)',
                              'morning\n(5-9 LMST)',
                              'day\n(9-17 LMST)']
            else:
                raise ValueError(f'Unknown value for grouping: {grouping}')

            self.save_quantiles(fnam=f'noise_{grouping}.npz')
        if grouping == 'quantiles':
            labels = ['10%', '33%', '67%', '90%']
        elif grouping == 'timewindows':
            if metal:
                labels = ['quiet\n(17-22 LMST)',
                          'NÖCTURNAL WAVEGÜIDE\n(22-5 LMST)',
                          'SÜNRÏSE\n(5-9 LMST)',
                          'SËISMIC NÖISË\n(9-17 LMST)']
            else:
                labels = ['quiet\n(17-22 LMST)', 'noisy night\n(22-5 LMST)',
                          'morning\n(5-9 LMST)', 'day\n(9-17 LMST)']

        fig = plt.figure(figsize=(16, 9))
        # ax_HF = ax[1
        # ax_LF = ax[0]
        h_base = 0.09
        w_base = 0.05
        w_LF = 0.85
        h_pad = 0.06
        h_LF = (0.97 - h_base - h_pad) / 2.

        if data_apss:
            h_apss = 0.2
            h_LF = (0.97 - h_base - 2 * h_pad - h_apss) / 2.
            # h_LF -= h_apss / 2. + h_pad

        ax_HF = fig.add_axes([w_base, h_base,
                              w_LF, h_LF])
        if data_apss:
            ax_apss = fig.add_axes([w_base, h_base + h_LF * 2 + h_pad * 2,
                                    w_LF, h_apss],
                                   sharex=ax_HF)
            if extra_data is not None:
                ax_extra = ax_apss.twinx()
                m = 's'
                ax_extra.plot(extra_data[0], extra_data[1],
                              markeredgecolor='k', markerfacecolor='C0',
                              marker=m, ls=None, lw=0., ms=4.,
                              label='DuDes per Sol\n(A. Spiga)')
                ax_extra.set_yticks(np.arange(0, 90, 15))
                ax_extra.set_ylim(0, 90)
                ax_extra.set_ylabel('number of DuDes')
                ax_extra.yaxis.label.set_color('C0')
                ax_extra.tick_params(axis='y', colors='C0')
                ax_extra.legend(loc='upper left')
                if tau_data is not None:
                    ax_tau = ax_apss.twinx()
                    # Offset the right spine of par2.  The ticks and label have
                    # already been
                    # placed on the right by twinx above.
                    ax_tau.spines["right"].set_position(("axes", 1.05))
                    # Having been created by twinx, par2 has its frame off, so
                    # the line of its
                    # detached spine is invisible.  First, activate the frame
                    # but make the patch
                    # and spines invisible.
                    make_patch_spines_invisible(ax_tau)
                    # Second, show the right spine.
                    ax_tau.spines["right"].set_visible(True)
                    m = 'o'
                    ax_tau.plot(tau_data[0], tau_data[1],
                                markeredgecolor='k', markerfacecolor='orange',
                                marker=m, ls=None, lw=0., ms=6.,
                                label='tau\n(M. Lemmon)')
                    ax_tau.set_yticks(np.arange(0.3, 2.0, 0.3))
                    ax_tau.set_ylim(0.3, 1.8)
                    ax_tau.set_ylabel('tau')
                    ax_tau.yaxis.label.set_color('orange')
                    ax_tau.tick_params(axis='y', colors='orange')
                    ax_tau.legend()
        else:
            ax_apss = fig.add_axes([1.2, 1.2, 0.1, 0.1])

        ax_LF = fig.add_axes([w_base, h_base + h_LF + h_pad,
                              w_LF, h_LF],
                             sharex=ax_HF)

        cols = ['black', 'darkgrey', 'grey', 'black']
        ls = ['dotted', 'dashed', 'dotted', 'dashed']
        for i in range(self.quantiles_LF.shape[1] - 1, -1, -1):
            ax_LF.plot(self.sols_quant - 1.,
                       10 * np.log10(self.quantiles_LF[:, i]),
                       label=labels[i], c=cols[i], ls=ls[i])
        ax_LF.fill_between(x=self.sols_quant - 1.,
                           y1=-300,
                           y2=10 * np.log10(self.quantiles_LF[:, 0]),
                           facecolor='lightgrey')

        for i in range(self.quantiles_HF.shape[1] - 1, -1, -1):
            ax_HF.plot(self.sols_quant - 1.,
                       10 * np.log10(self.quantiles_HF[:, i]),
                       label=labels[i], c=cols[i], ls=ls[i])
        ax_HF.fill_between(x=self.sols_quant - 1.,
                           y1=-300,
                           y2=10 * np.log10(self.quantiles_HF[:, 0]),
                           facecolor='lightgrey')
        if data_apss:
            for i in range(self.quantiles_press.shape[1] - 1, -1, -1):
                ax_apss.plot(self.sols_quant - 1.,
                             10 * np.log10(self.quantiles_press[:, i]),
                             label=labels[i], c=cols[i], ls=ls[i])
            ax_apss.fill_between(x=self.sols_quant - 1.,
                                 y1=-300,
                                 y2=10 * np.log10(self.quantiles_press[:, 0]),
                                 facecolor='lightgrey')

        # Add one-year marker
        SOLS_PER_YEAR = 668
        for ax in [ax_LF, ax_HF]:
            ax.axvline(x=sol_start + SOLS_PER_YEAR, ls='dashed',
                       color='black', lw=1)
        if data_apss:
            ax_apss.axvline(x=sol_start + SOLS_PER_YEAR, ls='dashed',
                            color='black', lw=1)

        HF_times = []
        HF_amps = []
        HF_dists = []
        LF_times = []
        LF_amps = []
        LF_dists = []
        TF_times = []
        TF_amps = []
        TF_dists = []

        if color_scheme == 'standard':
            symbols = {'2.4_HZ': 'v',
                       'HIGH_FREQUENCY': '^',
                       'VERY_HIGH_FREQUENCY': '*',
                       'LOW_FREQUENCY': 's',
                       # 'ABOVE': 10,
                       'BROADBAND': 'D'}
        elif color_scheme == 'Knapmeyer':
            # Knapmeyer edition
            symbols = {'2.4_HZ': 'o',
                       'HIGH_FREQUENCY': 's',
                       'VERY_HIGH_FREQUENCY': 'v',
                       'LOW_FREQUENCY': 'o',
                       'BROADBAND': 'v'}
            cols = {'2.4_HZ':              [0.57, 0.86, 0.98],
                    'HIGH_FREQUENCY':      [0.57, 0.86, 0.98],
                    'VERY_HIGH_FREQUENCY': [0.57, 0.86, 0.98],
                    'LOW_FREQUENCY':       [0.45, 0.35, 0.22],
                    'BROADBAND':           [0.45, 0.35, 0.22]}
        else:
            raise ValueError('invalid argument for color_scheme: ' +
                             color_scheme)

        markers_HF = []
        markers_LF = []
        markers_TF = []

        if cat is not None:
            cmap = plt.cm.get_cmap(cmap_dist)
            cmap.set_under('lightgrey')
            for event in cat.select(event_type=['HF', 'VF']):
                markers_HF.append(symbols[event.mars_event_type])

                #  Plot HF events with high amplitudes at top of plot
                if event.amplitudes['A_24'] < -180:
                    HF_amps.append(event.amplitudes['A_24'])
                else:
                    HF_amps.append(-181)

                #  Make events without a distance lightgrey
                if event.distance is None:
                    HF_dists.append(-1)
                else:
                    HF_dists.append(event.distance)
                HF_times.append(float(solify(event.starttime)) /
                                SECONDS_PER_DAY)

            for event in cat.select(event_type=['24']):
                markers_TF.append(symbols[event.mars_event_type])
                if event.distance is None:
                    TF_dists.append(-1.)
                else:
                    TF_dists.append(event.distance)
                TF_times.append(float(solify(event.starttime)) /
                                SECONDS_PER_DAY)
                TF_amps.append(event.amplitudes['A_24'])

            for event in cat.select(event_type=['LF', 'BB']):
                markers_LF.append(symbols[event.mars_event_type])
                if event.distance is None:
                    LF_dists.append(-1.)
                else:
                    LF_dists.append(event.distance)
                amp_P = event.pick_amplitude(
                    pick='Peak_MbP',
                    comp='vertical',
                    fmin=self.fmin_LF,
                    fmax=self.fmax_LF,
                    instrument='VBB'
                    )
                amp_S = event.pick_amplitude(
                    pick='Peak_MbS',
                    comp='vertical',
                    fmin=self.fmin_LF,
                    fmax=self.fmax_LF,
                    instrument='VBB'
                    )
                amp = max(i for i in (amp_P, amp_S, 0.0) if i is not None)
                LF_times.append(float(solify(event.starttime)) /
                                SECONDS_PER_DAY)
                LF_amps.append(20 * np.log10(amp))

            for event_type, m in symbols.items():
                if event_type != 'ABOVE':
                    bol = np.array(markers_LF) == m
                    if len(bol) > 0:
                        if color_scheme == 'standard':
                            sc = ax_LF.scatter(x=np.array(LF_times)[bol],
                                               y=np.array(LF_amps)[bol],
                                               c=np.array(LF_dists)[bol],
                                               vmin=00., vmax=100., cmap=cmap,
                                               edgecolors='k', linewidths=0.5,
                                               s=80., marker=m, zorder=100)
                        else:
                            sc = ax_LF.scatter(x=np.array(LF_times)[bol],
                                               y=np.array(LF_amps)[bol],
                                               c=cols[event_type],
                                               edgecolors='k', linewidths=0.5,
                                               s=80., marker=m, zorder=100)

            cax = fig.add_axes([w_LF + w_base + 0.02,
                                h_base + 0.165,
                                0.018,
                                1 - h_base - 0.55])
            cb = plt.colorbar(sc, ax=ax_LF, cax=cax)
            cb.ax.set_ylabel('distance / degree', rotation=270.,
                             labelpad=4.45)
            for event_type, m in symbols.items():
                bol = np.array(markers_TF) == m
                if len(bol) > 0:
                    if color_scheme == 'standard':
                        sc = ax_HF.scatter(x=np.array(TF_times)[bol],
                                           y=np.array(TF_amps)[bol],
                                           c=np.array(TF_dists)[bol],
                                           vmin=00., vmax=100., cmap=cmap,
                                           edgecolors='k', linewidths=0.5,
                                           s=20., marker=m, zorder=8)
                    else:
                        sc = ax_HF.scatter(x=np.array(TF_times)[bol],
                                           y=np.array(TF_amps)[bol],
                                           c=cols[event_type],
                                           edgecolors='k', linewidths=0.5,
                                           s=20., marker=m, zorder=8)
            for event_type, m in symbols.items():
                bol = np.array(markers_HF) == m
                if len(bol) > 0:
                    if color_scheme == 'standard':
                        sc = ax_HF.scatter(x=np.array(HF_times)[bol],
                                           y=np.array(HF_amps)[bol],
                                           c=np.array(HF_dists)[bol],
                                           vmin=00., vmax=100., cmap=cmap,
                                           edgecolors='k', linewidths=0.5,
                                           s=40., marker=m, zorder=10)
                    else:
                        sc = ax_HF.scatter(x=np.array(HF_times)[bol],
                                           y=np.array(HF_amps)[bol],
                                           c=cols[event_type],
                                           edgecolors='k', linewidths=0.5,
                                           s=40., marker=m, zorder=10)

            # cax = plt.colorbar(sc, ax=ax_HF, use_gridspec=True,
            #                    fraction=0.08)
            # cax.ax.set_ylabel('distance / degree', rotation=270.,
            #                   labelpad=12.45)
            # if data_apss:
            #     cax = plt.colorbar(sc, ax=ax_apss, use_gridspec=True)
            #     cax.ax.set_ylabel('distance / degree', rotation=270.,
            #                       labelpad=12.45)

        for a in [ax_HF, ax_LF, ax_apss]:
            a.grid(True)
            rect = Rectangle(xy=(267, -300), width=20., height=400, zorder=10,
                             facecolor='darkgrey', edgecolor='black')
            a.add_patch(rect)

        sc = ax_HF.scatter(0, -300, label='Marsquake',
                           edgecolors='k', linewidths=0.5,
                           c='royalblue', s=80., marker='.')
        ax_HF.set_xlabel('Sol number')
        ax_LF.set_ylabel('PSD, displ. %3.1f-%3.1f sec. [dB]' %
                         (1. / self.fmax_LF, 1. / self.fmin_LF))
        ax_HF.set_ylabel('PSD, displ. %3.1f-%3.1f Hz. [dB]' %
                         (self.fmin_HF, self.fmax_HF))

        ax_LF.set_ylim(-210., -165.)
        ax_LF.set_title('Seismic power, low frequency,  ' +
                        '%3.1f-%3.1f seconds and LF/BB events' %
                        (1. / self.fmax_LF, 1. / self.fmin_LF))
        ax_HF.set_title('Seismic power, high frequency,  ' +
                        '%3.1f-%3.1f Hz and HF/2.4 Hz events' %
                        (self.fmin_HF, self.fmax_HF))
        ax_HF.set_ylim(-225., -180.)
        ax_LF.set_xlim(sol_start, sol_end)

        # ax_HF.text(0.7, -0.12, s=str(self),
        #            transform=ax_HF.transAxes)

        # Daytimes or quantiles legend
        l_LF = ax_LF.legend(bbox_to_anchor=(w_LF + w_base, 0.61),
                            loc='lower left',
                            bbox_transform=fig.transFigure, framealpha=1.0)
        l_LF.set_zorder(50)

        handles = []
        labels = []
        EVENT_TYPES_PRINT['VERY_HIGH_FREQUENCY'] = 'very high\nfrequency'
        EVENT_TYPES_PRINT['HIGH_FREQUENCY'] = 'high\nfrequency'
        for event_type, m in symbols.items():
            h, = ax_HF.plot(-100, -100, markeredgecolor='k',
                            markerfacecolor='white',
                            marker=m, ls=None, lw=0., ms=8.)
            handles.append(h)
            labels.append(EVENT_TYPES_PRINT[event_type])
        #  Event type legend
        l_HF = ax_HF.legend(handles=handles, labels=labels,
                            bbox_to_anchor=(w_LF + w_base, h_base - 0.01),
                            loc='lower left',
                            bbox_transform=fig.transFigure, framealpha=1.0)
        l_HF.set_zorder(50)

        if data_apss:
            ax_apss.set_title('Pressure power %3.1f-%3.1f seconds' %
                              (1. / self.fmax_press, 1. / self.fmin_press))
            ax_apss.set_ylabel('PSD, pressure.\n%3.1f-%4.1f sec. [dB]' %
                               (1. / self.fmax_press, 1. / self.fmin_press))
            ax_apss.set_ylim(-50, -20)

        plt.savefig(fnam_out, dpi=200)

    def read_quantiles(self, fnam):
        data = np.load(fnam)
        self.quantiles_HF = data['quantiles_HF']
        self.quantiles_LF = data['quantiles_LF']
        self.quantiles_press = data['quantiles_press']
        self.sols_quant = data['sols']

    def save_quantiles(self, fnam):
        np.savez(file=fnam,
                 quantiles_HF=self.quantiles_HF,
                 quantiles_LF=self.quantiles_LF,
                 quantiles_press=self.quantiles_press,
                 sols=self.sols_quant)

    def calc_quantiles(self, sol_end, sol_start, qs):

        self.sols_quant = np.arange(sol_start, sol_end + 1)
        self.quantiles_LF = np.zeros(
                (sol_end - sol_start + 1, len(qs)))
        self.quantiles_HF = np.zeros_like(self.quantiles_LF)
        self.quantiles_press = np.zeros_like(self.quantiles_LF)

        values_HF = np.zeros(len(qs))
        values_LF = np.zeros_like(values_HF)
        values_press = np.zeros_like(values_HF)

        i = 0
        print('Calculating noise quantiles')
        df_HF = self.fmax_HF - self.fmin_HF
        df_LF = self.fmax_LF - self.fmin_LF
        df_press = self.fmax_press - self.fmin_press
        for isol in tqdm(self.sols_quant):
            bol_sol = self.sol == isol - 1
            if sum(bol_sol) > 1:
                disp_LF = self.stds_LF[bol_sol] ** 2. / df_LF
                disp_HF = self.stds_HF[bol_sol] ** 2. / df_HF
                disp_press = self.stds_press[bol_sol] ** 2. / df_press
                values_LF = np.nanquantile(disp_LF, q=qs)
                values_HF = np.nanquantile(disp_HF, q=qs)
                values_press = np.nanquantile(disp_press, q=qs)

                self.quantiles_HF[i, :] = np.nan_to_num(values_HF)
                self.quantiles_LF[i, :] = np.nan_to_num(values_LF)
                self.quantiles_press[i, :] = np.nan_to_num(values_press)
            i += 1

        # Mask outliers
        self.quantiles_LF = np.ma.masked_less(self.quantiles_LF, value=1e-23)
        self.quantiles_HF = np.ma.masked_less(self.quantiles_HF, value=1e-23)
        self.quantiles_press = np.ma.masked_less(self.quantiles_press,
                                                 value=1e-6)
        self.quantiles_LF = np.ma.masked_greater(self.quantiles_LF, value=1e-8)
        self.quantiles_HF = np.ma.masked_greater(self.quantiles_HF, value=1e-8)
        self.quantiles_press = np.ma.masked_greater(self.quantiles_press,
                                                    value=1e-2)

    def calc_time_windows(self, sol_end, sol_start,
                          time_windows_hour=[[17, 22.0],
                                             [22.0, 5.0],
                                             [5.0, 9.0],
                                             [9.0, 17.]]):

        self.sols_quant = np.arange(sol_start, sol_end + 1)
        self.quantiles_LF = np.zeros(
            (sol_end - sol_start + 1, len(time_windows_hour)))
        self.quantiles_HF = np.zeros_like(self.quantiles_LF)
        self.quantiles_press = np.zeros_like(self.quantiles_LF)

        values_HF = np.zeros(len(time_windows_hour))
        values_LF = np.zeros_like(values_HF)
        values_press = np.zeros_like(values_HF)

        i = 0
        print('Calculating noise medians')
        df_HF = self.fmax_HF - self.fmin_HF
        df_LF = self.fmax_LF - self.fmin_LF
        df_press = self.fmax_press - self.fmin_press
        for isol in tqdm(self.sols_quant):
            bol_sol = self.sol == isol - 1
            times_this_sol = (self.times_LMST[bol_sol] % 1.) * 24.
            if sum(bol_sol) > 1:
                for iwindow, time_window in enumerate(time_windows_hour):
                    if time_window[0] < time_window[1]:
                        bol_hours = np.array((
                            (time_window[0] < times_this_sol),
                            (times_this_sol < time_window[1]))).all(axis=0)
                    else:
                        bol_hours = np.array((
                            (time_window[0] < times_this_sol),
                            (times_this_sol < time_window[1]))).any(axis=0)

                    disp_LF = self.stds_LF[bol_sol] ** 2. / df_LF
                    disp_HF = self.stds_HF[bol_sol] ** 2. / df_HF
                    disp_press = self.stds_press[bol_sol] ** 2. / df_press
                    values_LF[iwindow] = np.nanmedian(disp_LF[bol_hours])
                    values_HF[iwindow] = np.nanmedian(disp_HF[bol_hours])
                    values_press[iwindow] = np.nanmedian(disp_press[bol_hours])

                self.quantiles_HF[i, :] = np.nan_to_num(values_HF)
                self.quantiles_LF[i, :] = np.nan_to_num(values_LF)
                self.quantiles_press[i, :] = np.nan_to_num(values_press)
            i += 1

        # Mask outliers
        self.quantiles_LF = np.ma.masked_less(self.quantiles_LF, value=1e-23)
        self.quantiles_HF = np.ma.masked_less(self.quantiles_HF, value=1e-23)
        self.quantiles_press = np.ma.masked_less(self.quantiles_press,
                                                 value=1e-6)
        self.quantiles_LF = np.ma.masked_greater(self.quantiles_LF, value=1e-8)
        self.quantiles_HF = np.ma.masked_greater(self.quantiles_HF, value=1e-8)
        self.quantiles_press = np.ma.masked_greater(self.quantiles_press,
                                                    value=1e-2)

    def compare_events(self,
                       catalog=None,
                       threshold_dB: float = 3.):
        ratios = []
        for event in tqdm(catalog.select(event_type=['24', 'HF', 'VF'])):
            nwins_below = sum(event.amplitudes['A_24'] >
                              threshold_dB + 20 * np.log10(self.stds_HF))
            event.ratio = nwins_below / len(self.stds_HF)
            ratios.append(event.ratio)

        for event in tqdm(catalog.select(event_type=['LF', 'BB'])):
            amp_P = event.pick_amplitude(
                pick='Peak_MbP',
                comp='vertical',
                fmin=self.fmin_LF,
                fmax=self.fmax_LF,
                instrument='VBB'
                )
            amp_S = event.pick_amplitude(
                pick='Peak_MbS',
                comp='vertical',
                fmin=self.fmin_LF,
                fmax=self.fmax_LF,
                instrument='VBB'
                )
            if amp_S is None:
                amp = amp_P
            elif amp_P is None:
                amp = amp_S
            else:
                amp = max((amp_P, amp_S))
            if amp is not None:
                nwins_below = sum(20 * np.log10(amp) >
                                  threshold_dB + 20 * np.log10(self.stds_LF))
                event.ratio = nwins_below / len(self.stds_LF)
            else:
                event.ratio = None
            ratios.append(event.ratio)
        # import matplotlib.pyplot as plt

        # plt.hist(ratios, bins=20)
        # plt.show()

def read_noise(fnam):
    data = np.load(fnam, allow_pickle=True)
    data_read = dict()
    for name in data.files:
        data_read[name] = data[name]

    return Noise(data=data_read)

if __name__ == '__main__':
    sc3_path = '/mnt/mnt_sc3data'
    inv = obspy.read_inventory('mqs_reports/data/inventory.xml')
    # noise = Noise(sc3dir=sc3_path,
    #               starttime=utct('20190202'),
    #               endtime=utct('20191026'),
    #               inv=inv,
    #               winlen_sec=120.
    #               )
    # noise.save('noise_0301_1025.npz')
    noise = read_noise('noise_0301_1025.npz')
    noise.plot_noise_stats()

    cat = Catalog(fnam_quakeml='mqs_reports/data/catalog_20191024.xml',
                  quality=['A', 'B'])

    cat.load_distances(fnam_csv='./mqs_reports/data/manual_distances.csv')
    cat.read_waveforms(inv=inv, sc3dir=sc3_path)
    cat.calc_spectra(winlen_sec=10.)
    noise.plot_daystats(cat, sol_start=200, sol_end=300)
    noise.compare_events(cat)
    noise.plot_noise_stats()

def calc_Ls(t_LMST):
    import marstime
    from mqs_reports.utils import UTCify
    t = float(UTCify(t_LMST)) * 1e3

    j2000_offset = marstime.j2000_offset_tt(
        marstime.julian_tt(marstime.julian(t)))
    ls = marstime.Mars_Ls(j2000_ott=j2000_offset)  # + 30 * 86400e3))))
    return ls


from django.conf.urls import url,include

from blog.admin_views import PostView, DeletePost, NewPost, UpdatePostIndexView, AddPost, \
    UpdateDraft, UpdatePost, UpdateEditor, LogoutView, CarouselIndexView, CarouselEditView, \
    AddCarousel, DeleteCarousel, CarouselUpdateView, UpdateCarousel, markdown_image_upload_handler, \
    tinymce_image_upload_handler, UserSetView, NewUserView, AddUser, avatar_image_upload_handler

urlpatterns = [
    url(r'^admin/', include([
        url(r'^$', PostView.as_view(), name='index'),
        url(r'^delete/(?P<pk>[0-9]+)$', DeletePost.as_view()),
        url(r'^new$', NewPost.as_view()),
        url(r'^add$', AddPost.as_view()),
        url(r'^update/draft/(?P<pk>[0-9]+)$', UpdateDraft.as_view()),
        url(r'^update/post/(?P<pk>[0-9]+)$', UpdatePost.as_view()),
        url(r'^update/(?P<pk>[0-9]+)$', UpdatePostIndexView.as_view()),
        url(r'^upload/markdown/post$', markdown_image_upload_handler),
        url(r'^upload/tinymce/post$', tinymce_image_upload_handler),
        url(r'^update/editor$', UpdateEditor.as_view()),
        url(r'^carousel$', CarouselIndexView.as_view()),
        url(r'^new/carousel$', CarouselEditView.as_view()),
        url(r'^add/carousel$', AddCarousel.as_view()),
        url(r'^delete/carousel/(?P<pk>[0-9]+)$', DeleteCarousel.as_view()),
        url(r'^update/carousel/(?P<pk>[0-9]+)$', CarouselUpdateView.as_view()),
        url(r'^update/carousel/id/(?P<pk>[0-9]+)$', UpdateCarousel.as_view()),
        url(r'^repository$', PostView.as_view(), name='index'),
        url(r'^userset$', UserSetView.as_view()),
        url(r'^new/user$', NewUserView.as_view()),
        url(r'^add/user$', AddUser.as_view()),
        url(r'^set/upload/avatar$', avatar_image_upload_handler),
        url(r'^logout$', LogoutView.as_view()),
    ])),
]


import sys, os
import pickle as pkl
import numpy as np
import matplotlib.pyplot as plt
from skimage.io import imread, imsave

import torch
import torch.nn as nn
import torchvision.transforms as transform
from torchvision.models import vgg16, vgg19, resnet50, resnet101, densenet121, densenet169

if __name__ == "__main__":
    lucky_num = 50756711264384381850616619995309447969109689825336919605444730053665222018857 % (2 ** 32)
    np.random.seed(lucky_num)

    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    plt.switch_backend('agg')
    plt.subplots_adjust(wspace = 0.3)

    chosenIndices = np.random.choice(200, size = 3)

    inputDir = sys.argv[1]
    outputDir = sys.argv[2]

    # Label names are found here:
    # http://files.fast.ai/models/imagenet_class_index.json
    with open('../label.pkl', 'rb') as f:
        Dict = pkl.load(f)

    Mean = np.array([0.485, 0.456, 0.406])
    Std = np.array([0.229, 0.224, 0.225])
    model = resnet50(pretrained = True)
    model.eval()

    criterion = nn.CrossEntropyLoss().cuda()
    trans = transform.Compose([transform.ToTensor(), transform.Normalize(mean = Mean, std = Std)])

    for idx in chosenIndices:
        fig, ax = plt.subplots(1, 3, figsize=(12, 4))

        Image = imread("%s/%03d.png" % (inputDir, idx))
        OriPredict = model(trans(imread("%s/%03d.png" % (inputDir, idx))).unsqueeze(0)).detach().numpy()
        AttPredict = model(trans(imread("%s/%03d.png" % (outputDir, idx))).unsqueeze(0)).detach().numpy()

        X = [0, 1, 2]
        OriSorted = sorted([(i, a) for i, a in enumerate(OriPredict.reshape(-1))], key = lambda k : (k[1], k[0]), reverse = True)[:3]
        AttSorted = sorted([(i, a) for i, a in enumerate(AttPredict.reshape(-1))], key = lambda k : (k[1], k[0]), reverse = True)[:3]

        ax[0].axis('off')
        ax[0].imshow(Image)
        ax[0].set_title('Image %3d' % (idx))

        ax[1].bar(X, [pair[1] for pair in OriSorted], width = 0.4, tick_label = [Dict[str(pair[0])][1] for pair in OriSorted], color = '#00bfff')
        ax[1].set_title('Original Prediction')

        ax[2].bar(X, [pair[1] for pair in AttSorted], width = 0.4, tick_label = [Dict[str(pair[0])][1] for pair in AttSorted], color = '#ff69b4')
        ax[2].set_title('Attacked Prediction')

        fig.savefig('Prob4_Result_%03d.png' % (idx), dpi = 150)


'''
Adapted from: https://medium.com/@andrewdoesdata/deep-learning-with-15-lines-of-python-397e43bcf843
'''

# conda install -c conda-forge tensorflow

from sklearn import datasets
from tensorflow import keras
from tensorflow.keras.layers import Dense

# 1) import data
iris = datasets.load_iris()

# 2) prepare inputs
input_x = iris.data

# 3) prepare outputs: a binary class matrix
output_y = keras.utils.to_categorical(iris.target, 3)

# 4a) Create the model
model = keras.models.Sequential()
model.add(Dense(8, input_dim=4, activation='relu'))
model.add(Dense(3, activation='softmax'))

# 4b) Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# 4c) Fit the model
model.fit(input_x, output_y, epochs=150, batch_size=15, verbose=1)

# 4d) Evaluate the model
score = model.evaluate(input_x, output_y, batch_size=15)
score


import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

# Baixe os recursos necessários
nltk.download('stopwords')
nltk.download('wordnet')

# Exemplo de texto
text = "Os gatos estão pulando sobre as cercas e correndo nos jardins"

# Tokenização
tokens = nltk.word_tokenize(text)

# Remoção de stopwords
stop_words = set(stopwords.words('portuguese'))
filtered_tokens = [token for token in tokens if token.lower() not in stop_words]

# Lematização
lemmatizer = WordNetLemmatizer()
lemmatized_tokens = [lemmatizer.lemmatize(token) for token in filtered_tokens]

# Resultados
print("Tokens originais:", tokens)
print("Tokens filtrados:", filtered_tokens)
print("Tokens lematizados:", lemmatized_tokens)


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report

# Load the dataset
data_url = "https://example.com/diabetes_dataset.csv"
df = pd.read_csv(data_url)

# Data preprocessing
X = df.drop("diabetes", axis=1)
y = df["diabetes"]

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Train a Random Forest classifier
rf_model = RandomForestClassifier(random_state=42)
rf_model.fit(X_train, y_train)

# Make predictions on the test set
y_pred = rf_model.predict(X_test)

# Evaluate the model
accuracy = accuracy_score(y_test, y_pred)
report = classification_report(y_test, y_pred)

# Print the results
print("Random Forest Classifier Results:")
print(f"Accuracy: {accuracy:.2f}")
print("Classification Report:")
print(report)


# @ clean-up pycwt vs wavelet calls
import pycwt
import numpy as np
import scipy
from scipy.ndimage.filters import gaussian_filter
import pycwt as wavelet
from pycwt.helpers import find
from tqdm import tqdm
import pandas as pd
import copy
from scipy.signal import butter, filtfilt, freqz

def standardize(
    s,
    standardize=True,
    detrend=False,
    remove_mean=False,
    bandpass_filter=False,
    bandpass_kwargs=None,
):
    '''
    Helper function for pre-processing data, specifically for wavelet analysis

    Parameters
    ----------
        s : numpy array of shape (n,) to be normalized
        standardize : divide by the standard deviation
        detrend : Linearly detrend s, default is False. Only one of detrend and
            bandpass_filter may be True.
        remove_mean : remove the mean of s. Only one of detrend, remove_mean, and
            bandpass_filter may be True.
        bandpass_filter : band pass the data. Only one of detrend and bandpass_filter
            may be True.
            NOTE: At the moment I just implement a high-pass filter. This needs to
            be updated in the future to be a band pass instead.
        bandpass_kwargs : Dictionary of kwargs passed to the bandpass filter. Must
            be provided if bandpass_filter = True.
            Arguments:
                cutoff : float, desired cutoff frequency of the filter in Hz
                fs : float, sampling frequency in Hz
                order : int, order of the filter. Default is 5

    Returns
    ----------
        snorm : numpy array of shape (n,)
    '''

    # @ Turn this into a decorator in the future for extra sexy python code
    def butter_highpass(cutoff, fs, order=5):
        nyq = 0.5 * fs
        normal_cutoff = cutoff / nyq
        b, a = butter(order, normal_cutoff, btype='high', analog=False)
        return b, a

    def butter_highpass_filter(data, cutoff, fs, order=5):
        b, a = butter_highpass(cutoff, fs, order=order)
        y = filtfilt(b, a, data)
        return y

    # Logic checking
    if detrend and remove_mean:
        raise ValueError('Only standardize by either removing secular trend or mean, not both.')
    if (detrend and bandpass_filter) or (remove_mean and bandpass_filter):
        raise ValueError(
            'Standardizing can only take one of the following as True:'
            ' remove_mean, detrend, bandpass_filer.'
        )
    if bandpass_filter and bandpass_kwargs is None:
        raise ValueError(
            'When using the bandpass filter the bandpass_kwargs must be supplied.'
        )

    # Derive the variance prior to any treatment
    smean = s.mean()

    # Remove the trend if requested
    if detrend:
        arbitrary_x = np.arange(0, s.size)
        p = np.polyfit(arbitrary_x, s, 1)
        snorm = s - np.polyval(p, arbitrary_x)
    else:
        snorm = s

    if remove_mean:
        snorm = snorm - smean

    if bandpass_filter:
        bandpass_kwargs.setdefault('order', 5)
        cutoff = bandpass_kwargs['cutoff']
        order = bandpass_kwargs['order']
        fs = bandpass_kwargs['fs']
        snorm = butter_highpass_filter(s, cutoff, fs, order=order)

    # Standardize by the variance only after the above
    # data treatment.
    std = snorm.std()
    if standardize:
        snorm = (snorm / std)

    return snorm

def wavelet_power(
    signal,
    dx,
    x,
    mother,
    octaves=None,
    scales_to_avg=None,
    glbl_power_var_scaling=True,
    norm_kwargs={'detrend': True, 'standardize': True},
    variance=None,
    rectify=True,
    significance_test=True,
    sig_kwargs={},
):
    '''
    Helper function for using the pycwt library since the steps in this
    function need to be completed for each realization of a CWT,

    Note: this function is based on the example from the pycwt library
    (https://pycwt.readthedocs.io/en/latest/tutorial.html)

    The resulting spectra are rectified following Liu 2007

    Parameters
    ----------
        signal : ndarray
        dx : float
        x : ndarray
        mother : pycwt wavelet object
        octaves : tuple, optional
        scales_to_avg : list, optional
        glbl_power_var_scaling : boolean, optional
        norm_kwargs : dict
        variance : float, optional
        rectify : boolean, optional
        significance_test : boolean, optional
        sig_kwargs : dict, optional

    Reutnrs
    ----------
    '''

    norm_kwargs.setdefault('standardize', True)
    norm_kwargs.setdefault('detrend', True)

    # If we do not normalize by the std, we need to find the variance
    if not norm_kwargs['standardize'] and variance is None:
        std = np.std(signal)
        var = std ** 2
    # For strongly non-stationary vectors, estimating the (white noise)
    # variance from the data itself is poorly defined. This option allows
    # the user to pass a pre-determined variance.
    elif variance is not None:
        var = variance
    # If the data were standardized, the variance should be 1 by definition
    # (assuming normally distributed processes)
    else:
        var = 1.

    if significance_test and 'sig_lvl' in sig_kwargs:
        sig_lvl = sig_kwargs['sig_lvl']
    else:
        sig_lvl = 0.95

    # This is hacky just to not break all of my scripts
    # by changing the number of outputs.
    if significance_test and 'local_sig_type' in sig_kwargs:
        local_sig_type = sig_kwargs['local_sig_type']
    else:
        local_sig_type = 'index'

    # Standardize/detrend/de-mean the input signal
    signal_norm = standardize(signal, **norm_kwargs)
    N = signal_norm.size

    # Wavelet properties
    # Starting scale is twice the resolution
    s0 = 2 * dx
    # Default wavelet values yields 85 periods
    if octaves is None:
        # Twelve sub-octaves per octaves
        dj = 1 / 12
        # Seven powers of two with dj sub-octaves
        J = 7 / dj
    else:
        # Number of sub-octaves per octaves
        dj = octaves[0]
        # Number of powers of two with dj sub-octaves
        J = octaves[1]

    # Perform the wavelet transform using the parameters defined above.
    wave, scales, freqs, coi, fft, fftfreqs = wavelet.cwt(
        signal_norm, dx, dj, s0, J, mother)

    # Calculate the normalized wavelet and Fourier power spectra,
    # as well as the Fourier equivalent periods for each wavelet scale.
    # Note that this power is not normalized as in TC98 equation 8, the factor
    # of dt^(1/2) is missing.
    power = np.abs(wave) ** 2
    fft_power = np.abs(fft) ** 2
    period = 1 / freqs

    # Do the significance testing for both the local and global spectra
    if significance_test:

        if 'alpha' not in sig_kwargs:
            # Lag-1 autocorrelation for red noise-based significance testing
            alpha, _, _ = wavelet.ar1(signal_norm)
        else:
            alpha = sig_kwargs['alpha']

        # Local power spectra significance test, where the ratio power / signif > 1.
        signif, fft_theor = wavelet.significance(var, dx, scales, 0, alpha,
                                                 significance_level=sig_lvl,
                                                 wavelet=mother)
        sig_ind = np.ones([1, N]) * signif[:, None]
        # The default behavior is to return a matrix that can be directly used
        # to draw a single contour following the pycwt_plot_helper functions.
        # The other option is to just return the local significance levels
        # as a matrix with dimensions equal to 'wave'.
        if local_sig_type == 'index':
            sig_ind = power / sig_ind

    # Rectify the power spectrum according to Liu et al. (2007)[2]
    if rectify:
        power /= scales[:, None]

    # Calculate the global wavelet spectrum
    glbl_power = power.mean(axis=1)
    if glbl_power_var_scaling:
        glbl_power = glbl_power / var

    # Do the significance testing for the global spectra
    if significance_test:

        # Global power spectra significance test. Note: this variable  is
        # different than the local significance variable. Here the global
        # spectra is reported directly.
        dof = N - scales  # Correction for padding at edges
        glbl_signif, tmp = wavelet.significance(var, dx, scales, 1, alpha,
                                                significance_level=sig_lvl,
                                                dof=dof, wavelet=mother)
    # Just return nans in the significance variables as they won't break
    # e.g. plotting routines.
    else:
        glbl_signif = np.ones_like(glbl_power) * np.nan
        sig_ind = np.ones_like(power) * np.nan

    if rectify:
        glbl_signif = glbl_signif / scales

    if scales_to_avg is not None:
        Cdelta = mother.cdelta
        num_intervals = np.shape(scales_to_avg)[0]
        scale_avg = np.ones((len(x), num_intervals))
        scale_avg_signif = np.ones(num_intervals)

        for nint in np.arange(num_intervals):
            # Calculate the scale average between two periods and the significance level.
            sel = find((period >= scales_to_avg[nint][0]) & (period < scales_to_avg[nint][1]))

            # In the pycwt example and TC98 eq. 24 the scale-avg power is scale rectified.
            # However, the rectification occurs above in the calculation of power. Do not repeat here.
            if rectify:
                scale_avg[:, nint] = dj * dx / Cdelta * power[sel, :].sum(axis=0)
            # If no rectification is requested, then for consistency with TC98
            # we return the rectified power here.
            elif not rectify:
                # The example code is just included here as a reference
                # scale_avg = (scales * np.ones((N, 1))).transpose()
                # scale_avg = power / scale_avg  # As in Torrence and Compo (1998) equation 24
                # TC98 eq. 24 does not include the variance as in the tutorial.
                scale_avg[:, nint] = dj * dx / Cdelta * power[sel, :].sum(axis=0) / scales[:, None]

            if significance_test:
                try:
                    scale_avg_signif[nint], _ = wavelet.significance(
                        var, dx, scales, 2, alpha,
                        significance_level=sig_lvl,
                        dof=[scales[sel[0]], scales[sel[-1]]],
                        wavelet=mother
                    )
                except IndexError:
                    # One of the scales to average was outside the range of the
                    # CWT's scales. Return no significance level for this averaging
                    # interval and move to the next one.
                    scale_avg_signif[nint] = np.nan
                    continue
            else:
                scale_avg_signif[nint] = np.nan

    else:
        scale_avg = None
        scale_avg_signif = None

    # @ Rename period and scales to better reflect that period is the inverse
    # fourier frequencies and that scales are the inverse wavelet frequencies.
    return (signal_norm, period, coi, power,
            glbl_signif, glbl_power, sig_ind,
            scale_avg, scale_avg_signif, scales)

def coi_scale_avg(coi, scales):
    '''
    Returns the upper and lower coi bounds for scale averaged wavelet power.

    Parameters
    ----------
        coi - np.array (or similar)
            location  of the coi in period space
        scales - np.array (or similar)
            The cwt scales that label the coi

    Returns
    ----------
        min_index : list
            indices of the coi scales for the shortest scale in
            the scale averaging interval
        max_index : list
            indices of the coi scales for the shortest scale in
            the scale averaging interval
    '''
    mindex1 = np.argmin(np.abs(coi[:len(coi)//2] - np.min(scales)))
    mindex2 = np.argmin(np.abs(coi[len(coi)//2:] - np.min(scales)))
    min_index = [mindex1, mindex2 + len(coi)//2]

    maxdex1 = np.argmin(np.abs(coi[:len(coi)//2] - np.max(scales)))
    maxdex2 = np.argmin(np.abs(coi[len(coi)//2:] - np.max(scales)))
    max_index = [maxdex1, maxdex2 + len(coi)//2]

    return min_index, max_index

def wavelet_coherent(
    s1,
    s2,
    dx,
    dj,
    s0,
    J,
    mother,
    norm_kwargs={'detrend': True, 'standardize': True},
    ):
    '''
    Calculate coherence between two vectors for a given wavelet. Currently
    only the Morlet wavelet is implemented. Returns the wavelet linear coherence
    (also known as coherency and wavelet coherency transform), the cross-wavelet
    transform, and the phase angle between signals.

    Parameters
    ----------
        s1 : 1D numpy array or similar object of length N.
            Signal one to perform the wavelet linear coherence with s2. Assumed
            to be pre-formatted by the `standardaize` function. s1 and s2 must
            be the same size.
        s2 : 1D numpy array or similar object of length N.
            Signal two to perform the wavelet linear coherence with s1. Assumed
            to be pre-formatted by the `standardaize` function. s1 and s2 must
            be the same size.
        dx : scalar or float. Spacing between elements in s1 and s2. s1 and s2
            are assumed to have equal spacing.
        dj : float, number of suboctaves per octave expressed as a fraction.
            Default value is 1 / 12 (12 suboctaves per octave)
        s0 : Smallest scale of the wavelet (if unsure = 2 * dt)
        J : float, number of octaves expressed as a power of 2 (e.g., the
            default value of 7 / dj means 7 powers of 2 octaves.
        mother : pycwt wavelet object, 'Morlet' is the only valid selection as
            a result of requiring an analytic expression for smoothing the
            wavelet.

    Returns
    ----------
        WCT : same type as s1 and s2, size PxN
            The biwavelet linear coherence transform.
        aWCT : same type as s1 and s2, size of PxN
            phase angle between s1 and s2.
        W12 : same type as s1 and s2, size PxN
            The cross-wavelet transform power, unrectified.
        W12_corr : the same type as s1 and s2, size PxN
            Cross-wavelet power, rectified following Veleda et al., 2012 and
            the R biwavelet package.
        period : numpy array, length P
            Fourier mode inverse frequencies.
        coi : numpy array  of length n, cone of influence
        angle : phase angle in degrees
        w1 : same type as s1, size PxN
            CWT for s1
        w2 : same type as s1, size PxN
            CWT for s2
    '''
    assert mother.name == 'Morlet', "XWT requires smoothing, which is only available to the Morlet mother."
    wavelet_obj = wavelet.wavelet._check_parameter_wavelet(mother)

    assert np.size(s1) == np.size(s2), 's1 and s2 must be the same size.'

    # s1 and s2 MUST be the same size
    assert s1.shape == s2.shape, "Input signals must share the exact same shape."

    s1_norm = standardize(s1, **norm_kwargs)
    s2_norm = standardize(s2, **norm_kwargs)

    # Calculates the CWT of the time-series making sure the same parameters
    # are used in both calculations.
    W1, sj, freq, coi, _, _ = wavelet.cwt(s1_norm, dx, dj, s0, J, mother)
    W2, _, _, _, _, _ = wavelet.cwt(s2_norm, dx, dj, s0, J, mother)

    # We need a 2D matrix for the math that follows
    scales = np.atleast_2d(sj)
    periods = np.atleast_2d(1 / freq)

    # Perform the cross-wavelet transform
    W12 = W1 * W2.conj()
    # Here I follow the R biwavelet package for the implementation of the
    # scale rectification. Note that the R package normalizes by the largest
    # wavelet scale. I choose to not include that scaling factor here.
    # W12_corr = W1 * W2.conj() * np.max(periods) / periods.T
    W12_corr = W1 * W2.conj() / periods.T

    # Coherence

    # Smooth the wavelet spectra before truncating.
    if mother.name == 'Morlet':
        sW1 = wavelet_obj.smooth((np.abs(W1) ** 2 / scales.T), dx, dj, sj)
        sW2 = wavelet_obj.smooth((np.abs(W2) ** 2 / scales.T), dx, dj, sj)
        sW12 = wavelet_obj.smooth((W12 / scales.T), dx, dj, sj)
    WCT = np.abs(sW12) ** 2 / (sW1 * sW2)
    aWCT = np.angle(W12)
    # @ fix this incorrect angle conversion.
    angle = (0.5 * np.pi - aWCT) * 180 / np.pi

    # @ better names to reflect fourier vs wavelet frequency/scale
    scales = np.squeeze(scales)

    return (WCT, aWCT, W12, W12_corr, 1 / freq, coi, angle, sW1, sW2)

def wct_mc_sig(
    wavelet,
    J,
    dj,
    dt,
    s0,
    sfunc_args1=[],
    sfunc_args2=[],
    sfunc_kwargs1={},
    sfunc_kwargs2={},
    mc_count=60,
    slen=None,
    sig_lvl=0.95,
    sfunc=None,
):
    '''
    Parameters
    ----------
        wavelet : pycwt wavelet object class
        J : int
            Wavelet's maximum scale.
        dj : float
            Number of suboctaves / number of octaves for the wavelet.
        dt : float
            Spacing of the time series in time. It is recommended to use the
            spacing of the data being tested for consistenct.
        s0 : float
            Minimum resolvable scale for the wavelet. Recommended value is
            2 * dt
        sfunc_args1 : list, optional
            positional arguments for sfunc for time series one.
        sfunc_args2 : list
            positional arguments for sfunc for time series two.
        sfunc_kwargs1 : dictionary, optional
        sfunc_kwargs2 : dictionary, optional
        mc_count : int, optional
        slen : int, optional
        sig_lvl : float, optional
        sfunc : function handle
            sfunc is used to generate the synthetic data
            for the Monte-Carlo simulation. The default function is
            pycwt.rednoise(). Function must accept `N` as the first argument
            and return an array of length `N`.

    Returns
    -------
        coh : ndarray of floats
            Coherence from each Monte-Carlo draw

    '''

    if slen is None:
        # Choose N so that largest scale has at least
        # some part outside the COI
        slen = s0 * (2 ** (J * dj)) / dt

    # Assign the length of the synthetic signal
    N = slen

    # Assign the function for generating synthetic data
    if sfunc is None:
        # @ Replace with a functional red noise generator
        sfunc = pcywt.rednoise

    # Peak the details of the cwt output for a single realization
    # of the noise function.
    noise1 = sfunc(N, *sfunc_args1, **sfunc_kwargs1)

    # Check that sfunc returns an array with the necessary properties
    if not len(noise1) == N:
        raise ValueError('sfunc must return data of length N')

    nW1, sj, freq, coi, _, _ = pycwt.cwt(
        noise1, dt=dt, dj=dj, s0=s0, J=J, wavelet=wavelet
    )

    period = np.ones([1, N]) / freq[:, None]
    coi = np.ones([J + 1, 1]) * coi[None, :]
    outsidecoi = (period <= coi)
    scales = np.ones([1, N]) * sj[:, None]
    sig_ind = np.zeros(J + 1)
    maxscale = find(outsidecoi.any(axis=1))[-1]
    sig_ind[outsidecoi.any(axis=1)] = np.nan

    coh = np.ma.zeros([J + 1, N, mc_count])

    # Displays progress bar with tqdm
    for n_mc in tqdm(range(mc_count)):#, disable=not progress):
        # Generates a synthetic signal using the provided function and parameters

        noise1 = sfunc(N, *sfunc_args1, **sfunc_kwargs1)
        noise2 = sfunc(N, *sfunc_args2, **sfunc_kwargs2)

        # Calculate the cross wavelet transform of both red-noise signals
        kwargs = dict(dt=dt, dj=dj, s0=s0, J=J, wavelet=wavelet)
        nW1, sj, freq, coi, _, _ = pycwt.cwt(noise1, **kwargs)
        nW2, sj, freq, coi, _, _ = pycwt.cwt(noise2, **kwargs)
        nW12 = nW1 * nW2.conj()

        # Smooth wavelet wavelet transforms and calculate wavelet coherence
        # between both signals.

        S1 = wavelet.smooth(np.abs(nW1) ** 2 / scales, dt, dj, sj)
        S2 = wavelet.smooth(np.abs(nW2) ** 2 / scales, dt, dj, sj)
        S12 = wavelet.smooth(nW12 / scales, dt, dj, sj)

        R2 = np.ma.array(np.abs(S12) ** 2 / (S1 * S2), mask=~outsidecoi)
        coh[:, :, n_mc] = R2

    period = period[:, 0]

    return coh, period, scales, coi

def coi_where(period, coi, data):
    '''
    Finds where the period by num samples array is outside the coi.
    Useful for creating masked numpy arrays.

    INPUTS:
    period : N array, the wavelet periods
    coi : M array, the coi location for each sample. M should
        be the number of samples.
    data : N x M array, the data to broadcast the COI mask to.

    RETURNS:
    outside_coi : N by M array of booleans. Is True where
        the wavelet is effected by the coi.

    EXAMPLE:
        outside_coi = pycwt_stat_helpers.coi_where(period, coi, WCT)
        masked_WCT = np.ma.array(WCT, mask=~outside_coi)
    '''

    coi_matrix = np.ones(np.shape(data)) * coi
    period_matrix = (np.ones(np.shape(data)).T * period).T

    outside_coi = period_matrix < coi_matrix

    return outside_coi

def ar1_generator(N, alpha, noise, init_n_len=3):
    '''
    Generates a Markov chain lag-1 process through a brute force approach.

    Parameters
    -------

    Returns
    -------

    '''

    y = np.zeros((init_n_len * N,))
    # Simulate a longer period and just return the last N point
    # to remove edge effects.
    for t in range(1, init_n_len * N):
        y[t] = alpha * y[t - 1] + np.random.normal(scale=noise)

    return y[-N:]

def nan_sequences(signal, dx, dim='time', units=None):
    '''
    Describes properties of NaN blocks, giving their indices, slices, and
    length.

    INPUTS:
    signal : xarray DataArray, data to search for contiguous NaN blocks
        within that has a labeled dimension specified by the 'dim'
        keyword.
    dx : Float, Step size of the dimension specified by the 'dim' keyword.
    dim : String, Dimension to index contiguous NaN blocks along in signal.
        Default is "time". If dim is "time" the units keyword must be
        specified.
    units : String, describes the units of dx. Necessary when operating
        along a "time" dimension.

    RETURNS:
    nan_seq_ind : 2 x N list, where N is the number of contiguous NaN blocks.
        First element of each list specifies the indices of the NaN block
        beginning while the second element specifies the end of the NaN.
    nan_seq_slc : list of length N, each element is a slice across the NaN
        block.
    nan_seq_len : list of length N, each element specifies the length of its
        respective NaN block.

    EXAMPLE:
        nan_seq_ind, nan_seq_slc, nan_seq_len = cwt_stat.nan_sequences(
            signal, dt, dim='time', units='seconds'
        )
    '''

    # Assure that the arithmetic of operating along the time dimension will
    # function by coercing dx and the 'time' dimension to have compatible
    # data types.
    if dim == 'time':
        if units is None:
            raise ValueError(
                'When operating along the time dimension, the units keyword'\
                ' must be provided.'
            )
        units_kwarg = {units: dx}
        dx_td = pd.Timedelta(**units_kwarg)

    # First, find the indices for the nans
    # The drop=True makes the first loop easy, but the
    # indices are now no longer referenced to the original
    # signal dataset
    nanind = signal.where(np.isnan(signal), drop=True)

    # If no NaNs are present return empty lists
    if nanind.size == 0:
        nan_seq_len = []
        nan_seq_slc = []
        nan_seq_ind = []
        return nan_seq_ind, nan_seq_slc, nan_seq_len

    ind_beg = [0]
    ind_end = []
    for nt2, t2 in enumerate(nanind[dim][1:]):
        if dim == 'time':
            t1 = pd.Timestamp(nanind[dim].values[nt2])
            t2 = pd.Timestamp(t2.values)
        else:
            t1 = nanind[dim].values[nt2]
            t2 = t2.values
        # Maybe this extra step of checking for time is unnecessary
        # if we overwrite dx with the Timedelta value if dim=='time'.
        # I don't have time right now to check.
        if (dim == 'time') and (t2 - t1 > dx_td):
            ind_end.append(nt2)
            ind_beg.append(nt2 + 1)
        elif (not dim == 'time') and (t2 - t1 > dx):
            ind_end.append(nt2)
            ind_beg.append(nt2 + 1)

    ind_end.append(-1)

    # 3 steps:
    # 1) Convert back to the original signal's indices
    # 2) Find the duration of a nan sequence
    # 3) Give the slice (in time coordinates) for the nan block
    nan_seq_len = []
    nan_seq_slc = []
    nan_seq_ind = []

    for b, e in zip(ind_beg, ind_end):
        # Duration of the nan sequence in units of time
        if dim=='time':
            t1 = pd.Timestamp(nanind.time.values[b])
            t2 = pd.Timestamp(nanind.time.values[e])
        else:
            # Duration of the nan sequence in units of dx
            t1 = nanind[dim].values[b]
            t2 = nanind[dim].values[e]

        # Slices of nan blocks
        nan_seq_slc.append(slice(t1, t2))

        # The indices of nan blocks
        # (index beg of block, index end of block)
        nan_seq_ind.append(
            [
                np.flatnonzero(signal[dim] == t1)[0],
                np.flatnonzero(signal[dim] == t2)[0],
            ]
        )

        tdiff = np.flatnonzero(signal[dim] == t2)[0] - np.flatnonzero(signal[dim] == t1)[0]
        nan_seq_len.append(tdiff)

    return nan_seq_ind, nan_seq_slc, nan_seq_len

def nan_coi(nan_seq_ind, nan_seq_len, coi, period, signal, dx):
    '''
    Rebuilds the COI to have a new, scale-aware COI around nan gaps

    Parameters
    -------

    Returns
    -------

    '''
    if (not nan_seq_ind) or (not nan_seq_len):
        return coi

    # We need two copies of the data.
    # One is just the original coi. We will manipulate it
    # at the very end when we ammend it with the nan-coi gaps
    coi_mask = copy.deepcopy(coi)
    # And now we need just half of the coi
    coi_half = copy.deepcopy(coi[0:len(coi)//2])
    # Remove anything with a length longer than the resolvable
    coi_half[coi_half > np.max(period)] = np.nan

    coi_half_nanless = copy.deepcopy(coi_half)
    coi_half_nanless = coi_half_nanless[np.flatnonzero(~np.isnan(coi_half_nanless))]
    coi_half_nanless_len = len(coi_half_nanless)

    # @ Is the reference to "signal" necessary here? Or could we
    # just use the COI variable instead?
    nan_mask = np.zeros_like(signal) * np.nan

    for ns, (n1, n2) in enumerate(nan_seq_ind):
        nan_mask[n1:n2 + 1] = 0

        coi_fill_len = np.min(
            (
                nan_seq_len[ns] + 1,
                coi_half_nanless_len,
                n1
            )
        )
        coi_nanseq_beg = n1 - coi_fill_len
        fill_values = coi_half_nanless[coi_fill_len - 1::-1]
        fill_values[fill_values > nan_seq_len[ns] * dx] = nan_seq_len[ns] * dx
        nan_mask[coi_nanseq_beg:n1] = fill_values

        coi_fill_len = np.min(
            (
                nan_seq_len[ns] + 1,
                coi_half_nanless_len,
                len(signal) - (n2 + 1)
            )
        )
        coi_nanseq_end = n2 + coi_fill_len
        fill_values = coi_half_nanless[0:coi_fill_len]
        fill_values[fill_values > nan_seq_len[ns] * dx] = nan_seq_len[ns] * dx
        nan_mask[n2:coi_nanseq_end] = fill_values

    # Don't overwrite the original COI from edge effects.
    nan_mask[nan_mask > coi] = coi[nan_mask > coi]

    # Fill in all the places without nans with the original COI
    coi_mask[~np.isnan(nan_mask)] = nan_mask[~np.isnan(nan_mask)]

    return(coi_mask)


from typing import Any

import pytest

from app.handlers import get_handler_from_string, get_handler_from_url
from app.handlers.base import ServiceHandler
from app.handlers.exceptions import HandlerNotFoundError
from app.services.ytdlp import AwaitingTranscodingError, FormatNotFoundError

def test_get_handler_from_url() -> None:
    """
    Tests the `get_handler_from_url` function.
    """
    url = "https://www.youtube.com/watch?v=1234567890"
    handler = get_handler_from_url(url=url)
    assert handler.__class__.__name__ == "YoutubeHandler"

    url = "https://www.rumble.com/v1234567890"
    handler = get_handler_from_url(url=url)
    assert handler.__class__.__name__ == "RumbleHandler"

    url = "https://www.bitchute.com/video/1234567890"
    with pytest.raises(HandlerNotFoundError):
        get_handler_from_url(url=url)

def test_get_handler_from_string() -> None:
    """
    Tests the `get_handler_from_string` function.
    """
    handler_string = "YoutubeHandler"
    handler = get_handler_from_string(handler_string=handler_string)
    assert handler.__class__.__name__ == "YoutubeHandler"

    handler_string = "RumbleHandler"
    handler = get_handler_from_string(handler_string=handler_string)
    assert handler.__class__.__name__ == "RumbleHandler"

    handler_string = "BitchuteHandler"
    with pytest.raises(HandlerNotFoundError):
        get_handler_from_string(handler_string=handler_string)

async def test_map_video_info_dict_entity_to_video_dict_format_id_keyerror(
    mocked_entry_info_dict: dict[str, Any]
) -> None:
    # Test missing format_id key
    with pytest.raises(FormatNotFoundError):
        entry_info_dict = mocked_entry_info_dict.copy()
        entry_info_dict.pop("format_id")
        ServiceHandler()._get_format_info_dict_from_entry_info_dict(entry_info_dict=entry_info_dict)  # type: ignore

    # Test missing formats key
    with pytest.raises(FormatNotFoundError):
        entry_info_dict = mocked_entry_info_dict.copy()
        entry_info_dict.pop("formats")
        ServiceHandler()._get_format_info_dict_from_entry_info_dict(entry_info_dict=entry_info_dict)  # type: ignore

    # Test missing formats key, but awaiting_transcoding is True
    with pytest.raises(AwaitingTranscodingError):
        entry_info_dict = mocked_entry_info_dict.copy()
        entry_info_dict.pop("formats")
        entry_info_dict["awaiting_transcoding"] = True
        ServiceHandler()._get_format_info_dict_from_entry_info_dict(entry_info_dict=entry_info_dict)  # type: ignore

    # Test m3u8 in url
    with pytest.raises(AwaitingTranscodingError):
        entry_info_dict = mocked_entry_info_dict.copy()
        entry_info_dict["formats"][0]["url"] = "https://somedomain.com/somepath.m3u8"
        ServiceHandler()._get_format_info_dict_from_entry_info_dict(entry_info_dict=entry_info_dict)  # type: ignore


import cv2
import numpy as np

cap = cv2.VideoCapture('คงจะดีนะ.mp4')
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
vdo_writer = cv2.VideoWriter('out.mp4', fourcc, 30, (426, 240))
n_buffer = 10
buffer = None
iframe = 0
while True:
    ret, frame = cap.read()
    if not ret:
        break
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    if buffer is None:
        buffer = np.zeros((n_buffer, *gray.shape))
    buffer[iframe % n_buffer] = gray
    if iframe >= n_buffer:
        # M = np.mean(buffer, axis=0)
        M = np.median(buffer, axis=0)
        diff = cv2.absdiff(gray, np.uint8(M))
        fg = np.uint8((diff > .4 * np.max(diff)) * 255)
        cv2.imshow('fg', fg)
    cv2.imshow('frame', frame)
    if cv2.waitKey(1) == ord('q') or not ret:
        break
    iframe += 1
    vdo_writer.write(frame)

vdo_writer.release()


# importing the requests library
import requests

def testget(geturl,val):
    # defining a params dict for the parameters to be sent to the API
    PARAMS = {'ID':val}

    # sending get request and saving the response as response object
    r = requests.get(url = geturl, params = PARAMS)

    # extracting data in json format
    data = r.json()
    print('GET returned: {}'.format(data))
    #check the HTTP status code
    print('GET status: {}'.format(r.status_code))

def testpost(posturl,val1,val2):
    # defining a params dict for the parameters to be sent to the API
    PARAMS = {'name':val1, 'surName':val2}

    # sending get request and saving the response as response object
    r = requests.post(url = posturl, params = PARAMS)
    #nothing is returned, just check the HTTP status code
    print('POST status: {}'.format(r.status_code))

def main():
    url = 'http://localhost:8080/test/GetName'
    testget(url,'foo')
    url = 'http://localhost:8080/test/SaveName'
    testpost(url,'name1','name2')

if __name__ == '__main__':
  main()


import builtins
from collections import namedtuple
import inspect
from .. import context

AllVars = namedtuple('AllVars', 'nonlocals globals builtins')

def getallvars(obj):
    """
    Get the mapping of *all* variables to their current values.

    Returns a named tuple of dicts mapping the current nonlocal, global
    and builtin references as seen by the body of the function. A final
    set of unbound names that could not be resolved is also provided.
    """

    # FIXME:
    # `getclosurevars` doesn't capture type annotation, it is the reason that we use frame
    # to get all vars in `f_locals` but not those from func.__closure__ and code.__freevars__.
    # Also, `code.co_names` has the same issue that forbidding us to use `getclosurevars`.
    #
    # Commit fa090dbbabb99859afe701f736731104bce77d8e tried to use `getclosurevars` but failed
    # when trying to get Class definations in locals.
    #
    # This version is rather a redundent implementation to save all vars to module info. Anyone
    # can optimize it?

    frame = inspect.currentframe()
    while frame:
        if frame.f_locals.get(obj.__name__, None) is obj:
            break
        frame = frame.f_back

    if frame is None:
        nonlocal_vars = {}
    else:
        nonlocal_vars = frame.f_locals

    if inspect.isfunction(obj):
        global_ns = obj.__globals__
    elif frame is not None:
        global_ns = frame.f_globals
    else:
        global_ns = {}

    builtin_ns = global_ns.get("__builtins__", builtins.__dict__)
    if inspect.ismodule(builtin_ns):
        builtin_ns = builtin_ns.__dict__

    return AllVars(nonlocal_vars, global_ns, builtin_ns)

class ModuleAnalysis:

    def __init__(self) -> None:
        self.cls_ctx = None

    def run_impl(self, node: context.ASTNode):
        if node.module is None:
            # parser module info
            node.module = context.ModuleInfo()
            node.module.raw = inspect.getmodule(node.raw)

            global_syms = [o for o in inspect.getmembers(node.module.raw)]
            node.module.globals.update(global_syms)

            node.module.name = node.module.raw.__name__

            all_vars = getallvars(node.raw)
            node.module.globals.update(all_vars.builtins)
            node.module.globals.update(all_vars.nonlocals)
            node.module.globals.update(all_vars.globals)

            try:
                if inspect.isfunction(node.raw):
                    vars = inspect.getclosurevars(node.raw)
                    node.module.globals.update(vars.globals)
                    node.module.globals.update(vars.nonlocals)
                    node.module.globals.update(vars.builtins)
                elif inspect.isclass(node.raw):
                    try:
                        vars = inspect.getclosurevars(node.raw.__init__)
                        node.module.globals.update(vars.globals)
                        node.module.globals.update(vars.nonlocals)
                        node.module.globals.update(vars.builtins)
                    except:
                        pass

                    for func in inspect.getmembers(node.raw, inspect.isfunction):
                        if func.__name__ == '__init__':
                            continue
                        vars = inspect.getclosurevars(func)
                        node.module.globals.update(vars.globals)
                        node.module.globals.update(vars.nonlocals)
                        node.module.globals.update(vars.builtins)

            except:
                pass

            for name, obj in node.module.globals.items():
                try:
                    node.module.imports[name] = inspect.getabsfile(obj)
                except TypeError:
                    pass

    def run(self, sc_ctx: context.ScriptContext):
        self.run_impl(sc_ctx.main_node)
        for dep_node in sc_ctx.deps_node:
            self.run_impl(dep_node)


import shortest_path_switch
import simple_switch
from ryu.controller.handler import MAIN_DISPATCHER, set_ev_cls
from ryu.controller import ofp_event
import os
import socket
import json
from ryu.lib import hub

SOCKFILE = '/tmp/hello_sock'

class ShortestRestSwitch(simple_switch.SimpleSwitch13):

	def __init__(self, *args, **kwargs):
		super(ShortestRestSwitch, self).__init__(*args, **kwargs)
		self.sock = None
		self.config = {}
		self.start_sock_server()

	def set_vtable(self, host, vlan):
		if self.vtable[host] != vlan:
			self.vtable.update({host:vlan})
			self.SimpleSwitchDeleteFlow(self.default_datapath, host)

	def recv_loop(self):
		print('start loop')
		while True:
			print('wait for recev')
			data = self.sock.recv(1024)
			print('Receive new vtable from web')
			print(data)
			msg = json.loads(data)
			if msg:
				print('get msg')

			for host, vlan in msg.items():
				self.set_vtable(host, vlan)

	def start_sock_server(self):
		if os.path.exists(SOCKFILE):
			os.unlink(SOCKFILE)

		self.sock = hub.socket.socket(hub.socket.AF_UNIX, hub.socket.SOCK_DGRAM)
		self.sock.bind(SOCKFILE)
		hub.spawn(self.recv_loop)


from django.urls import path
from . import views

app_name = 'app'

urlpatterns = [
    path('', views.index, name='index'),
    path('contacts/', views.contacts, name='contacts'),
    path('register/', views.register, name='register'),
    path('logout/', views.logout_request, name='logout'),
    path('login/', views.login_request, name='login'),
]


import argparse
import sys
import warnings

from train_acc import *
from train_auc import *
from train_binary import *
from train_noContent import *
from train_noContentGRL2Spoof import *
from train_auc_no_depth import *
from train_auc_triplet_ours import *
from train_auc_triplet_ssdg import *
from train_auc_2domain import *
from train_celeba import *
from train_shared_auc import *
from train_shared_nodomain import *
from train_shared_nocontent import *
from test_auc import *

def main(args):
    if args.type == 'train_auc':
        train_auc(args)

    elif args.type == 'train_shared_auc':
        train_shared_auc(args)

    elif args.type == 'train_shared_auc_no_domain':
        train_shared_auc_no_domain(args)

    if args.type == 'train_shared_auc_no_content':
        train_shared_auc_no_content(args)

    elif args.type == 'train_acc':
        train_acc(args)

    elif args.type == 'train_binary':
        train_binary(args)

    elif args.type == 'train_noContent':
        train_noContent(args)

    elif args.type == 'train_noContentGRL2Spoof':
        train_noContentGRL2Spoof(args)

    elif args.type == 'train_noDepth':
        train_noDepth(args)

    elif args.type == 'train_aucTripletOurs':
        train_auc_triplet_ours(args)

    elif args.type == 'train_aucTripletSSDG':
        train_auc_triplet_ssdg(args)

    elif args.type == "train_2domain":
        train_auc_2domain(args)

    elif args.type == 'train_celeba':
        train_celeba(args)

    elif args.type == 'test_auc':
        test_auc(args)

    else:
        warnings.warn("Please check your training type")
        sys.exit()

if __name__ == '__main__':
    # path: model will be saved to path/model/target_domain/number_folder/model.pt
    # dataset_path: folder of training and testing file (includes pr_depth_map)
    # target_domain
    # number_folder
    # img_size: resize img
    # depth_size: resize depth img
    # batch_size
    # batch_triplet
    # lr
    # n_epoch

    parser = argparse.ArgumentParser(description="FRANK")

    parser.add_argument('--seed', type=int, default=0)
    parser.add_argument('--path', type=str, default="./")
    parser.add_argument('--dataset_path', type=str, default="./pr_depth_map_256")
    parser.add_argument('--type', type=str, default='train_auc')
    parser.add_argument('--gpu_id', type=str, default='0')
    parser.add_argument('--spoof_encoder', type=str, default="/home/tsaolin/Face_Anti-Spoofing/FaceAntiSpoofing-WACV/model/oulu/224_0/oulu_spoof_encoder.pt")
    parser.add_argument('--spoof_classifier', type=str, default="/home/tsaolin/Face_Anti-Spoofing/FaceAntiSpoofing-WACV/model/oulu/224_0/oulu_spoof_classify.pt")

    # datasets
    parser.add_argument('--target_domain', type=str, default='oulu')
    parser.add_argument('--number_folder', type=str, default='0')

    # model
    parser.add_argument('--scratch', type=str, default='True')
    parser.add_argument('--load_folder', type=str, default='0')

    # optimizer
    parser.add_argument('--lr', type=float, default=0.0001)

    # training configs
    parser.add_argument('--img_size', type=int, default=256)
    parser.add_argument('--depth_size', type=int, default=64)
    parser.add_argument('--batch_size', type=int, default=10)
    parser.add_argument('--test_batch_size', type=int, default=64)
    parser.add_argument('--batch_triplet', type=int, default=4)
    parser.add_argument('--n_epoch', type=int, default=100)

    print(parser.parse_args())
    main(parser.parse_args())


#!/opt/python/bin/python
#
# Wrapper around mediainfo that enables comparing two files side-by-side
#
# Known issues:
#    - It only compares the first video and audio stream it finds, 2nd+ are ignored,
#    - It outputs to three columns to provide a side-by-side view, but column widths aren't pretty
#    - Column width is set up the 'General:Complete_name' field, and field longer will get truncated
#      This affects the 'Video:Encoding_settings' and a few other fields. Is by design.
#    - It tries to keep the order, but fields in the 2nd file not present in the 1st get shuffled
#      to the bottom rather than injected into the middle. I'm not sure how to clean that up.
# Future ideas:
#    - only print out interesting fields, drop things like 'Encoding_settings'
#    - add a -t option that creates a tab-separated output instead of columns, won't need to truncate
#
from collections import OrderedDict
from collections import defaultdict
import subprocess
import lxml
import argparse
import xmltodict
import json
# Parse the command line arguments
#
parser = argparse.ArgumentParser(description='Wrapper for mediainfo to compare two files side-by-side')
parser.set_defaults(v=False)
parser.add_argument('-v', '--verbose', required=False,
                    dest='verbose', default=False, action='store_true',
                    help='include verbose debugging output')
parser.add_argument('file1',
                    help='the first media file')
parser.add_argument('file2',
                    help='the second media file')
args = parser.parse_args()
# logging output to validate arguments are read correctly
if args.verbose: print('file 1 = ' + args.file1)
if args.verbose: print('file 2 = ' + args.file2)
if args.verbose: print('')

# execute mediainfo against specified file
#
def exec_mediainfo(file):
    mediainfo_out = subprocess.Popen(['mediainfo','--output=XML',file],
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT)
    returncode = mediainfo_out.wait()
    #logging output to validate mediainfo execution produced what we expected
    if args.verbose: print('mediainfo returned {0}'.format(returncode))
    if args.verbose: print('mediainfo for ' + file)# + ':\n' + mediainfo_out.stdout.read())
    # put the output in 'tracks', trim off the top layers of the tree
    output = mediainfo_out.stdout.read()
    x2dict = xmltodict.parse(output)
    tracks = x2dict['Mediainfo']['File']['track']
    # tracks is a list of dict, one list item per track with dict items per data point
    # next block converts the list to a dict using track id as the key
    # this allows us to compare files with different numbers of tracks
    if args.verbose: print('Converting XML to dict...')
    data = OrderedDict()
    for track in tracks:
        if track['@type'] == 'General':
            key = 'g'
        elif track['@type'] == 'Video':
            key = 'v' + str(track['ID'])
        elif track['@type'] == 'Audio':
            key = 'a' + str(track['ID'])
        elif track['@type'] == 'Text':
            key = 's' + str(track['ID'])
        else:
            if args.verbose: print('Im lost')
        if args.verbose: print('Processing track: ' + key)
        if args.verbose: print(str(track.items()))
        data[key] = track
    # now return the dict of dict with track data
    #if args.verbose: print('Returning this: ' + str(data))
    return data
#
# end of exec_mediainfo()

# merge two dicts where {k1:v1} + {k1:v2} = {k1:[v1,v2]}
#
def merge_dict(dict1,dict2):
    if args.verbose: print('dict1: ' + str(dict1))
    if args.verbose: print('dict2: ' + str(dict2))
    # prepare to catch
    dict3 = OrderedDict()
    # iterate through first dict, create all the keys and blanks for the other dict
    for k, v in dict1.iteritems():
        dict3[k] = [v,'']
    # iterate through second dict, update or add depending on if key already exists
    for k, v in dict2.iteritems():
        if k in dict3:
            dict3[k][1] = v
        else:
            dict3[k] = ['',v]
    return dict3
#
# end of merge_dict()

# get the data from both files into dictionaries
#
if args.verbose: print('Executing mediainfo...')
mediainfo1 = exec_mediainfo(args.file1)
mediainfo2 = exec_mediainfo(args.file2)

# merge the two dictionaries together
#
if args.verbose: print('Merging data...')
# setup a new dict to catch, with tracks in correct order
# if left on its own they got out of order, so hard coding them to force the order
bothinfo = OrderedDict()
bothinfo['g'] = ''
bothinfo['v1'] = ''
bothinfo['a2'] = ''
# step through each track, combining the two inputs
for tracks in bothinfo:
    track = merge_dict(mediainfo1[tracks],mediainfo2[tracks])
    bothinfo[tracks] = track
if args.verbose: print('Merged: ')
if args.verbose: print(json.dumps(bothinfo,indent=2))

# print out the results
#
if args.verbose: print('Output time...')
# set column widths using file name as the width to match
# painfully long lines (e.g. 'Unique_ID', 'Encoding_settings') get truncated
c1 = c2 = c3 = 20
for track in bothinfo:
    for key in bothinfo[track].keys():
        if len(key) > c1: c1 = len(key) + 2
c2 = len(bothinfo['g']['Complete_name'][0])
c3 = len(bothinfo['g']['Complete_name'][1])
for track in bothinfo:
    print('')
    if args.verbose: print('Column widths: ' + str(c1) + ', ' + str(c2) + ', ' + str(c3))
    for key, values in bothinfo[track].iteritems():
        print("{k:<{c1}}\t".format(k=key, c1=c1) +
              "{v1:<{c2}.{t2}}\t".format(v1=str(values[0]), c2=c2, t2=c2) +
              "{v2:<{c3}.{t3}}".format(v2=str(values[1]), c3=c3, t3=c3))


import torch
import sys
sys.path.append("..")
import utils

"""
梯度下降的问题
对于多维梯度下降, 可能在某个方向下降较快, 另一个方向下降较慢.
"""
def f_2d(x1, x2):
    return 0.1 * x1 ** 2 + 2 * x2 **2

"""
动量法
VdW = βVdW + (1-β)dW # 指数加权平均
Vdb = βVdb + (1-β)db
W := W - αVdW, b := b - αVdb
"""
def momentum_2d(x1, x2, v1, v2):
    v1 = gamma * v1 + eta * 0.2 * x1
    v2 = gamma * v2 + eta * 4 * x2
    return x1 - v1, x2 - v2, v1, v2

eta, gamma = 0.4, 0.5
utils.show_trace_2d(f_2d, utils.train_2d(momentum_2d))

"""
从零开始实现
"""
features, labels = utils.get_data_ch7()

def init_momentum_states():
    v_w = torch.zeros((features.shape[1], 1), dtype=torch.float)
    v_b = torch.zeros(1, dtype=torch.float)
    return (v_w, v_b)

def sgd_momentum(params, states, hyperparams):
    for p, v in zip(params, states):
        v.data = hyperparams['momentum'] * v.data + hyperparams['lr'] * p.grad.data
        p.data -= v.data

utils.train_ch7(sgd_momentum, init_momentum_states(), {'lr':0.02, 'momentum':0.5},
    features, labels)
utils.train_ch7(sgd_momentum, init_momentum_states(), {'lr':0.02, 'momentum':0.9},
    features, labels) # 不够平滑
utils.train_ch7(sgd_momentum, init_momentum_states(), {'lr':0.004, 'momentum':0.9},
    features, labels)

"""
简洁实现
"""
# 指定SGD的momentum即可实现动量法
utils.train_pytorch_ch7(torch.optim.SGD, {'lr':0.004, 'momentum':0.9}, features, labels)


__author__ = 'Robbert Harms'
__date__ = '2023-02-18'
__maintainer__ = 'Robbert Harms'
__email__ = 'robbert@xkls.nl'
__licence__ = 'LGPL v3'

import os
import re
import warnings
from abc import ABCMeta, abstractmethod
from io import BytesIO, IOBase
from itertools import chain
from pathlib import Path
from pprint import pprint
from typing import BinaryIO, Union, Type, Dict, Any
from xml.etree import ElementTree

import elementpath
from elementpath import XPathToken, XPath1Parser, XPath2Parser
from elementpath.tdop import Parser

import lxml
from elementpath.xpath3 import XPath3Parser
from elementpath.xpath31 import XPath31Parser
from lxml import etree, objectify
from lxml.etree import _ElementTree, _Element
from ruamel import yaml

from pyschematron.parsers.ast import Schema, Check, Assert, SchematronNode, Pattern, Rule, Report, Variable, Paragraph, \
    ConcreteRule, AbstractRule, ExtendsById, Extends, ExtendsExternal, ExternalRule
from pyschematron.parsers.xml_to_ast.builders import ConcreteRuleBuilder, ExternalRuleBuilder, AbstractRuleBuilder
from pyschematron.parsers.xml_to_ast.utils import node_to_str, resolve_href
from pyschematron.utils import load_xml

class ParsingContext:

    def __init__(self,
                 xpath_parser: Parser,
                 parser_factory: ParserFactory = None,
                 base_path: Path = None):
        """Create the parser context we use while parsing the Schematron XML.

        Args:
            xpath_parser: the xpath parser we can use during parsing of the Schematron XML
            parser_factory: the factory we use to create sub parsers, defaults to :class:`DefaultParserFactory`.
            base_path: the base path to use for file inclusions, if not provided it is set to the current working
                directory.
        """
        self.xpath_parser = xpath_parser
        self.parser_factory = parser_factory or DefaultParserFactory()
        self.base_path = base_path or os.getcwd()

    @classmethod
    def from_schematron_root(cls, xml: _Element, parser_factory: ParserFactory = None, base_path: Path = None):
        """Prepares the parsing context from the provided Schematron XML root.

        Args:
            xml: the Schematron root node, we use this to look up the namespaces and the query binding to
                create the right xpath parser
            parser_factory: the factory we can use to create sub parsers, if not provided we use the default parser
                factory.
            base_path: base path to use for file inclusions
        """
        xpath_parser = cls.get_xpath_parser(xml)
        return cls(xpath_parser, parser_factory, base_path)

    @staticmethod
    def get_xpath_parser(xml: _Element) -> Parser:
        """Determine the Xpath parser from the Schematron root.

        Args:
            xml: the Schematron XML root node

        Returns:
            A `elementpath` parser instance

        Raises:
            ValueError if the query binding is not supported by this library.
        """
        xpath_parsers = {
            'xslt': XPath1Parser,
            'xslt2': XPath2Parser,
            'xslt3': XPath31Parser,
            'xpath': XPath1Parser,
            'xpath2': XPath2Parser,
            'xpath3': XPath3Parser,
            'xpath31': XPath31Parser
        }
        query_binding = xml.attrib.get('queryBinding', 'xslt')

        if query_binding not in xpath_parsers:
            raise ValueError(f'The provided queryBinding {query_binding} is not available in this package.')

        if query_binding.startswith('xslt'):
            warnings.warn(f'XSLT queryBinding support is limited to xpath expressions only.')

        namespaces = {}
        for xpath_namespace in elementpath.select(xml, '/schema/ns'):
            namespaces[xpath_namespace.attrib['prefix']] = xpath_namespace.attrib['uri']

        return xpath_parsers[query_binding](namespaces=namespaces)

class ElementParser(metaclass=ABCMeta):

    @abstractmethod
    def parse(self, element: _Element, context: ParsingContext) -> SchematronNode:
        """Parse a piece of XML data into a SchematronNode.

        Args:
            element: the Schematron XML element we want to parse
            context: the context we use for parsing this Schematron XML element
        """

    @staticmethod
    def _parse_child_tags(element: _Element, context: ParsingContext, xml_tag: str) -> list[SchematronNode]:
        """Parse a sequence of XML elements with the same tag name.

        Args:
            element: the element which we search
            context: the parsing context
            xml_tag: the XML tag to search for
        """
        parser = context.parser_factory.get_parser(xml_tag)

        items = []
        for child in elementpath.select(element, xml_tag):
            items.append(parser.parse(child, context))

        return items

    @staticmethod
    def get_rich_content(element: _Element, context: ParsingContext) -> list[str | XPathToken]:
        """Get the rich content of the provided node.

        Args:
            element: the element for which to get the content.
            context: the parsing context, we use this to parse <value-of> nodes.

        Returns:
            A list of text and XPathToken nodes. All rich content like <emph> and <b> are rendered as string content.
                Nodes of type <value-of> are converted into XPathTokens.
        """
        content = [element.text]

        for child in element.getchildren():
            if child.tag == '{http://purl.oclc.org/dsdl/schematron}value-of':
                content.append(context.xpath_parser.parse(child.attrib['select']))
                content.append(child.tail)
            else:
                content.append(node_to_str(child))
                content.append(child.tail)

        return content