# This program is in the public domain
# Author: Paul Kienzle
"""
SNS data loaders

The following instruments are defined::

    Liquids, Magnetic

These are :class:`resolution.Pulsed` classes tuned with
default instrument parameters and loaders for reduced SNS data.
See :mod:`resolution` for details.
"""

import re
import math

import numpy as np

from bumps.data import parse_file

from .rebin import rebin
from .instrument import Pulsed
from . import resolution
from .probe import make_probe

## Estimated intensity vs. wavelength for liquids reflectometer
LIQUIDS_FEATHER = np.array([
    (2.02555, 20.6369),
    (2.29927, 23.6943),
    (2.57299, 23.6943),
    (2.87409, 21.1146),
    (3.22993, 15.5732),
    (3.58577, 12.8981),
    (4.07847, 9.4586),
    (4.5438, 6.59236),
    (5.11861, 4.68153),
    (5.7208, 3.05732),
    (6.37774, 1.91083),
    (7.19891, 1.24204),
    (8.04745, 0.955414),
    (9.06022, 0.573248),
    (10.1825, 0.477707),
    (11.4142, 0.382166),
    (12.8102, 0.191083),
    (14.3431, 0.286624),
]).T

def load(filename, instrument=None, **kw):
    """
    Return a probe for SNS data.
    """
    if instrument is None:
        instrument=Pulsed()
    header, data = parse_sns_file(filename)
    header.update(**kw) # calling parameters override what's in the file.
    #print "\n".join(k+":"+str(v) for k, v in header.items())
    # Guess what kind of data we have
    if has_columns(header, ('Q', 'dQ', 'R', 'dR', 'L')):
        probe = QRL_to_data(instrument, header, data)
    elif has_columns(header, ('time_of_flight', 'data', 'Sigma')):
        probe = TOF_to_data(instrument, header, data)
    else:
        raise IOError("Unknown columns: "+", ".join(header['columns']))
    probe.title = header['title']
    probe.date = header['date']
    probe.instrument = header['instrument']
    return probe

def has_columns(header, v):
    return (len(header['columns']) == len(v)
            and all(ci == si for ci, si in zip(header['columns'], v)))

def QRL_to_data(instrument, header, data):
    """
    Convert data to T, L, R
    """
    Q, dQ, R, dR, L = data
    dL = resolution.binwidths(L)
    if 'angle' in header and 'slits_at_Tlo' in header:
        T = header.pop('angle', header.pop('T', None))
        probe = instrument.probe(L=L, dL=dL, T=T, data=(R, dR),
                                 **header)
    else:
        T = resolution.QL2T(Q[0], L[0])
        dT = resolution.dQdL2dT(Q[0], dQ[0], L[0], dL[0])
        probe = make_probe(T=T, dT=dT, L=L, dL=dL, data=(R, dR),
                           **header)
    return probe

def TOF_to_data(instrument, header, data):
    """
    Convert TOF data to neutron probe.

    Wavelength is set from the average of the times at the edges of the
    bins, not the average of the wavelengths.  Wavelength resolution is
    set assuming the wavelength at the edges of the bins defines the
    full width at half maximum.

    The correct answer is to look at the wavelength distribution within
    the bin including effects of pulse width and intensity as a function
    wavelength and use that distribution, or a gaussian approximation
    thereof, when computing the resolution effects.
    """
    TOF, R, dR = data
    Ledge = resolution.TOF2L(instrument.d_moderator, TOF)
    L = resolution.TOF2L(instrument.d_moderator, (TOF[:-1]+TOF[1:])/2)
    dL = (Ledge[1:]-Ledge[:-1])/2.35   # FWHM is 2.35 sigma
    R = R[:-1]
    dR = dR[:-1]
    min_time, max_time = header.get('TOF_range', instrument.TOF_range)
    keep = (np.isfinite(R)
            & np.isfinite(dR)
            & (TOF[:-1] >= min_time)
            & (TOF[1:] <= max_time))
    L, dL, R, dR = [v[keep] for v in (L, dL, R, dR)]
    T = np.array([header.get('angle', header.get('T', None))], 'd')
    T, dT, L, dL = instrument.resolution(L=L, dL=dL, T=T, **header)
    probe = make_probe(T=T, dT=dT, L=L, dL=dL, data=(R, dR), **header)
    return probe

def parse_sns_file(filename):
    """
    Parse SNS reduced data, returning *header* and *data*.

    *header* dictionary of fields such as 'data', 'title', 'instrument'
    *data* 2D array of data
    """
    raw_header, data = parse_file(filename)
    header = {}

    # guess instrument from file name
    original_file = raw_header.get('F', 'unknown')
    if 'REF_L' in original_file:
        instrument = 'Liquids'
    elif 'REF_M' in original_file:
        instrument = 'Magnetic'
    else:
        instrument = 'unknown'
    header['instrument'] = instrument
    header['filename'] = original_file
    header['radiation'] = 'neutron'

    # Plug in default instrument values for slits
    if 'instrument' in header and header['instrument'] in INSTRUMENTS:
        instrument = INSTRUMENTS[header['instrument']]
        header['d_s1'] = instrument.d_s1
        header['d_s2'] = instrument.d_s2

    # Date-time field for the file
    header['date'] = raw_header.get('D', '')

    # Column names and units
    columnpat = re.compile(r'(?P<name>\w+)\((?P<units>[^)]*)\)')
    columns, units = zip(*columnpat.findall(raw_header.get('L', '')))
    header['columns'] = columns
    header['units'] = units

    # extra information like title, angle, etc.
    commentpat = re.compile(r'(?P<name>.*)\s*:\s*(?P<value>.*)\s*\n')
    comments = dict(commentpat.findall(raw_header.get('C', '')))
    header['title'] = comments.get('Title', '')
    header['description'] = comments.get('Notes', '')

    # parse values of the form "Long Name: (value, 'units')" in comments
    valuepat = re.compile(r"[(]\s*(?P<value>.*)\s*, \s*'(?P<units>.*)'\s*[)]")
    def parse_value(valstr):
        d = valuepat.match(valstr).groupdict()
        return float(d['value']), d['units']
    if 'Detector Angle' in comments:
        header['angle'], _ = parse_value(comments['Detector Angle'])

    return header, data

def write_file(filename, probe, original=None, date=None,
               title=None, notes=None, run=None, charge=None):
    """
    Save probe as SNS reduced file.
    """
## Example header
#F /SNSlocal/REF_L/2007_1_4B_SCI/2895/NeXus/REF_L_2895.nxs
#E 1174593434.7
#D 2007-03-22 15:57:14
#C Run Number: 2895
#C Title: MK NR4 dry 032007_No2Rep0
#C Notes: MK NR 4 DU 53 dry from air
#C Detector Angle: (0.0, 'degree')
#C Proton Charge: 45.3205833435

#S 1 Spectrum ID ('bank1', (85, 151))
#N 3
#L time_of_flight(microsecond) data() Sigma()
    from datetime import datetime as dt

    parts = []
    if original is None: original = filename
    if date is None:
        date = dt.strftime ( dt.now(), '%Y-%m-%d %H:%M:%S')
    parts.append('#F '+original)
    parts.append('#D '+date)
    if run is not None:
        parts.append('#C Run Number: %s'%run)
    if title is not None:
        parts.append('#C Title: %s'%title)
    if notes is not None:
        parts.append('#C Notes: %s'%notes)
    parts.append("#C Detector Angle: (%g, 'degree')"%probe.T[0])
    if charge is not None:
        parts.append('#C Proton Charge: %s'%charge)
    parts.append('')
    parts.append('#N 5')
    parts.append('#L Q(1/A) dQ(1/A) R() dR() L(A)')
    parts.append('')
    header = "\n".join(parts)
    probe.write_data(filename, columns=['Q', 'dQ', 'R', 'dR', 'L'],
                     header=header)

class SNSData(object):
    def load(self, filename, **kw):
        return load(filename, instrument=self, **kw)

# TODO: print "Insert correct slit distances for Liquids and Magnetic"
class Liquids(SNSData, Pulsed):
    """
    Loader for reduced data from the SNS Liquids instrument.
    """
    instrument = "Liquids"
    radiation = "neutron"
    feather = LIQUIDS_FEATHER
    wavelength = 2., 15.
    #wavelength = 0.5, 5
    #wavelength = 5.5, 10
    #wavelength = 10.5, 15
    dLoL = 0.02
    d_s1 = 230.0 + 1856.0
    d_s2 = 230.0
    d_moderator = 14.850 # moderator to detector distance
    TOF_range = (6000, 60000)

class Magnetic(SNSData, Pulsed):
    """
    Loader for reduced data from the SNS Magnetic instrument.
    """
    instrument = "Magnetic"
    radiation = "neutron"
    wavelength = 1.8, 14
    dLoL = 0.02
    d_s1 = 75*2.54
    d_s2 = 14*2.54

# Instrument names assigned by reflpak
INSTRUMENTS = {
    'Liquids': Liquids,
    'Magnetic': Magnetic,
    }

# ===== utils ==============

def intensity_from_spline(Lrange, dLoL, feather):
    L0, L1 = Lrange
    n = math.ceil(math.log(L1/L0)/math.log(1+dLoL))
    L = L0*(1+dLoL)**np.arange(0, n)
    return (L[:-1]+L[1:])/2, rebin(feather[0], feather[1], L)

def boltzmann_feather(L, counts=100000, range=None):
    """
    Return expected intensity as a function of wavelength given the TOF
    feather range and the total number of counts.

    TOF feather is approximately a boltzmann distribution with gaussian
    convolution.  The following looks pretty enough; don't know how well it
    corresponds to the actual SNS feather.
    """
    import scipy.stats
    y = np.linspace(-4, 4, 10)
    G = np.exp(-y**2/10)
    x = np.arange(12, 85)
    B = scipy.stats.boltzmann.pmf(x, 0.05, counts, loc=16)
    BGz = np.convolve(B, G, mode='same')
    #if range is None: range = L[0], L[-1]
    #if range[0] > range[1]: range = range[::-1]
    #range = range[0]*(1-1e-15), range[1]*(1+1e-15)
    #z = np.linspace(range[0], range[1], len(BGz))
    z = np.linspace(2, 16.5, len(BGz))  # Wavelength range for liquids
    pL = np.interp(L, z, BGz, left=0, right=0)
    nL = pL/sum(pL)*counts
    return  nL


import numpy as np
from string import punctuation
from random import shuffle
from gensim.test.utils import get_tmpfile
import gensim
import pandas as pd
from gensim.models.word2vec import Word2Vec
from gensim.models import KeyedVectors
import time
from nltk.tokenize import TweetTokenizer

def load1_6million(path, tokenizer = gensim.utils.simple_preprocess, limit = np.inf):
    #tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True)
    data = []
    target = []
    i = 0
    with open(path, encoding='latin-1') as f:
        for line in f:
            i += 1
            line = line.replace('"', '')
            sentiment = line.split(',')[1]
            if sentiment not in ('0','1','2','3','4'):
                continue
            if sentiment in  ('0'):
                sentiment = 0
            elif sentiment in ('4'):
                sentiment = 1
            target.append(sentiment)
            tweet = line.split(',')[-1]
            data.append(tokenizer(tweet))
            #data.append(tokenizer.tokenize(tweet))
            if (i == limit):
                return data, target
    return data,target

def saveWordvec(wordvec):
    kv = './twitter_vectors.kv'
    model = "./twitter_vectors.model"
    #fname = get_tmpfile("./twitter_vectors.kv") # if we want to save the file in the default temporary system directory
    wordvec.wv.save(kv) #save the KeyedVectors
    #fname = get_tmpfile("./twitter_vectors.model")
    wordvec.save(model) #save the whole model

def readWordvec(file, kv = True):
    if kv == True:
        return KeyedVectors.load(file, mmap='r')
    else:
        return Word2Vec.load(file)

# Read each line of the text file and return a list of tokens for each tweet and the associated sentiment
data,sentiment = load1_6million ("./kaggle_sentiment140/random_training.csv", tokenizer=gensim.utils.simple_preprocess)

# Creating the custom embedings for our specific domain
begin = time.perf_counter()
# SkipGram = True
model = gensim.models.Word2Vec(data, size=200, window=5, min_count=5, workers=10, sg=1)
model.train(data, total_examples=len(data), epochs=10)
end = time.perf_counter()
print("\nTime elapsed: " + str((end-begin)/60) + " min")
saveWordvec(model)

#model = readWordvec("./twitter_vectors.kv_V1", kv = True)
#print(model.most_similar('good'))

### Augumenting the word vectors
# Read each line of the text file and return a list of tokens for each tweet and the associated sentiment
#nltk.tokenize.TweetTokenizer # a tweet tokenizer from nltk
#data,sentiment = loadTwitterKaggle("./kaggle_Twitter_sentiment_analysis/train.csv", tokenizer=gensim.utils.simple_preprocess)
# # Retrain word2vec using new dataset
# model = readWordvec("./twitter_vectors.model_V1", kv = False)
# # Creating the custom embedings for our specific domain
# begin = time.perf_counter()
# # adding more vocabulary into the previous one
# model.build_vocab(data, update=True)
# model.train(data, total_examples=len(data), epochs=model.epochs)
# end = time.perf_counter()
# print("\nTime elapsed: " + str((end-begin)/60) + " min")
# saveWordvec(model)


import numpy as np
from skimage import measure
from sklearn.metrics import auc

def run(score_imgs, labeled_imgs, fpr_thresh=0.3, max_steps=2000, class_name=None):
    labeled_imgs = np.array(labeled_imgs)
    labeled_imgs[labeled_imgs <= 0.45] = 0
    labeled_imgs[labeled_imgs > 0.45] = 1
    labeled_imgs = labeled_imgs.astype(np.bool)
    score_imgs = np.array(score_imgs)

    max_th = score_imgs.max()
    min_th = score_imgs.min()
    delta = (max_th - min_th) / max_steps

    ious_mean = []
    ious_std = []
    pros_mean = []
    pros_std = []
    threds = []
    fprs = []
    binary_score_maps = np.zeros_like(score_imgs, dtype=np.bool)
    for step in range(max_steps):
        thred = max_th - step * delta
        # segmentation
        binary_score_maps[score_imgs <= thred] = 0
        binary_score_maps[score_imgs > thred] = 1

        pro = []  # per region overlap
        iou = []  # per image iou
        # pro: find each connected gt region, compute the overlapped pixels between the gt region and predicted region
        # iou: for each image, compute the ratio, i.e. intersection/union between the gt and predicted binary map
        for i in range(len(binary_score_maps)):  # for i th image
            # pro (per region level)
            label_map = measure.label(labeled_imgs[i], connectivity=2)
            props = measure.regionprops(label_map)
            for prop in props:
                x_min, y_min, x_max, y_max = prop.bbox
                cropped_pred_label = binary_score_maps[i][x_min:x_max, y_min:y_max]
                # cropped_mask = masks[i][x_min:x_max, y_min:y_max]
                cropped_mask = prop.filled_image  # corrected!
                intersection = np.logical_and(cropped_pred_label, cropped_mask).astype(np.float32).sum()
                pro.append(intersection / prop.area)
            # iou (per image level)
            intersection = np.logical_and(binary_score_maps[i], labeled_imgs[i]).astype(np.float32).sum()
            union = np.logical_or(binary_score_maps[i], labeled_imgs[i]).astype(np.float32).sum()
            if labeled_imgs[i].any() > 0:  # when the gt have no anomaly pixels, skip it
                iou.append(intersection / union)
        # against steps and average metrics on the testing data
        ious_mean.append(np.array(iou).mean())
        #             print("per image mean iou:", np.array(iou).mean())
        ious_std.append(np.array(iou).std())
        pros_mean.append(np.array(pro).mean())
        pros_std.append(np.array(pro).std())
        # fpr for pro-auc
        masks_neg = ~labeled_imgs
        fpr = np.logical_and(masks_neg, binary_score_maps).sum() / masks_neg.sum()
        fprs.append(fpr)
        threds.append(thred)

    # as array
    threds = np.array(threds)
    pros_mean = np.array(pros_mean)
    pros_std = np.array(pros_std)
    fprs = np.array(fprs)

    # default 30% fpr vs pro, pro_auc
    idx = fprs <= fpr_thresh  # find the indexs of fprs that is less than expect_fpr (default 0.3)
    fprs_selected = fprs[idx]
    fprs_selected = rescale(fprs_selected)  # rescale fpr [0,0.3] -> [0, 1]
    pros_mean_selected = pros_mean[idx]
    pro_auc_score = auc(fprs_selected, pros_mean_selected)
    # print("pro auc ({}% FPR):".format(int(expect_fpr * 100)), pro_auc_score)
    return pro_auc_score

def rescale(x):
    return (x - x.min()) / (x.max() - x.min())


import sys
import os
from PyQt5 import QtWidgets, uic, QtCore
import pyqtgraph as pg
import numpy as np
from ..communication import SCPI_mannager, upload
import json

class graph_view(pg.GraphicsLayoutWidget):
    def __init__(self, top_window):
        super().__init__(show=True)
        self.top_window : Top_window = top_window # parent
        self.plt1 = self.addPlot()

        self.vals = []
        y,x = np.histogram([])
        self.hist : pg.PlotDataItem= self.plt1.plot(x, y, stepMode="center", fillLevel=0, fillOutline=True, brush=(0,0,255,150))
        self.update_hist()

        self.line_photon1 = pg.InfiniteLine(movable=True, angle=90, label='1:{value:0.0f}',
                       labelOpts={'position':0.1, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon2 = pg.InfiniteLine(movable=True, angle=90, label='2:{value:0.0f}',
                       labelOpts={'position':0.2, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon3 = pg.InfiniteLine(movable=True, angle=90, label='3:{value:0.0f}',
                       labelOpts={'position':0.3, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon4 = pg.InfiniteLine(movable=True, angle=90, label='4:{value:0.0f}',
                       labelOpts={'position':0.4, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon5 = pg.InfiniteLine(movable=True, angle=90, label='5:{value:0.0f}',
                       labelOpts={'position':0.5, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon6 = pg.InfiniteLine(movable=True, angle=90, label='6:{value:0.0f}',
                       labelOpts={'position':0.6, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon7 = pg.InfiniteLine(movable=True, angle=90, label='7:{value:0.0f}',
                       labelOpts={'position':0.7, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})
        self.line_photon8 = pg.InfiniteLine(movable=True, angle=90, label='8:{value:0.0f}',
                       labelOpts={'position':0.8, 'color': (200,200,100), 'fill': (200,200,200,50), 'movable': False})

        self.plt1.addItem(self.line_photon1)
        self.plt1.addItem(self.line_photon2)
        self.plt1.addItem(self.line_photon3)
        self.plt1.addItem(self.line_photon4)
        self.plt1.addItem(self.line_photon5)
        self.plt1.addItem(self.line_photon6)
        self.plt1.addItem(self.line_photon7)
        self.plt1.addItem(self.line_photon8)

        def temp_func():
            self.on_line_dragged()
            self.top_window.display_value()

        self.line_photon1.sigPositionChangeFinished.connect(temp_func)
        self.line_photon2.sigPositionChangeFinished.connect(temp_func)
        self.line_photon3.sigPositionChangeFinished.connect(temp_func)
        self.line_photon4.sigPositionChangeFinished.connect(temp_func)
        self.line_photon5.sigPositionChangeFinished.connect(temp_func)
        self.line_photon6.sigPositionChangeFinished.connect(temp_func)
        self.line_photon7.sigPositionChangeFinished.connect(temp_func)
        self.line_photon8.sigPositionChangeFinished.connect(temp_func)

    def photon_threshold_set(self, value_dict: dict):
        assert len(value_dict) == 8
        self.line_photon1.setValue(value_dict['photon1'])
        self.line_photon2.setValue(value_dict['photon2'])
        self.line_photon3.setValue(value_dict['photon3'])
        self.line_photon4.setValue(value_dict['photon4'])
        self.line_photon5.setValue(value_dict['photon5'])
        self.line_photon6.setValue(value_dict['photon6'])
        self.line_photon7.setValue(value_dict['photon7'])
        self.line_photon8.setValue(value_dict['photon8'])

    def update_hist(self):
        bins = 1 if len(self.vals) == 0 else np.linspace(min(self.vals) - 0.5, max(self.vals) + 0.5 , max(self.vals) - min(self.vals) + 2)
        y,x = np.histogram(self.vals, bins= bins )
        self.hist.setData(x,y)

    def append_data(self, array: list):
        self.vals = self.vals + array
        self.update_hist()

    def on_line_dragged(self):
        value_dict = {
            'photon1':round(self.line_photon1.value()),
            'photon2':round(self.line_photon2.value()),
            'photon3':round(self.line_photon3.value()),
            'photon4':round(self.line_photon4.value()),
            'photon5':round(self.line_photon5.value()),
            'photon6':round(self.line_photon6.value()),
            'photon7':round(self.line_photon7.value()),
            'photon8':round(self.line_photon8.value())
        }

        self.top_window.photon_threshold_set(value_dict)

    def reset_graph(self):
        self.vals = []
        self.update_hist()

class Top_window(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super().__init__(parent)
        dir_name = os.path.dirname(__file__)
        uic.loadUi(os.path.join(dir_name, 'top.ui'), self)
        self.setWindowTitle('Redpitaya photon number resolver controller')
        self.setFixedSize(self.size())
        self.attach_graph()
        self.toggle_config_input(False)
        self.scpi_mannager: SCPI_mannager = None

        self.pushButton_upload.clicked.connect(self.push_upload)
        self.pushButton_connect.clicked.connect(self.push_connect)
        self.pushButton_disconnect.clicked.connect(self.push_disconnect)

        self.lineEdit_port.setText('5025')

        def trigger_level_slider_change(value):
            self.spinBox_trigger_level.setValue(value)
            self.display_value()
        def trigger_level_spin_change(value):
            self.horizontalSlider_trigger_level.setValue(value)
            self.display_value()

        self.horizontalSlider_trigger_level.valueChanged.connect(trigger_level_slider_change)
        self.spinBox_trigger_level.valueChanged.connect(trigger_level_spin_change)

        self.spinBox_trigger_delay.valueChanged.connect(self.display_value)
        self.spinBox_trigger_clearance.valueChanged.connect(self.display_value)
        self.spinBox_photon1.valueChanged.connect(self.display_value)
        self.spinBox_photon2.valueChanged.connect(self.display_value)
        self.spinBox_photon3.valueChanged.connect(self.display_value)
        self.spinBox_photon4.valueChanged.connect(self.display_value)
        self.spinBox_photon5.valueChanged.connect(self.display_value)
        self.spinBox_photon6.valueChanged.connect(self.display_value)
        self.spinBox_photon7.valueChanged.connect(self.display_value)
        self.spinBox_photon8.valueChanged.connect(self.display_value)

        self.pushButton_read.clicked.connect(self.push_read)
        self.pushButton_write.clicked.connect(self.push_write)
        self.pushButton_graph_reset.clicked.connect(self.push_graph_reset)

        self.pushButton_graph_load.clicked.connect(self.push_graph_load)

    def ui_components(self):
        # just type hint
        self.graph_container                    : QtWidgets.QGridLayout = None
        self.pushButton_upload                  : QtWidgets.QGridLayout = None
        self.pushButton_connect                 : QtWidgets.QPushButton = None
        self.pushButton_disconnect              : QtWidgets.QPushButton = None
        self.pushButton_read                    : QtWidgets.QPushButton = None
        self.pushButton_write                   : QtWidgets.QPushButton = None
        self.pushButton_graph_reset             : QtWidgets.QPushButton = None
        self.pushButton_graph_load              : QtWidgets.QPushButton = None
        self.lineEdit_ip                        : QtWidgets.QLineEdit   = None
        self.lineEdit_port                      : QtWidgets.QLineEdit   = None
        self.spinBox_trigger_level              : QtWidgets.QSpinBox    = None
        self.spinBox_trigger_delay              : QtWidgets.QSpinBox    = None
        self.spinBox_trigger_clearance          : QtWidgets.QSpinBox    = None
        self.spinBox_photon1                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon2                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon3                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon4                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon5                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon6                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon7                    : QtWidgets.QSpinBox    = None
        self.spinBox_photon8                    : QtWidgets.QSpinBox    = None
        self.horizontalSlider_trigger_level     : QtWidgets.QSlider     = None

        self.label_trigger_level_display        : QtWidgets.QLabel      = None
        self.label_trigger_delay_display        : QtWidgets.QLabel      = None
        self.label_trigger_clearance_display    : QtWidgets.QLabel      = None
        self.label_photon1_display              : QtWidgets.QLabel      = None
        self.label_photon2_display              : QtWidgets.QLabel      = None
        self.label_photon3_display              : QtWidgets.QLabel      = None
        self.label_photon4_display              : QtWidgets.QLabel      = None
        self.label_photon5_display              : QtWidgets.QLabel      = None
        self.label_photon6_display              : QtWidgets.QLabel      = None
        self.label_photon7_display              : QtWidgets.QLabel      = None
        self.label_photon8_display              : QtWidgets.QLabel      = None
        self.label_graph_samples                : QtWidgets.QLabel      = None

        self.graph_container                    : QtWidgets.QGridLayout = None

        self.groupBox_controll                  : QtWidgets.QGroupBox   = None
        self.groupBox_trigger_level             : QtWidgets.QGroupBox   = None
        self.groupBox_timing_controll           : QtWidgets.QGroupBox   = None
        self.groupBox_misc_config               : QtWidgets.QGroupBox   = None
        self.groupBox_threshold                 : QtWidgets.QGroupBox   = None
        self.groupBox_graph_control             : QtWidgets.QGroupBox   = None

        self.checkBox_trig_pos_edge             : QtWidgets.QCheckBox   = None
        self.checkBox_trig_is_a                 : QtWidgets.QCheckBox   = None
        self.checkBox_pnr_sig_inverse           : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon0    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon1    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon2    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon3    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon4    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon5    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon6    : QtWidgets.QCheckBox   = None
        self.checkBox_dac_logic_mask_photon7    : QtWidgets.QCheckBox   = None

    def attach_graph(self):
        self.graph = graph_view(self) #pg.GraphicsLayoutWidget(show=True)
        self.graph_container.addWidget(self.graph)

    def toggle_config_input(self, bool):
        self.groupBox_controll.setEnabled(bool)
        self.groupBox_trigger_level.setEnabled(bool)
        self.groupBox_timing_controll.setEnabled(bool)
        self.groupBox_misc_config.setEnabled(bool)
        self.groupBox_threshold.setEnabled(bool)
        self.groupBox_graph_control.setEnabled(bool)

    def push_upload(self):
        host         = self.lineEdit_ip.text().strip()
        ssh_port     = 22
        user         = 'root'
        passwd       = 'root'
        upload_cfg   = json.load(open("upload_cfg.json"))
        assets       = upload_cfg["assets"]
        exec_command = upload_cfg["exec_command"]

        upload(host, ssh_port, user, passwd, assets, exec_command)

    def push_connect(self):
        try:
            self.scpi_mannager = SCPI_mannager(host=self.lineEdit_ip.text().strip(), port=int(self.lineEdit_port.text()))
            result = self.scpi_mannager.idn()
            if result == None: raise Exception('fail for IDN')
            self.pushButton_connect.setEnabled(False)
            self.pushButton_disconnect.setEnabled(True)
            self.toggle_config_input(True)
        except Exception as e:
            print(e)
            pass

    def push_disconnect(self):
        self.pushButton_disconnect.setEnabled(False)
        self.pushButton_connect.setEnabled(True)
        self.toggle_config_input(False)
        self.scpi_mannager: SCPI_mannager = None

    def display_value(self):
        gen_time = lambda x: f'= {8*x} ns'
        gen_volt = lambda x: f'= {x/8.192:.5g} mv'

        self.label_trigger_level_display.setText(gen_volt(self.spinBox_trigger_level.value()))
        self.label_trigger_delay_display.setText(gen_time(self.spinBox_trigger_delay.value()))
        self.label_trigger_clearance_display.setText(gen_time(self.spinBox_trigger_clearance.value()))
        self.label_photon1_display.setText(gen_volt(self.spinBox_photon1.value()))
        self.label_photon2_display.setText(gen_volt(self.spinBox_photon2.value()))
        self.label_photon3_display.setText(gen_volt(self.spinBox_photon3.value()))
        self.label_photon4_display.setText(gen_volt(self.spinBox_photon4.value()))
        self.label_photon5_display.setText(gen_volt(self.spinBox_photon5.value()))
        self.label_photon6_display.setText(gen_volt(self.spinBox_photon6.value()))
        self.label_photon7_display.setText(gen_volt(self.spinBox_photon7.value()))
        self.label_photon8_display.setText(gen_volt(self.spinBox_photon8.value()))

        value_dict = {
            'photon1':self.spinBox_photon1.value(),
            'photon2':self.spinBox_photon2.value(),
            'photon3':self.spinBox_photon3.value(),
            'photon4':self.spinBox_photon4.value(),
            'photon5':self.spinBox_photon5.value(),
            'photon6':self.spinBox_photon6.value(),
            'photon7':self.spinBox_photon7.value(),
            'photon8':self.spinBox_photon8.value()
        }
        self.graph.photon_threshold_set(value_dict)

    def push_read(self):
        self.spinBox_trigger_level.setValue(self.scpi_mannager.read_trigger_level())
        self.spinBox_trigger_delay.setValue(self.scpi_mannager.read_trigger_delay())
        self.spinBox_trigger_clearance.setValue(self.scpi_mannager.read_trigger_clearance())
        self.spinBox_photon1.setValue(self.scpi_mannager.read_photon_threshold(1))
        self.spinBox_photon2.setValue(self.scpi_mannager.read_photon_threshold(2))
        self.spinBox_photon3.setValue(self.scpi_mannager.read_photon_threshold(3))
        self.spinBox_photon4.setValue(self.scpi_mannager.read_photon_threshold(4))
        self.spinBox_photon5.setValue(self.scpi_mannager.read_photon_threshold(5))
        self.spinBox_photon6.setValue(self.scpi_mannager.read_photon_threshold(6))
        self.spinBox_photon7.setValue(self.scpi_mannager.read_photon_threshold(7))
        self.spinBox_photon8.setValue(self.scpi_mannager.read_photon_threshold(8))
        self.checkBox_pnr_sig_inverse.setChecked(self.scpi_mannager.read_pnr_sig_inverse())
        self.checkBox_trig_pos_edge.setChecked(self.scpi_mannager.read_trig_positive_edge())
        self.checkBox_trig_is_a.setChecked(self.scpi_mannager.read_trig_is_a())

        dac_logic_mask = self.scpi_mannager.read_dac_logic_mask()
        self.checkBox_dac_logic_mask_photon0.setChecked(bool(dac_logic_mask & (1<<0)))
        self.checkBox_dac_logic_mask_photon1.setChecked(bool(dac_logic_mask & (1<<1)))
        self.checkBox_dac_logic_mask_photon2.setChecked(bool(dac_logic_mask & (1<<2)))
        self.checkBox_dac_logic_mask_photon3.setChecked(bool(dac_logic_mask & (1<<3)))
        self.checkBox_dac_logic_mask_photon4.setChecked(bool(dac_logic_mask & (1<<4)))
        self.checkBox_dac_logic_mask_photon5.setChecked(bool(dac_logic_mask & (1<<5)))
        self.checkBox_dac_logic_mask_photon6.setChecked(bool(dac_logic_mask & (1<<6)))
        self.checkBox_dac_logic_mask_photon7.setChecked(bool(dac_logic_mask & (1<<7)))

    def push_write(self):
        self.scpi_mannager.set_trigger_level(self.spinBox_trigger_level.value())
        self.scpi_mannager.set_trigger_delay(self.spinBox_trigger_delay.value())
        self.scpi_mannager.set_trigger_clearance(self.spinBox_trigger_clearance.value())
        self.scpi_mannager.set_photon_threshold(1, self.spinBox_photon1.value())
        self.scpi_mannager.set_photon_threshold(2, self.spinBox_photon2.value())
        self.scpi_mannager.set_photon_threshold(3, self.spinBox_photon3.value())
        self.scpi_mannager.set_photon_threshold(4, self.spinBox_photon4.value())
        self.scpi_mannager.set_photon_threshold(5, self.spinBox_photon5.value())
        self.scpi_mannager.set_photon_threshold(6, self.spinBox_photon6.value())
        self.scpi_mannager.set_photon_threshold(7, self.spinBox_photon7.value())
        self.scpi_mannager.set_photon_threshold(8, self.spinBox_photon8.value())
        self.scpi_mannager.set_pnr_sig_inverse(self.checkBox_pnr_sig_inverse.isChecked())
        self.scpi_mannager.set_trig_positive_edge(self.checkBox_trig_pos_edge.isChecked())
        self.scpi_mannager.set_trig_is_a(self.checkBox_trig_is_a.isChecked())

        dac_logic_mask = 0
        dac_logic_mask += self.checkBox_dac_logic_mask_photon0.isChecked() << 0
        dac_logic_mask += self.checkBox_dac_logic_mask_photon1.isChecked() << 1
        dac_logic_mask += self.checkBox_dac_logic_mask_photon2.isChecked() << 2
        dac_logic_mask += self.checkBox_dac_logic_mask_photon3.isChecked() << 3
        dac_logic_mask += self.checkBox_dac_logic_mask_photon4.isChecked() << 4
        dac_logic_mask += self.checkBox_dac_logic_mask_photon5.isChecked() << 5
        dac_logic_mask += self.checkBox_dac_logic_mask_photon6.isChecked() << 6
        dac_logic_mask += self.checkBox_dac_logic_mask_photon7.isChecked() << 7
        self.scpi_mannager.set_dac_logic_mask(dac_logic_mask)

    def push_graph_reset(self):
        self.graph.reset_graph()
        self.scpi_mannager.reset_adc_fifo()
        self.label_graph_samples.setText(f'Samples:{len(self.graph.vals)}')

    def push_graph_load(self):
        array = self.scpi_mannager.read_pnr_adc_fifo()
        self.graph.append_data(array)
        self.label_graph_samples.setText(f'Samples:{len(self.graph.vals)}')

    def photon_threshold_set(self, value_dict: dict):
        assert len(value_dict) == 8
        self.spinBox_photon1.setValue(value_dict['photon1'])
        self.spinBox_photon2.setValue(value_dict['photon2'])
        self.spinBox_photon3.setValue(value_dict['photon3'])
        self.spinBox_photon4.setValue(value_dict['photon4'])
        self.spinBox_photon5.setValue(value_dict['photon5'])
        self.spinBox_photon6.setValue(value_dict['photon6'])
        self.spinBox_photon7.setValue(value_dict['photon7'])
        self.spinBox_photon8.setValue(value_dict['photon8'])


import os
import sys
sys.path.append("../..")

import time
import math
from z3 import *
import argparse
from multiprocessing import Process
from multiprocessing.spawn import freeze_support

from common.utils import show_output, read_instance, write_output, COLORS

def write_report(instance_name, sol, report_file):
    if isinstance(sol, str):
        out = instance_name + ' \t ' + '---\n'
    else:
        out = instance_name + ' \t ' + sol['msg'].split(' ')[-2] + '\n'
    if os.path.isfile(report_file):
        with open(report_file, 'a') as file:
                file.write(out)
    else:
        with open(report_file, 'w') as file:
            file.write('Instance name \t Optimal solution\n')
            file.write(out)

def vlsi_optimize(inst_name, width, num_circuits, w, h, rotation=False, show_sol=True, out_path='../out-rot/', out_img_path='../out-imgs-rot/', report_file='report-rot.csv'):
    area_min = sum([h[i]*w[i] for i in range(num_circuits)])
    max_height = sum(h)
    min_height = math.floor(area_min/width)
    height = min_height
    sol = 'unsat'
    start = time.time()
    while sol == 'unsat' and height <= max_height:
        sol = vlsi_smt(width, height, num_circuits, w, h, start, rotation)
        height +=1
    out_name = inst_name.replace('ins', 'out')
    write_output(out_path + out_name, sol)
    write_report(out_name, sol, report_file)
    show_output(sol, show_sol, out_img_path + out_name.replace('txt', 'png'))

def vlsi_smt(width, height, num_circuits, w, h, start_time, rotation=False):

    circuits_x = [Int(f"x_{i}") for i in range(num_circuits)]
    circuits_y = [Int(f"y_{i}") for i in range(num_circuits)]

    circuits_w = [Int(f"w_{i}") for i in range(num_circuits)]
    circuits_h = [Int(f"h_{i}") for i in range(num_circuits)]

    opt = Solver()
    if rotation:
        opt.add([Or(And(circuits_w[i] == w[i], circuits_h[i] == h[i]), And(circuits_w[i] == h[i], circuits_h[i] == w[i])) for i in range(num_circuits)])
    else:
        opt.add([And(circuits_w[i] == w[i], circuits_h[i] == h[i]) for i in range(num_circuits)])

    non_overlap = [Or(i==j,
                    circuits_x[i] + circuits_w[i] <= circuits_x[j],
                    circuits_x[i] >= circuits_x[j] + circuits_w[j],
                    circuits_y[i] + circuits_h[i] <= circuits_y[j],
                    circuits_y[i] >= circuits_y[j] + circuits_h[j]
                    ) for i in range(num_circuits) for j in range(num_circuits)]

    in_limits = [And(circuits_x[i] + circuits_w[i] <= width,
                circuits_y[i] + circuits_h[i] <= height,
                circuits_x[i] >= 0,
                circuits_y[i] >= 0) for i in range(num_circuits)]

    opt.add(non_overlap + in_limits)

    # Implied constraint on the sum of the circuits' height at position x_thr
    for x_thr in range(width):
        opt.add(Sum([If(And(circuits_x[j] <= x_thr, (circuits_x[j] + circuits_w[j]) > x_thr), circuits_h[j], 0) for j in range(num_circuits)]) <= height)

    # Implied constraint on the sum of the circuits' width at position y_thr
    for y_thr in range(height):
        opt.add(Sum([If(And(circuits_y[j] <= y_thr, (circuits_y[j] + circuits_h[j]) > y_thr), circuits_w[j], 0) for j in range(num_circuits)]) <= width)

    # Symmetry breaking constraints
    lex_x = lex_lesseq(circuits_x, flip(circuits_x, circuits_w, width))
    lex_y = lex_lesseq(circuits_y, flip(circuits_y, circuits_h, height))

    #same_y = [If(
    #    And(i != j, Or(circuits_h[i] == circuits_h[j], circuits_w[i] == circuits_w[j]), circuits_y[i] == circuits_y[j]),
    #    Or(circuits_x[i] < circuits_x[j], circuits_x[j] < circuits_x[i]),
    #    True) for i in range(num_circuits) for j in range(num_circuits)]
    #same_x = [If(
    #    And(i != j, Or(circuits_w[i] == circuits_w[j], circuits_h[i] == circuits_h[j]), circuits_x[i] == circuits_x[j]),
    #    Or(circuits_y[i] < circuits_y[j], circuits_y[j] < circuits_y[i]),
    #    True) for i in range(num_circuits) for j in range(num_circuits)]

    opt.add([lex_x, lex_y])

    if str(opt.check()) == 'sat':
        model = opt.model()
        sol = {
        'width': width,
        'height': height,
        'num_circuits': num_circuits,
        'circuits_w': [int(model.evaluate(circuits_w[i]).as_string()) for i in range(num_circuits)],
        'circuits_h': [int(model.evaluate(circuits_h[i]).as_string()) for i in range(num_circuits)],
        'circuits_x': [int(model.evaluate(circuits_x[i]).as_string()) for i in range(num_circuits)],
        'circuits_y': [int(model.evaluate(circuits_y[i]).as_string()) for i in range(num_circuits)],
        'msg': 'Optimal solution found in {0:.2f} seconds'.format(time.time() - start_time)
        }
    else:
        sol = str(opt.check())

    return sol

def flip(arr, lenghts, max):
    return [max - arr[i] - lenghts[i] for i in range(len(lenghts))]

def lex_lesseq(list_xy, flipped_xy):
    if len(list_xy) == 0:
        return True
    return Or(list_xy[0] <= flipped_xy[0], And(lex_lesseq(list_xy[1:], flipped_xy[1:]), list_xy[0] == flipped_xy[0]))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--show-sols', default=False, help='Whether to show each instance solution (optimal if found, otherwise sub-optimal) at the end of the execution or simply store them.')
    parser.add_argument('--output-folder', default='../out/', help='Path were the solutions .txt files should be stored.')
    parser.add_argument('--output-imgs-folder', default='../out-imgs/', help='Path were to store the graphical representation of each solution.')
    parser.add_argument('--rotation', default=False, help='Whether the SMT model to be executed should allow rotation or not.')
    parser.add_argument('--report-file', default='report.csv', help='File path were to save a summary of the solved instances and the relative timing information')

    args = parser.parse_args()

    if not os.path.isdir(args.output_folder):
        os.mkdir(args.output_folder)

    if not args.show_sols and not os.path.isdir(args.output_imgs_folder):
        os.mkdir(args.output_imgs_folder)

    freeze_support()
    instances_path = '../../common/instances/'
    for instance_file in os.listdir(instances_path):
        print(COLORS['green'], f'Starting to solve {instance_file}...', COLORS['endc'])
        width, num_circuits, circuits_w, circuits_h = read_instance(instances_path + instance_file)
        run = Process(target=vlsi_optimize, args=(instance_file, width, num_circuits, circuits_w, circuits_h,
                    args.rotation, args.show_sols, args.output_folder, args.output_imgs_folder, args.report_file), name='optimize_instance')
        run.start()
        run.join(timeout=310)
        if run.is_alive():
            print(COLORS['red'], f'No solution found for {instance_file}.', COLORS['endc'])
            run.terminate()


import re
import requests
from bs4 import BeautifulSoup

def baidu_search(word: str) -> str:
    """
    百度百科检索问题
    :param word: 需要查询的问题
    :return: 百度百科查询结果
    """
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62"
    }

    url = f'https://baike.baidu.com/item/{word}'
    response = requests.get(url=url, headers=headers, timeout=10)
    html_content = response.text
    soup = BeautifulSoup(html_content, 'lxml')

    li_list = soup.select('.lemma-summary')

    results = [re.sub(r'\[[0-9 \-]+]', '', i.text).strip() for i in li_list]
    result = ''.join(results)
    if len(result) > 100:
        result = result[:100] + "……"
    return result


#!/usr/bin/env python3
import asyncio
import sys
from getpass import getpass
from pathlib import Path
from typing import Dict

import httpx
import hug

IGNORED_AUTHOR_LOGINS = {"deepsource-autofix[bot]"}

REPO = "pycqa/isort"
GITHUB_API_CONTRIBUTORS = f"https://api.github.com/repos/{REPO}/contributors"
GITHUB_USER_CONTRIBUTIONS = f"https://github.com/{REPO}/commits?author="
GITHUB_USER_TYPE = "User"
USER_DELIMITER = "-" * 80
PER_PAGE = 100

_ACK_FILE = Path(__file__).parent.parent / "docs" / "contributing" / "4.-acknowledgements.md"
ACKNOWLEDGEMENTS = _ACK_FILE.read_text().lower()

def _user_info(user: Dict[str, str], verbose=False) -> str:
    login = "@" + user["login"]
    name = user.get("name")
    display_name = f"{name} ({login})" if name else login
    user_info = f"- {display_name}"
    if verbose:
        contributions = f"  {GITHUB_USER_CONTRIBUTIONS}{user['login']}"
        user_info += "\n" + contributions
    return user_info

@hug.cli()
async def main():
    auth = (input("Github Username: "), getpass())
    async with httpx.AsyncClient() as client:
        page = 0
        results = []
        contributors = []
        while not page or len(results) == PER_PAGE:
            page += 1
            response = await client.get(
                f"{GITHUB_API_CONTRIBUTORS}?per_page={PER_PAGE}&page={page}", auth=auth
            )
            results = response.json()
            contributors.extend(
                (
                    contributor
                    for contributor in results
                    if contributor["type"] == GITHUB_USER_TYPE
                    and contributor["login"] not in IGNORED_AUTHOR_LOGINS
                    and f"@{contributor['login'].lower()}" not in ACKNOWLEDGEMENTS
                )
            )

        unacknowledged_users = await asyncio.gather(
            *(client.get(contributor["url"], auth=auth) for contributor in contributors)
        )
        unacknowledged_users = [request.json() for request in unacknowledged_users]

        if not unacknowledged_users:
            sys.exit()

        print("Found unacknowledged authors:")
        print()

        for user in unacknowledged_users:
            print(_user_info(user, verbose=True))
            print(USER_DELIMITER)

        print()
        print("Printing again for easy inclusion in Markdown file:")
        print()
        for user in unacknowledged_users:
            print(_user_info(user))

        sys.exit(1)

if __name__ == "__main__":
    main.interface.cli()


import argparse
import os

from collector.github_repo_collector import GithubRepositoryCollector
from const.constants import GITHUB_ACCESS_TOKEN

def parse_args() -> argparse.Namespace:
    """
    Parse arguments.
    :return: a namespace with the arguments
    """
    parser = argparse.ArgumentParser(description='Get repositories from GitHub.')
    parser.add_argument('--total', type=int, default=100, help='the total number of repositories to save')
    return parser.parse_args()

if __name__ == '__main__':
    if GITHUB_ACCESS_TOKEN not in os.environ:
        raise Exception('GITHUB_ACCESS_TOKEN not found in environment variables.')
    args = parse_args()
    g = GithubRepositoryCollector()
    g.collect(args.total)


#import the spaCy library
import spacy

# Load the spaCy model for NER(this is the sm version so faster but less accurate)
nlp = spacy.load("en_core_web_sm")
# Define a function to anonymize personal information
def anonymize_text(text):
     # Use spaCy NER to process the input text
    doc = nlp(text)
    # Create an empty list to store the anonymized text
    anonymized_text = []
    # Iterate over each token in the processed text
    for token in doc:
        # Check if the token is a ORG entity
        if token.ent_type_ == "ORG":
            # If so, add "[ANONYMIZED]" to the anonymized text
            anonymized_text.append("[ANONYMIZED ORG]")
            # Check if the token is a TIME entity
        elif token.ent_type_ == "TIME":
            # If so, add "[ANONYMIZED]" to the anonymized text
            anonymized_text.append("[ANONYMIZED TIME]")
            # Check if the token is a Person entity(not working find out why)
        elif token.ent_type_ == "PERSON":
            # If so, add "[ANONYMIZED]" to the anonymized text
            anonymized_text.append("[ANONYMIZED NAME]")
        else:
            # Add the token's text to the anonymized text if it isn't
            anonymized_text.append(token.text)
            # Join the list of tokens into a single string
    anonymized_text = " ".join(anonymized_text)
    # Print the anonymized text
    print(anonymized_text)
###############################################################################
# Variable for participation and reuse of code
cont = input("Would you like to Anonymize text? (Y/N) ")
# If N, then stop running code.
while(cont != "N"):
    # If Y, then run the code below
  if (cont == "Y"):
    # Get input text from user
    text = input("Enter text: ")
    # Anonymize the text
    anonymize_text(text)
  else:
    # Reasks if there is more text to be added
    print("Please enter 'Y' or 'N'")
  cont = input("More Text? (Y/N) ")


#!/usr/bin/python3

import argparse
import cv2
import numpy as np

def onTrackbar(threshold):
    print("Selected threshold " + str(threshold) + " for limit")

def main():
    # parse the argument
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--image', type=str, required=True, help='Full path to image file.')
    args = vars(parser.parse_args())

    # read the image original
    image = cv2.imread(args['image'], cv2.IMREAD_COLOR)  # Load an image
    cv2.imshow("original", image)

    # convert to gray scale
    image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    cv2.imshow("image_gray", image_gray)

    # convert to hsv
    image_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    cv2.imshow("image_hsv", image_hsv)

    # dictionary with ranges
    ranges_pcss = {"b": {"min": 100, "max": 256},
                   "g": {"min": 100, "max": 256},
                   "r": {"min": 100, "max": 256},
                   }

    # create the trackbars
    cv2.namedWindow('image_process')
    cv2.createTrackbar("min B", "image_process", 0, 255, onTrackbar)
    cv2.createTrackbar("max B", "image_process", 0, 255, onTrackbar)
    cv2.createTrackbar("min G", "image_process", 0, 255, onTrackbar)
    cv2.createTrackbar("max G", "image_process", 0, 255, onTrackbar)
    cv2.createTrackbar("min R", "image_process", 0, 255, onTrackbar)
    cv2.createTrackbar("max R", "image_process", 0, 255, onTrackbar)

    # create the trackbars
    cv2.namedWindow('mask')
    cv2.createTrackbar("min", "mask", 0, 255, onTrackbar)
    cv2.createTrackbar("max", "mask", 0, 255, onTrackbar)

    # cycle for editing the images_recog
    while True:
        # close windows on ESC
        k = cv2.waitKey(1) & 0xFF
        if k == 27:
            break

        min_b_pcss = cv2.getTrackbarPos("min B", "image_process")
        max_b_pcss = cv2.getTrackbarPos("max B", "image_process")
        min_g_pcss = cv2.getTrackbarPos("min G", "image_process")
        max_g_pcss = cv2.getTrackbarPos("max G", "image_process")
        min_r_pcss = cv2.getTrackbarPos("min R", "image_process")
        max_r_pcss = cv2.getTrackbarPos("max R", "image_process")

        ranges_pcss["b"]["min"] = min_b_pcss
        ranges_pcss["b"]["max"] = max_b_pcss
        ranges_pcss["g"]["min"] = min_g_pcss
        ranges_pcss["g"]["max"] = max_g_pcss
        ranges_pcss["r"]["min"] = min_r_pcss
        ranges_pcss["r"]["max"] = max_r_pcss

        mins_pcss = np.array([ranges_pcss['b']['min'], ranges_pcss['g']['min'], ranges_pcss['r']['min']])
        maxs_pcss = np.array([ranges_pcss['b']['max'], ranges_pcss['g']['max'], ranges_pcss['r']['max']])

        image_process = cv2.inRange(image, mins_pcss, maxs_pcss)
        cv2.imshow("image_process", image_process)

        # gray image --------------------------------------------------------
        min_gray = cv2.getTrackbarPos("min", "mask")
        max_gray = cv2.getTrackbarPos("max", "mask")

        mask = cv2.inRange(image_gray, min_gray, max_gray)
        cv2.imshow("mask", mask)

if __name__ == '__main__':
    main()


from astropy.io import fits
import pandas as pd
import numpy as np
import time
import traceback
import os
flux1350=[]
flux3000=[]
df=pd.read_csv('/media/richard/Backup Plus/candidate_dr16_0.8_final.csv',low_memory=False)
groupid=df['GroupID_1']
specname=df['specname_new']
z=df['Z']
area_1350=df['LINEAREA_1350']
area_3000=df['LINEAREA_3000']
f=open('/media/richard/Backup Plus/error_log.txt',"a")

for i in range(len(specname)):
    try:
        if os.path.exists('/media/richard/Backup Plus/sdss_16_pair/'+str(specname[i]))==True:
            fit=fits.open('/media/richard/Backup Plus/sdss_16_pair/'+str(specname[i]))
            data_fit=fit[1].data
            for j in range(len(data_fit.field('loglam'))):
                lam=(10**(data_fit.field('loglam')[j]))/(z[i]+1)
                if lam>1350-25 and lam<1350+25:
                    flux1350.append(data_fit.field('flux')[j])
                elif lam>3000-25 and lam<3000+25:
                    flux3000.append(data_fit.field('flux')[j])
            df.loc[df.specname_new==df.specname_new[i],'LINEAREA_1350']=np.mean(flux1350)
            df.loc[df.specname_new==df.specname_new[i],'LINEAREA_3000']=np.mean(flux3000)
            print(specname[i])
            print(df['LINEAREA_1350'][i])
            print(df['LINEAREA_3000'][i])
            flux1350.clear()
            flux3000.clear()
        else:
            df.loc[df.specname_new==df.specname_new[i],'LINEAREA_1350']='nan'
            df.loc[df.specname_new==df.specname_new[i],'LINEAREA_3000']='nan'
            continue
    except TypeError:
        print(specname[i])
        traceback.print_exc()
        df.loc[df.specname_new==df.specname_new[i],'LINEAREA_1350']='nan'
        df.loc[df.specname_new==df.specname_new[i],'LINEAREA_3000']='nan'
        f.write('cannot caculate the area:%s'%specname[i])
        pass
    except Exception:
        df.loc[df.specname_new==df.specname_new[i],'LINEAREA_1350']='nan'
        df.loc[df.specname_new==df.specname_new[i],'LINEAREA_3000']='nan'
        print('failed')
        pass
df.to_csv('/home/richard/data/change-look-AGN/dr16_0.8_final.csv')


import requests
import json
import time
from string import ascii_letters,digits
import random
from base64 import b64encode
from hashlib import sha256
from os import urandom

def request_verify_code(phone_number):

    random.seed(urandom(8))

    h=sha256()
    code_verifier=''.join([ random.choice(ascii_letters+digits) for _ in range(43)])
    h.update(code_verifier.encode())
    code_challenge=b64encode(h.digest()).decode().replace("=","").replace("+","-").replace("/","_")

    header={
        "Content-Type":"application/json",
        "User-Agent": "okhttp/3.12.1",
    }

    data={
        "code_challenge": code_challenge,
        "code_challenge_method": "S256",
        "username": "+886"+phone_number[1:]
    }

    requests.post("https://auth.ridegoshareapi.com/v1/sms_code",data=json.dumps(data),headers=header)

    return code_verifier

def submit_verify_code(phone_number,code_verifier,sms_code):

    header={
        "User-Agent": "okhttp/3.12.1",
        "Authorization":"Basic NTFlZmVkNjktYjQ0MC00N2Q3LThhNTMtMmEzY2ViZTY1YzkyOg=="
    }

    data={
        "grant_type":"sms",
        "username":"+886"+phone_number[1:],
        "uuid":"54ed48f9-9f9e-4162-8542-2ed8f35ebb50",
        "sms_code":sms_code,
        "code_verifier":code_verifier
    }

    res=requests.post("https://auth.ridegoshareapi.com/oauth/token",data=data,headers=header)

    data=res.json()

    print(data)

    access_token=data["access_token"]
    refresh_token=data['refresh_token']

    return access_token,refresh_token

def get_scooter_id(plate):
    res=requests.get("https://rental.ridegoshareapi.com/v2/cities/514f2d1d-9faf-490b-b9b2-fe8ce4dce584/scooters")
    for scooter in res.json()['upsert_lst']:
        if scooter['plate']==plate:
            return scooter["id"]
    return None

def reserve(scooter_id,access_token):

    data={
        "corporate_type": 0,
        "scooter_id": scooter_id,
        "source": 0
    }

    header={
        "User-Agent": "okhttp/3.12.1",
        "Authorization":f"Bearer {access_token}",
        "Content-Type":"application/json"
    }

    res=requests.post("https://rental.ridegoshareapi.com/v2.1/rentals",data=json.dumps(data),headers=header)

    data=res.json()

    return data["id"]

def cancel_reserve(reserve_id,access_token):

    data={
        "action": 0,
        "payment_method": 0
    }

    header={
        "User-Agent": "okhttp/3.12.1",
        "Authorization":f"Bearer {access_token}",
        "Content-Type":"application/json"
    }

    res=requests.patch(f"https://rental.ridegoshareapi.com/v2.1/rentals/{reserve_id}",data=json.dumps(data),headers=header)

    #assert res.json()['id']==reserve_id
    #assert res.json()['state']==4

def refresh_token(refresh_token):

    header={
        "User-Agent": "okhttp/3.12.1",
        "Authorization":"Basic NTFlZmVkNjktYjQ0MC00N2Q3LThhNTMtMmEzY2ViZTY1YzkyOg=="
    }

    data={
        "grant_type":"refresh_token",
        "uuid":"54ed48f9-9f9e-4162-8542-2ed8f35ebb50",
        "refresh_token":refresh_token
    }

    res=requests.post("https://auth.ridegoshareapi.com/oauth/token",data=data,headers=header)

    data=res.json()

    access_token=data["access_token"]
    refresh_token=data['refresh_token']

    return access_token,refresh_token


# Factor Analysis

import matplotlib.pyplot as plt

def Factor_EM(Input_data,K_sub=3):
    trans=Input_data
    (N,D)=trans.shape
    #Initialize
    u=np.mean(trans,axis=0)
    Input_data=FTI
    sigma_full=np.cov(FTI.transpose())
    [U_matrix,D_matrix,V_matrix]=np.linalg.svd(sigma_full)
    eta_start=np.multiply(np.sqrt(D_matrix[0:K_sub]),U_matrix[:,0:K_sub])#
    sigma_start=np.diag(np.diag(sigma_full)-np.diag(np.dot(eta_start,eta_start.transpose())))

    sigma_current=sigma_start
    eta_current=eta_start

    Exp_hh=np.zeros((N,K_sub,K_sub))
    sum_Exp_hh=np.zeros((K_sub,K_sub))

    for t in range(60):

        sigma_current_inv=np.linalg.pinv(sigma_current)
        a=np.dot(np.dot(eta_current.transpose(),sigma_current_inv),eta_current)

        b=np.linalg.pinv(a+np.identity(K_sub))
        d=trans-u

        Exp_h=np.dot(np.dot(np.dot(b,eta_current.transpose()),sigma_current_inv),d.transpose()).transpose()#N*K

        for i in range(N):
            Exp_hh[i,:,:]= np.outer(Exp_h[i,:],Exp_h[i,:])+b
            sum_Exp_hh=sum_Exp_hh+Exp_hh[i,:,:]

        e=np.dot(Exp_h.transpose(),d)

        eta_next=np.dot(e.transpose(),np.linalg.pinv(sum_Exp_hh))
        sigma_next=np.diag(np.diag(np.dot(d.transpose(),d))-np.diag(np.dot(eta_next,e)))/N#D*D

        #checking convergence
        delta_eta=np.linalg.norm(eta_next-eta_current)/np.linalg.norm(eta_current)
        delta_sigma=np.linalg.norm(sigma_next-sigma_current)/np.linalg.norm(sigma_current)

        eta_current=eta_next
        sigma_current=sigma_next

    eta=eta_current
    sigma=sigma_current

    return([u,eta,sigma])

def Factor_logP(Input_data_orig,F=True,K_sub=3):
    if(F==True):#for F
        Input_data=Input_data_orig
        [u,eta,sigma]=Factor_EM(FTI,K_sub=K_sub)
    else:#for NonF
        Input_data=Input_data_orig
        [u,eta,sigma]=Factor_EM(NonFTI,K_sub=K_sub)

    temp_1=np.dot(eta,eta.transpose())+sigma
    temp_2=Input_data-u
    log_p=-(1/2)*np.sum(np.log(np.linalg.svd(temp_1)[1]))-(1/2)*np.sum(np.multiply(np.dot(temp_2,np.linalg.pinv(temp_1)),temp_2),axis=1)
    return(log_p)

def Label_Factor(Input_data_orig,K_sub=3,threshold=0.5):
    delta=Factor_logP(Input_data_orig,F=True,K_sub=K_sub)-Factor_logP(Input_data_orig,F=False,K_sub=K_sub)
    rt=np.log(threshold/(1-threshold))

    if(isinstance(threshold,np.ndarray)==False):
        estimated_label=np.zeros(Input_data_orig.shape[0])
        estimated_label[[i for i in range(Input_data_orig.shape[0]) if delta[i]>rt]]=1
    return(estimated_label)

#FR
def FR_Factor(Input_data_orig,true_label,K_sub=3,threshold=0.5):
    N=Input_data_orig.shape[0]
    delta=Factor_logP(Input_data_orig,F=True,K_sub=K_sub)-Factor_logP(Input_data_orig,F=False,K_sub=K_sub)#log_p_F-log_p_nonF
    rt=np.log(threshold/(1-threshold))
    if(isinstance(threshold,np.ndarray)==False):#threshold is a scalar
        #F_or_nonF
        estimated_label=np.zeros(N)
        estimated_label[[i for i in range(N) if delta[i]>rt]]=1
        #False Rate
        FR=np.zeros(3)
        FR[0]=np.mean(estimated_label[[i for i in range(N) if true_label[i]==0]])
        FR[1]=1-np.mean(estimated_label[[i for i in range(N) if true_label[i]==1]])
        FR[2]=np.mean(np.abs(estimated_label-true_label))
        return(FR)

def Factor_ROC(Input_data_orig,true_label,rt_seq,K_sub=3):
    N=Input_data_orig.shape[0]
    delta=Factor_logP(Input_data_orig,F=True,K_sub=K_sub)-Factor_logP(Input_data_orig,F=False,K_sub=K_sub)#log_p_F-log_p_nonF
    if(isinstance(rt_seq,np.ndarray)):#threshold is a seq
        FR=np.zeros((2,len(rt_seq)))
        for i in range(len(rt_seq)):
            #F_or_nonF
            rt=rt_seq[i]
            estimated_label=np.zeros(N)
            estimated_label[[i for i in range(N) if delta[i]>rt]]=1
            #False Rate
            FR[0,i]=np.mean(estimated_label[[i for i in range(N) if true_label[i]==0]])
            FR[1,i]=1-np.mean(estimated_label[[i for i in range(N) if true_label[i]==1]])
        plt.plot(FR[0,:],1-FR[1,:],"r--")
        plt.show()

#Evaluating the learned model on the testing images
Test_true_label=np.zeros(200)
Test_true_label[0:100]=1
TTL=np.zeros(2000)
TTL[0:1000]=1

print(FR_Factor(TI,true_label=TTL,threshold=0.5,K_sub=3))
print(FR_Factor(Test_images,true_label=Test_true_label,threshold=0.5,K_sub=3))
Factor_ROC(TI,true_label=TTL,rt_seq=np.arange(-1500,1500,100),K_sub=3)

[F_u,F_eta,F_sigma]=Factor_EM(FTI,3)
[NonF_u,NonF_eta,NonF_sigma]=Factor_EM(NonFTI,3)

#mean
plt.subplot(2, 2, 1)
plt.imshow(F_u.reshape((10,10,3)).astype(int))
plt.title("mean-Face")

plt.subplot(2, 2, 2)
plt.imshow(NonF_u.reshape((10,10,3)).astype(int))
plt.title("mean-NonFace")

#covariance
plt.subplot(2, 2, 3)
cov_diag=np.diag(np.dot(F_eta,F_eta.transpose())+F_sigma)
[min_v,max_v]=[np.min(cov_diag),np.max(cov_diag)]
norm_cov_diag=(cov_diag-min_v)/(max_v-min_v)*255
plt.imshow(norm_cov_diag.reshape((10,10,3)).astype(int))
plt.title("cov-Face")

plt.subplot(2, 2, 4)
cov_diag=np.diag(np.dot(NonF_eta,NonF_eta.transpose())+NonF_sigma)
[min_v,max_v]=[np.min(cov_diag),np.max(cov_diag)]
norm_cov_diag=(cov_diag-min_v)/(max_v-min_v)*255
plt.imshow(norm_cov_diag.reshape((10,10,3)).astype(int))
plt.title("cov-NonFace")

# FLASE POSITIVE RATE, FALSE NEGATIVE RATE AND MISCLASSIFICATION RATE FOR ALL THE MODELS

print(FR_Gaussian(Test_images,true_label=Test_true_label,threshold=0.5))

print(FR_Mix_Gaussian(Test_images,Test_true_label,K=6,threshold=0.5))

print(FR_T(Test_images,true_label=Test_true_label,threshold=0.5,v_start=5))

print(FR_Factor(Test_images,true_label=Test_true_label,threshold=0.5,K_sub=3))


import cv2
import librosa
import numpy as np
import random
from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler, scale

def scale_feature(feature, featureSize):

    widthTarget, heightTarget = featureSize
    height, width = feature.shape

    # scale according to factor
    newSize = (int(width / 4),41)
    #print ('newSize ={}, old size = {}'.format(newSize, feature.shape ))
    feature = cv2.resize(feature, newSize)
    # Normalization
    scaler = StandardScaler()
    feature = scaler.fit_transform(feature)
    feature = np.pad(feature, ((0, 0), (0, widthTarget - feature.shape[1])), 'constant')
    #transpose
    feature = np.transpose(feature)

    return feature

def sample_preprocess(sample_path):

	file = str(sample_path)
	y,sr=librosa.load(file)

	ZCR = librosa.feature.zero_crossing_rate(y, frame_length=2048, hop_length=512, center=True)

	mel_spec = librosa.feature.melspectrogram(y=y, sr=sr, S=None, n_fft=2048,
                                        	hop_length=512, win_length=None, window='hann',
                                            center=True, pad_mode='reflect', power=2.0)

	chroma_stft = librosa.feature.chroma_stft(y=y, sr=sr, S=None, norm=None, n_fft=2048,
                                    		hop_length=512, win_length=None, window='hann',
                                    		center=True, pad_mode='reflect', tuning=None, n_chroma=12)

	MFCC = librosa.feature.mfcc(y=y, sr=sr, S=None, n_mfcc=20, dct_type=2, norm='ortho', lifter=0)

	spectral_centroid = librosa.feature.spectral_centroid(y=y, sr=sr, S=None, n_fft=2048,
                                          	hop_length=512, freq=None, win_length=None, window='hann',
                                            center=True, pad_mode='reflect')

	spectral_bandwidth = librosa.feature.spectral_bandwidth(y=y, sr=sr, S=None, n_fft=2048,
                                           	hop_length=512, win_length=None, window='hann',
                                           	center=True, pad_mode='reflect', freq=None, centroid=None, norm=True, p=2)

	spectral_rolloff = librosa.feature.spectral_rolloff(y=y, sr=sr, S=None, n_fft=2048,
                                         	hop_length=512, win_length=None, window='hann',
                                         	center=True, pad_mode='reflect', freq=None, roll_percent=0.85)

	feature = np.concatenate((mel_spec,chroma_stft,MFCC,ZCR,spectral_centroid,spectral_bandwidth,spectral_rolloff),
                                 axis=0)
	feature = librosa.power_to_db(feature, ref=np.max)
	#length = aug_feature.shape[1]
	max_length = 1292
	scaled_feature = scale_feature(feature,featureSize = (int(max_length/4), 41)) # 323 = 1292/4, 41 = 164/4 (from original data)
	scaled_feature = scaled_feature.reshape(-1, int(max_length/4), 41, 1)

	return scaled_feature


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import base64
import requests

from contextlib import closing

from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials

def authenticate_gmail_api(credentials):
    """Authenticate email API from credentials declared from an env variable.

    Args:
        credentials: A string containing the filepath to the API credentials.
        token:

    Returns:
        API service object.
    """

    SCOPE = ['https://www.googleapis.com/auth/gmail.readonly']

    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    # TODO: Resolve contextlib closures once token and credentials are determined on GCP
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPE)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                credentials, SCOPE)
            creds = flow.run_local_server(port=50683)
        # Save the credentials for the next run
        with closing(open('token.json', 'w')) as token:
            token.write(creds.to_json())

    service = build('gmail', 'v1', credentials=creds)

    return service

def get_email_content(service, query = None):
    """Use the GMail API to query email bodies and decode into string format.

    Args:
        service: GMail API service object.
        query: A string containing GMail search syntax.

    Returns:
        decoded_list: A list of email bodies.
    """

    # Call the Gmail API
    emails = service.users().messages().list(userId='me',q=query, maxResults=101,includeSpamTrash=False).execute()
    emails = emails.get('messages')

    if emails is not None:

        id_list = [id['id'] for id in emails]

        body_list = []

        # Check for container MIME message parts.
        body = service.users().messages().get(userId='me',id=id_list[0]).execute().get('payload').get('body').get('data')

        if body == None:
            for email in id_list:
                body = service.users().messages().get(userId='me',id=email).execute().get('payload').get('parts')
                text = body[0].get('body').get('data')
                html = body[1].get('body').get('data')
                body = tuple((text,html))
                body_list.append(html)

        else:
            for email in id_list:
                body = service.users().messages().get(userId='me',id=email).execute().get('payload').get('body').get('data')
                body_list.append(body)

        bytes_list = [bytes(str(x),encoding='utf-8') for x in body_list]
        decoded_list = [base64.urlsafe_b64decode(x) for x in bytes_list]
        str_list = [str(x) for x in decoded_list]

        return str_list

    service.close()

def get_rescuetime_daily(KEY):
    """Use the RescueTime API to get daily totals for the past two weeks of time spent on personal digital devices.

    Args:
        KEY: RescueTime API Key.

    Returns:
        rescuetime_tuple: List of tuples containing daily total, productive, distracting, and neutral hours.
    """

    # Append API key to API URL.
    url = f'https://www.rescuetime.com/anapi/daily_summary_feed?key={KEY}'

    with closing(requests.Session()) as session:
        r = session.get(url)
        iter_result = r.json()

        days = [day.get('date') for day in iter_result]
        prod_hours = [day.get('all_productive_hours') for day in iter_result]
        dist_hours = [day.get('all_distracting_hours') for day in iter_result]
        neut_hours = [day.get('neutral_hours') for day in iter_result]

        rescuetime_tuple = [(day,p,d,n) for (day,p,d,n) in zip(days,prod_hours,dist_hours,neut_hours)]

    session.close()

    return rescuetime_tuple


import numpy as np
import pygame

class GridEnvironment:

    def __init__(self):
        self.grid = np.zeros((7,7))
        self.starting_position = [0,0]
        self.goal_position = [5,4]
        self.reset()
        self.walls = [[3, 2], [2, 3], [1, 4]]
        #actions are up, down, left, right mapped to 0, 1, 2, 3

    def reset(self):
        self.agent = self.starting_position
        self.special_blocks = {
            -2: [4, 1],
            2: [2, 2],
            1: [2, 4],
            3: [4, 6]
        }
        return self.agent

    def valid_move(self, state, action):
        # Check for walls or outside grid
        new_position = list(state)
        if action == 0:
            new_position[0] -= 1
        elif action == 1:
            new_position[0] += 1
        elif action == 2:
            new_position[1] -= 1
        elif action == 3:
            new_position[1] += 1

        if new_position in self.walls or new_position[0] < 0 or new_position[0] > 6 or new_position[1] < 0 or new_position[1] > 6:
            return False
        return True

    def check_value(self, state):
        for reward_value, coordinates in self.special_blocks.items():
            if coordinates == state:
                return reward_value
        return 0

    def remove_value(self, state):
        for reward_value, coordinates in list(self.special_blocks.items()):
            if coordinates == state:
                self.special_blocks.pop(reward_value)
                break

    def next_state(self, action):
        new_position = list(self.agent)
        if action == 0:
            new_position[0] -= 1
        elif action == 1:
            new_position[0] += 1
        elif action == 2:
            new_position[1] -= 1
        elif action == 3:
            new_position[1] += 1

        if self.valid_move(self.agent, action):
            return new_position
        else:
            return self.agent

    def step(self, action):
        next_position = self.next_state(action)
        reward = -1  # default reward

        special_reward = self.check_value(next_position)
        if special_reward:
            reward += special_reward
            self.remove_value(next_position)

        self.agent = next_position  # updating the agent's position

        if self.agent == self.goal_position:
            reward += 10
            return next_position, reward, True

        return next_position, reward, False

    def render(self, q_table):
        pygame.init()
        self.reset()

        # Configurations
        BLOCK_SIZE = 60  # Increased block size
        WINDOW_SIZE = [self.grid.shape[0] * BLOCK_SIZE, self.grid.shape[1] * BLOCK_SIZE]
        screen = pygame.display.set_mode(WINDOW_SIZE)
        pygame.display.set_caption('Q-table Visualization')

        colors = {
            'white': (255, 255, 255),
            'red': (255, 0, 0),
            'green': (0, 255, 0),
            'blue': (0, 0, 255),
            'yellow': (255, 255, 0),
            'gray': (200, 200, 200)
        }

        actions = ["up", "down", "left", "right"]
        font = pygame.font.SysFont(None, 25)  # Reduced font size

        clock = pygame.time.Clock()
        done = False

        while not done:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    done = True

            screen.fill(colors['white'])

            for i in range(self.grid.shape[0]):
                for j in range(self.grid.shape[1]):
                    state = (i, j)
                    max_action = np.argmax(q_table[state])

                    # Choose color based on max action value
                    max_val = q_table[state][max_action]
                    color = colors['green'] if max_val > 0 else colors['red'] if max_val < 0 else colors['blue']

                    pygame.draw.rect(screen, color, [j * BLOCK_SIZE, i * BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE])

                    # Display action text
                    action_text = font.render(actions[max_action], True, colors['white'])
                    screen.blit(action_text, (j * BLOCK_SIZE + 10, i * BLOCK_SIZE + 10))

                    # Check and draw special blocks and their values
                    for reward_value, coordinates in self.special_blocks.items():
                        if coordinates == [i, j]:
                            pygame.draw.circle(screen, colors['yellow'],
                                               (j * BLOCK_SIZE + BLOCK_SIZE // 2, i * BLOCK_SIZE + BLOCK_SIZE // 2),
                                               BLOCK_SIZE // 3)
                            reward_text = font.render(str(reward_value), True, colors['gray'])
                            screen.blit(reward_text,
                                        (j * BLOCK_SIZE + BLOCK_SIZE // 2 - 10, i * BLOCK_SIZE + BLOCK_SIZE // 2 - 10))

                    # Draw walls
                    if [i, j] in self.walls:
                        pygame.draw.rect(screen, colors['gray'],
                                         [j * BLOCK_SIZE, i * BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE])

            pygame.display.flip()
            clock.tick(60)

        pygame.quit()


import networkx as nx
import numpy as np
from scipy import random
import pandas as pd
import copy
import random
from collections import OrderedDict, Counter
from multiprocessing import Pool
import itertools

import matplotlib.pyplot as plt
#%matplotlib inline

def generate_my_simplicial_complex_d2(N,p1,p2):

    """Our model"""

    #I first generate a standard ER graph with edges connected with probability p1
    G = nx.fast_gnp_random_graph(N, p1, seed=None)

    if not nx.is_connected(G):
        giant = list(nx.connected_components(G))[0]
        G = nx.subgraph(G, giant)
        print('not connected, but GC has order %i ans size %i'%(len(giant), G.size()))

    triangles_list = []
    G_copy = G.copy()

    #Now I run over all the possible combinations of three elements:
    for tri in combinations(list(G.nodes()),3):
        #And I create the triangle with probability p2
        if random.random() <= p2:
            #I close the triangle.
            triangles_list.append(tri)

            #Now I also need to add the new links to the graph created by the triangle
            G_copy.add_edge(tri[0], tri[1])
            G_copy.add_edge(tri[1], tri[2])
            G_copy.add_edge(tri[0], tri[2])

    G = G_copy

    #Creating a dictionary of neighbors
    node_neighbors_dict = {}
    for n in list(G.nodes()):
        node_neighbors_dict[n] = G[n].keys()

    #print len(triangles_list), 'triangles created. Size now is', G.size()

    #avg_n_triangles = 3.*len(triangles_list)/G.order()

    #return node_neighbors_dict, node_triangles_dict, avg_n_triangles
    #return node_neighbors_dict, triangles_list, avg_n_triangles
    return node_neighbors_dict, triangles_list

def get_p1_and_p2_correction(k1,k2,N):
    p2 = (2.*k2)/((N-1.)*(N-2.))
    p1 = (k1 - 2.*k2)/((N-1.)- 2.*k2)
    if (p1>=0) and (p2>=0):
        return p1, p2
    else:
        raise ValueError('Negative probability!')

def one_realization(ii):
    print('ITERATION %i'%ii)

    #Generating a simplicial complex with the given parameters
    node_neighbors_dict, triangles_list = generate_my_simplicial_complex_d2(N,p1,p2)

    return node_neighbors_dict, triangles_list

def parse_results(results):

    degree_list = []
    tri_degree_list = []

    for res in results:
        node_neighbors_dict, triangles_list = res

        #Adding the degree sequence of this realization to the global list
        degree_list += [len(neighs) for neighs in node_neighbors_dict.itervalues()]

        #creating a triangles degree sequence and adding it to the global list
        vertex_in_a_triangle = []
        for tri in triangles_list:
            vertex_in_a_triangle += tri

        tri_degree_list += Counter(vertex_in_a_triangle).values()

    degree_counter = Counter(degree_list)
    triangle_degree_counter = Counter(tri_degree_list)

    return degree_counter, triangle_degree_counter

#Simplicial Complex
N = 2000
k1 = 20
k2 = 6
p1, p2 = get_p1_and_p2_correction(k1,k2,N)

n_simulations = 80
n_processes = 8

#################################################

iteration_numbers = range(n_simulations)

pool = Pool(n_processes)
results = pool.map(one_realization, iteration_numbers)
degree_counter, triangle_degree_counter = parse_results(results)

fig = plt.figure(figsize=(4.5,3));
plt.rcParams['xtick.major.width'] = 1.2
plt.rcParams['ytick.major.width'] = 1.2
plt.rcParams['axes.linewidth'] = 1.2

ax = plt.subplot(111)

norm_k1 = 1.*np.array(degree_counter.values())/sum(degree_counter.values())
norm_k2 = 1.*np.array(triangle_degree_counter.values())/sum(triangle_degree_counter.values())

#################### plot pdf
ax.plot(degree_counter.keys(), norm_k1, 's-', label='$k_1$',
        clip_on=True, mfc='white');
ax.plot(triangle_degree_counter.keys(), norm_k2, 'o-', label='$k_2$',
        clip_on=True, mfc='white');

#################### real mean
sum_of_numbers = sum(number*count for number, count in degree_counter.iteritems())
count = sum(count for n, count in degree_counter.iteritems())
mean_k1 = float(sum_of_numbers) / count

sum_of_numbers = sum(number*count for number, count in triangle_degree_counter.iteritems())
count = sum(count for n, count in triangle_degree_counter.iteritems())
mean_k2 = float(sum_of_numbers) / count

ax.vlines(mean_k1,0,0.2, linewidth=1.5, linestyles='--',
          color=u'#1f77b4', label=r'$\langle k_1 \rangle$');
ax.vlines(mean_k2,0,0.2, linewidth=1.5, linestyles='--',
          color=u'#ff7f0e', label=r'$\langle k_2 \rangle$');

#################### expected mean
ax.vlines(k2,0,0.2, linewidth=1.5, linestyles='-', color='lightgray', zorder=1, label=r'$\approx$');
ax.vlines(k1,0,0.2, linewidth=1.5, linestyles='-', color='lightgray', zorder=1);

ax.tick_params(axis='both', which='major', labelsize=16)
ax.set_xlabel('Generalized degree, $k_\omega$', size=18)
ax.set_ylabel('$P(k_\omega)$', size=18)

ax.set_ylim(0,0.2);
ax.set_xlim(0,40);

plt.legend(fontsize=14, loc='upper right', handlelength=1, frameon=False, borderpad=0)

plt.tight_layout()

simplagion-master/Generalized degree distribution of the Random Simplicial Complex model.py


# -*- coding: utf-8 -*-
import os
import sys
import time
import pickle
import requests
import pandas as pd
from tqdm import tqdm
from urllib import response
from bs4 import BeautifulSoup

class Scraper:

    def __init__(self):
        self.base_url = "https://scholar.google.com/"

        dir_persistent = os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "..", "..", "data", "interim", "H5Index")
        if not os.path.exists(dir_persistent):
            os.makedirs(dir_persistent)
        self.path_persistent = os.path.join(
                dir_persistent, "h5_index_rankings.pkl")

    def scrape(self):
        if not self._load_rankings():
            print("H5-Index rankings not scraped yet. Scraping now...\n")
            categories = self._scrape_categories()
            subcategories = self._scrape_subcategories(categories)
            self.rankings = self._scrape_rankings(subcategories)
            print("Finished.\n")
            self._save_rankings()
        else:
            print("H5-Index rankings already persistent.\n")

        # Show some statistics
        print("Statistics")
        print("\tNumber categories: {}.".format(len(
                self.rankings.category.unique())))
        print("\tNumber subcategories: {}.".format(len(
                self.rankings.subcategory.unique())))
        print("\tNumber rankings: {}.\n".format(len(
                self.rankings.h5_index)))
        df = self.rankings.groupby("category")["subcategory"].agg(
                set).reset_index()
        df["len"] = df.subcategory.apply(lambda x: len(x))
        min_len = df["len"].min()
        max_len = df["len"].max()
        print("\tMean number of subcategories: {}.".format(df["len"].mean()))
        print("\tMin number of subcategories is {} for category {}.".format(
                min_len, list(df[df["len"] == min_len].category)[0]))
        print("\tMax number of subcategories is {} for category {}.\n".format(
                max_len, list(df[df["len"] == max_len].category)[0]))

    def _scrape_categories(self):
        print("Scraping categories...")
        url = self.base_url + "/citations?view_op=top_venues&hl=en"
        try:
            data = requests.get(url)
        except Exception as e:
            print(str(e))

        if data.status_code != 200:
            raise ConnectionError(
                    "Failed to open url: {} (status code: {}).".format(
                            url, response.status_code))

        soup = BeautifulSoup(data.text, 'lxml')
        categories = list()
        for item in soup.find_all(
                'a', attrs={"class": "gs_md_li",
                            "role": "menuitem", "tabindex": "-1"}):
            category = item.get_text()
            link = item.get("href")
            if len(category.split()) > 1:
                categories.append((category, self.base_url + link))
        categories_df = pd.DataFrame(categories, columns=["category", "url"])
        print("Scraped {} categories.\n".format(len(categories_df)))
        return categories_df

    def _scrape_subcategories(self, categories):
        print("Scraping subcategories...")
        subcategories = list()
        count_categories = len(categories)
        with tqdm(desc="Scraping subcategories: ",
                  total=count_categories) as pbar:
            for idx in range(count_categories):
                url = categories.url.iloc[idx]
                try:
                    data = requests.get(url)
                except Exception as e:
                    print(str(e))

                if data.status_code != 200:
                    raise ConnectionError(
                            "Failed to open url: {} (status code: {}).".format(
                                    url, response.status_code))

                soup = BeautifulSoup(data.text, "lxml")
                for item in soup.find_all(
                        'a', attrs={"class": "gs_md_li", "role": "menuitem",
                                    "tabindex": "-1"}):
                    subcategory = item.get_text()
                    link = item.get("href")
                    if len(subcategory.split()) > 1:
                        subcategories.append((
                                categories.category.iloc[idx],
                                subcategory,
                                self.base_url + link))
                time.sleep(5)
                pbar.update(1)
        subcategories_df = pd.DataFrame(
                subcategories, columns=["category", "subcategory", "url"])
        print("Scraped {} subcategories for {} categories.\n".format(
                len(subcategories_df), count_categories))
        return subcategories_df

    def _scrape_rankings(self, subcategories):
        print("Scraping rankings...")
        category = list()
        subcategory = list()
        publication = list()
        h5_index = list()
        h5_median = list()
        count_subcategories = len(subcategories)
        with tqdm(desc="Scraping rankings: ",
                  total=count_subcategories) as pbar:
            for idx in range(count_subcategories):
                url = subcategories.url.iloc[idx]
                try:
                    data = requests.get(url)
                except Exception as e:
                    print(str(e))
                if data.status_code != 200:
                    raise ConnectionError(
                            "Failed to open url: {} (status code: {}).".format(
                                    url, response.status_code))

                soup = BeautifulSoup(data.text, "lxml")
                for content in soup.body.find_all(
                        'table', attrs={'class' : 'gsc_mp_table'}):
                    for table in content.find_all('tr'):
                        i = 0
                        for infos in table.find_all("td"):
                            if i == 1:
                                publication.append(infos.get_text())
                            if i == 2:
                                h5_index.append(infos.get_text())
                            if i == 3:
                                h5_median.append(infos.get_text())
                            i += 1

                category.extend([subcategories.category.iloc[idx]] * 20)
                subcategory.extend(
                        [subcategories.subcategory.iloc[idx]] * 20)
                time.sleep(5)
                pbar.update(1)

        rankings = pd.DataFrame({
                "category": category, "subcategory": subcategory,
                "publication": publication, "h5_index": h5_index,
                "h5_median": h5_median})
        print("Finished scraping {} rankings for {} subcategories and {} categories.\n".format(
                len(rankings.h5_index), len(rankings.category.unique()),
                len(rankings.subcategory.unique())))
        return rankings

    def _save_rankings(self):
        print("Saving rankings to disk...")
        with open(self.path_persistent, "wb") as f:
            pickle.dump(self.rankings, f)
        print("Saved.\n")

    def _load_rankings(self):
        if os.path.isfile(self.path_persistent):
            print("Loading H5-index rankings...")
            with open(self.path_persistent, "rb") as f:
                self.rankings = pickle.load(f)
            print("Loaded.\n")
            return True
        return False

    def main():
        print("Starting...\n")
        from H5IndexScraper import Scraper
        scraper = Scraper()
        scraper.scrape()
        print("Finished.")

    if __name__ == "__main__":
        main()


import sys
from collections import deque
input = sys.stdin.readline

def bfs(i, j):
    visited[i][j] = 1
    union = [(i, j)]
    _sum = graph[i][j]
    q.append((i, j))
    while q:
        x, y = q.popleft()
        for dx, dy in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
            nx = x + dx
            ny = y + dy
            if 0 <= nx < n and 0 <= ny < n and l <= abs(graph[x][y] - graph[nx][ny]) <= r and not visited[nx][ny]:
                visited[nx][ny] = 1
                q.append((nx, ny))
                union.append((nx, ny))
                _sum += graph[nx][ny]

    if len(union) > 1:
        avg = _sum // len(union)
        for x, y in union:
            graph[x][y] = avg
            _next.append((x, y))
        return True

    return False

n, l, r = map(int, input().split())
graph = [list(map(int, input().split())) for _ in range(n)]
visited = []
_next = deque()
q = deque()
cnt = 0

for i in range(n):
    for j in range(n):
        _next.append((i, j))

while True:
    is_moved = False
    visited = [[0] * n for _ in range(n)]

    for _ in range(len(_next)):
        i, j = _next.popleft()
        if not visited[i][j]:
            if bfs(i, j): is_moved = True

    if not is_moved:
        print(cnt)
        break

    cnt += 1


# Create a Scraper that extracts information about job descriptions
# 1. Open up website
# 2. Parse the HTML and gather content objects from the indeed page
#    - list of Job Titles
#    - list of Job Descriptions

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import json
import time

with open("aggregate_job_info.json", "r") as f:
    aggregate_job_info = json.load(f)

try:
    driver_path = r"chromedriver.exe"  # Replace with your Chrome driver path
    service = Service(driver_path)
    driver = webdriver.Chrome(service=service)

    driver.get(r"https://www.indeed.com/jobs?q=computer+science+internship&start=90&pp=gQAPAAAAAAAAAAAAAAACFjq7MgAqAQAIDFM_StAtKUnkYv0UY6MkuMvhXH8rBs1KaUeq58G4VwL8sAAcULK8AAA&vjk=6cb6793d38e2cf21")

    # aggregate_job_info = []

    running = True
    while running:
        time.sleep(1)

        link_elements = driver.find_elements(By.CLASS_NAME, "jcs-JobTitle.css-jspxzf.eu4oa1w0")

        for link_element in link_elements:

            job_info = {}
            req_list = []

            link_element.click()
            time.sleep(0.5)

            job_title_element = driver.find_element(By.CLASS_NAME, "jobsearch-JobInfoHeader-title-container")
            # job_title_element = driver.find_element(By.CLASS_NAME, "icl-u-xs-mb--xs.icl-u-xs-mt--none.jobsearch-JobInfoHeader-title.is-embedded")
            job_info["title"] = job_title_element.text

            job_description_element = driver.find_element(By.ID, "jobDescriptionText")
            unordered_lists = job_description_element.find_elements(By.TAG_NAME, "ul")

            for unordered_list in unordered_lists:
                list_items = unordered_list.find_elements(By.TAG_NAME, "li")
                for list_item in list_items:
                    print(list_item.text)
                    req_list.append(list_item.text)
            job_info["reqs"] = req_list
            job_info["link"] = driver.current_url
            aggregate_job_info.append(job_info)

        next_buttons = driver.find_elements(By.CLASS_NAME, "e8ju0x50")
        if(len(next_buttons) == 6):
            running = False
        else:
            next_buttons[len(next_buttons)-2].click()
except Exception as e:
    print(f"An exception occurred: {e}")

if 'driver' in locals():
    driver.quit()

with open("aggregate_job_info.json", "w") as f:
    json.dump(aggregate_job_info, f)

driver.quit()