import os
import shutil
from pathlib import Path

import pandas as pd
import streamlit as st

from dataset_splitting_by_language import main as split_dataset_by_language

@st.cache_data
def convert_to_csv(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_csv().encode('utf_8_sig')

@st.cache_data
def convert_to_jsonl(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_json(orient='records', lines=True,
                      force_ascii=False).encode('utf_8_sig')

def split_dataset(dataset_file, target_dir):

    del_file = False

    if dataset_file is not None:
        file_contents = dataset_file.getvalue()
        with open(dataset_file.name, 'wb') as f:
            f.write(file_contents)
        dataset_path = dataset_file.name
        del_file = True
    else:
        dataset_path = st.session_state.get('default_demo_dataset')
    split_dataset_by_language(dataset_path, target_dir)

    if del_file:
        os.remove(dataset_file.name)

class Visualize:

    @staticmethod
    def setup():
        st.set_page_config(
            page_title='Data-Juicer',
            page_icon=':smile',
            layout='wide',
            # initial_sidebar_state="expanded",
        )

        readme_link = 'https://github.com/alibaba/data-juicer'
        st.markdown(
            '
  Data-Juicer \

',
            unsafe_allow_html=True,
        )
        st.markdown(
            f'
 A One-Stop Data Processing System for \
                Large Language Models, \
                see more details in our Github
',
            unsafe_allow_html=True,
        )

    @staticmethod
    def split():
        col1, col2 = st.columns(2)
        with col1:
            dataset_file = st.file_uploader(
                label='Upload your custom dataset(jsonl/parquet)',
                type=['json', 'jsonl', 'parquet'])

        with col2:
            st.text_input(label='Default Demo dataset',
                          disabled=True,
                          key='default_demo_dataset',
                          value='data/demo-dataset.jsonl')

        start_btn = st.button('Start to split dataset by language ',
                              use_container_width=True)

        with st.expander('Splitted Results', expanded=True):

            target_dir = 'outputs'
            if start_btn:
                with st.spinner('Wait for splitting...'):
                    shutil.rmtree(target_dir, ignore_errors=True)
                    split_dataset(dataset_file, target_dir)

            splitted_datasets = Path(target_dir).glob('*.jsonl')
            datasets_dict = {
                ds_file.stem: ds_file
                for ds_file in splitted_datasets
            }
            count = len(datasets_dict)
            if count > 1:
                st.header(f'There are :red[{len(datasets_dict)}] languages:\
                         {[ lang for lang in datasets_dict.keys()]}')
            elif count == 1:
                st.header(f'There is :red[{len(datasets_dict)}] language: \
                        {[ lang for lang in datasets_dict.keys()]}')

            for k, v in datasets_dict.items():
                df = pd.read_json(str(v), lines=True)
                count = len(df.index)
                be = 'is' if count <= 1 else 'are'
                samples = 'sample ' if count <= 1 else 'samples'
                st.subheader(
                    f'There {be} :red[ {count} ] {samples} in :red[{k}]')
                st.dataframe(df, use_container_width=True)
                st.download_button(f'Download {k} as JSONL',
                                   data=convert_to_jsonl(df),
                                   file_name=v.name)

            shutil.rmtree(target_dir, ignore_errors=True)

    @staticmethod
    def visualize():
        Visualize.setup()
        Visualize.split()

def main():
    Visualize.visualize()

if __name__ == '__main__':
    main()


#!/usr/bin/python

import urllib
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime
import os.path
import time

urls = ["http://www.bbc.co.uk/sport/football/scottish-championship/table?print=true",
        "http://www.bbc.co.uk/sport/football/scottish-league-one/table?print=true",
        "http://www.bbc.co.uk/sport/football/scottish-league-two/table?print=true"]

def getTables():
    date = str(datetime.now().strftime("%Y%m%d-%H%M%S"))
    current_url = 0

    for url in urls:
        html = urllib.urlopen(url)
        soup = BeautifulSoup(html, "html.parser")

        column_headers = [th.getText() for th in soup.findAll('tr', limit=2)[0].findAll('th')]

        data_rows = soup.findAll('tr')[2:12]
        team_data = [[td.getText() for td in data_rows[i].findAll('td')] for i in range(len(data_rows))]

        df = pd.DataFrame(team_data, columns=column_headers)

        # Drop unnecessary columns
        df = df.drop('Last 10 games results', 1)
        df = df.drop('Match status', 1)
        df = df.drop(df.columns[[0]], axis=1)

        # Replace unnecessary strings with simple integers
        for x in range(0,10):
            df['Position'][x] = x + 1

        # Pick a filename depending on the URL we're scraping
        if current_url == 0:
            filename = "/home/richusx/graded-unit/app/tables/archive/championship_%s.html" % (date)
        elif current_url == 1:
            filename = "/home/richusx/graded-unit/app/tables/archive/league-one_%s.html" % (date)
        elif current_url == 2:
            filename = "/home/richusx/graded-unit/app/tables/archive/league-two_%s.html" % (date)

        # Return HTML table
        df.to_html(filename, index=False)

        if current_url == 0:
            filename = "/home/richusx/graded-unit/app/tables/championship.html"
        elif current_url == 1:
            filename = "/home/richusx/graded-unit/app/tables/league-one.html"
        elif current_url == 2:
            filename = "/home/richusx/graded-unit/app/tables/league-two.html"

        if os.path.exists(filename):
            os.remove(filename)

        df.to_html(filename, index=False, classes='table table-striped table-bordered')
        #print ""
        #print df.head(10)
        current_url += 1

def mergeHTML():
    timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

    # Make sure the file doesn't already exist
    if os.path.exists('/home/richusx/graded-unit/app/templates/tables.html'):
        os.remove('/home/richusx/graded-unit/app/templates/tables.html')

    # Merge all 3 HTML files into one
    tm = open('/home/richusx/graded-unit/app/templates/tables.html', 'a')
    t1 = open('/home/richusx/graded-unit/app/tables/championship.html')
    t2 = open('/home/richusx/graded-unit/app/tables/league-one.html')
    t3 = open('/home/richusx/graded-unit/app/tables/league-two.html')

    tm.write('{% extends "base.html" %}\n{% block content %}')

    tm.write("

Championship League
")
    for line in t1.readlines():
        tm.write(line)
    t1.close()
    tm.write("

League One
")
    for line in t2.readlines():
        tm.write(line)
    t2.close()
    tm.write("

League Two
")
    for line in t3.readlines():
        tm.write(line)
    t3.close()
    tm.write("
Last Updated: %s
Tables get updated automatically every 2 hours\n{%% endblock %%}" % (timestamp))
    tm.close()

if __name__ == "__main__":
    getTables()
    mergeHTML()


from pyspark.sql import SparkSession

# Suppose you used the upload script to upload your file
# named "data.csv" to hdfs in a directory called "input"
# placed at the root of the shared file system
hdfs_path = "hdfs://namenode:9000/input/data.csv"

# initialize a spark session
spark = SparkSession.builder\
    .appName("example-app")\
    .getOrCreate()

# read data from hdfs
data = spark.read.text(hdfs_path).cache()

# print the data
print(data.collect())

# stop the session
spark.stop()


import pandas as pd
import os
from config import *
import matplotlib.pyplot as plt
import numpy as np
from pydts.data_generation import EventTimesSampler

final_dfs = []
n_patients_list = [250, 500]
for n_patients in n_patients_list:
    case = f'n{n_patients}_beta_results_comparison.csv'

    df = pd.read_csv(os.path.join(OUTPUT_DIR, case))
    df.index = [r'$\beta_{11}$', r'$\beta_{12}$', r'$\beta_{13}$', r'$\beta_{14}$', r'$\beta_{15}$',
                 r'$\beta_{21}$', r'$\beta_{22}$', r'$\beta_{23}$', r'$\beta_{24}$', r'$\beta_{25}$']

    df.columns = pd.MultiIndex.from_tuples([('', 'True'),
                                            ('Lee et al.', 'Estimate'),
                                            ('Lee et al.', 'Estimated SE'),
                                            ('two-step', 'Estimate'),
                                            ('two-step', 'Estimated SE'),
                                            ('two-step', 'Empirical SE'),
                                            ('two-step', 'Coverage Rate')])

    final_dfs.append(df.astype(float).round(3))

final_df = pd.concat(final_dfs, keys=n_patients_list)
print(final_df.to_latex(escape=False))

filename = 'alpha_small_dataset.png'

d_times = 7
n_cov = 5
j_events = 2

real_coef_dict = {
    "alpha": {
        1: lambda t: -1.4 + 0.4 * np.log(t),
        2: lambda t: -1.3 + 0.4 * np.log(t)
    },
    "beta": {
        1: -0.7*np.log([0.8, 3, 3, 2.5, 2]),
        2: -0.6*np.log([1, 3, 4, 3, 2])
    }
}

ets = EventTimesSampler(d_times=d_times, j_event_types=j_events)
covariates = [f'Z{i + 1}' for i in range(n_cov)]

first_model_name = 'Lee et al.'
second_model_name = 'two-step'
times = range(1, d_times + 1)

lee_colors = ['tab:blue', 'tab:green']
two_step_colors = ['navy', 'darkgreen']
true_colors = ['tab:blue', 'tab:green']

fig, axes = plt.subplots(1, 2, figsize=(12, 5))

for idn, n_patients in enumerate(n_patients_list):
    case = f'n{n_patients}_results_comparison.csv'

    np.random.seed(idn)
    patients_df = pd.DataFrame(data=pd.DataFrame(data=np.random.uniform(0, 1, size=[n_patients, n_cov]),
                                                 columns=covariates))

    patients_df = ets.sample_event_times(patients_df, hazard_coefs=real_coef_dict, seed=idn)
    patients_df = ets.sample_independent_lof_censoring(patients_df,
                                                       prob_lof_at_t=0.02 * np.ones(d_times))
    patients_df = ets.update_event_or_lof(patients_df)
    patients_df.index.name = 'pid'
    patients_df = patients_df.reset_index()
    counts = patients_df.groupby(['J', 'X'])['pid'].count().unstack('J').fillna(0)

    alpha_res_df = pd.read_csv(os.path.join(OUTPUT_DIR, case))

    ax = axes[int(idn % 2)]
    ax.set_title(f'n={n_patients}', fontsize=15)
    ax.tick_params(axis='both', which='major', labelsize=15)
    ax.tick_params(axis='both', which='minor', labelsize=15)

    for j in [1, 2]:
        tmp_lee_alpha = alpha_res_df.iloc[((d_times+n_cov)*(j-1)):((d_times+n_cov)*(j-1)+d_times), 1]
        ax.scatter(times, tmp_lee_alpha.values,
                   label=f'J={j} ({first_model_name})', color=lee_colors[j - 1], marker='o', alpha=0.4, s=40)

        tmp_two_step_alpha = alpha_res_df.iloc[((d_times+n_cov)*(j-1)):((d_times+n_cov)*(j-1)+d_times), 3]
        ax.scatter(times, tmp_two_step_alpha.values,
                   label=f'J={j} ({second_model_name})', color=two_step_colors[j - 1], marker='*', alpha=0.7, s=20)

        true_values = [real_coef_dict['alpha'][j](t) for t in times]
        ax.plot(times, true_values, label=f'J={j} (True)', ls='--', color=true_colors[j - 1])

        ax.set_xlabel(r'Time', fontsize=18)
        ax.set_ylabel(r'$\alpha_{jt}$', fontsize=18)
        ax.legend(loc='upper right', fontsize=12)
        ax.set_ylim([-2.2, 1])
        ax.set_xlim([0, d_times+1])

    ax2 = ax.twinx()
    ax2.bar(counts.index, counts[1].values.squeeze(), label='J=1', color='tab:red', alpha=0.4, width=0.5)
    ax2.bar(counts.index, counts[2].values.squeeze(), label='J=2', color='tab:brown', alpha=0.6, align='edge',
            width=0.5)
    ax2.legend(loc='upper left', fontsize=12)
    ax2.set_ylabel('Number of observed events', fontsize=16, color='red')
    ax2.tick_params(axis='y', colors='red')
    ax2.set_ylim([0, 150])
    ax2.tick_params(axis='both', which='major', labelsize=15)
    ax2.tick_params(axis='both', which='minor', labelsize=15)
fig.tight_layout()

if filename is not None:
    fig.savefig(os.path.join(OUTPUT_DIR, filename), dpi=300)


#!/usr/bin/env python
import sys

import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot
from pandas.plotting import scatter_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.naive_bayes import ComplementNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from common import classification_metrics, describe_data, test_env

def read_data(file):
    """Return pandas dataFrame read from csv file"""
    try:
        return pd.read_excel(file)
    except FileNotFoundError:
        sys.exit('ERROR: ' + file + ' not found')

def encode_categorical(df, categorical_columns):
    for column in categorical_columns:
        df[column] = df[column].fillna(value='Missing')
        df = pd.get_dummies(df, prefix=[column], columns=[column],
                            drop_first=True)

    return df

def preprocess_data(df, verbose=False):
    y_column = 'In university after 4 semesters'

    # Features can be excluded by adding column name to list
    drop_columns = []

    categorical_columns = [
        'Faculty',
        'Paid tuition',
        'Study load',
        'Previous school level',
        'Previous school study language',
        'Recognition',
        'Study language',
        'Foreign student'
    ]

    # Handle dependent variable
    if verbose:
        print('Missing y values: ', df[y_column].isna().sum())

    y = df[y_column].values
    # Encode y. Naive solution
    y = np.where(y == 'No', 0, y)
    y = np.where(y == 'Yes', 1, y)
    y = y.astype(float)

    # Drop also dependent variable variable column to leave only features
    drop_columns.append(y_column)
    df = df.drop(labels=drop_columns, axis=1)

    # Remove drop columns for categorical columns just in case
    categorical_columns = [
        i for i in categorical_columns if i not in drop_columns]

    # encode categorical features
    df = encode_categorical(df, categorical_columns)

    # Handle missing data. At this point only exam points should be missing
    # It seems to be easier to fill whole data frame as only particular columns
    if verbose:
        describe_data.print_nan_counts(df)

    # handle missing values
    df.fillna(value=0, inplace=True)

    if verbose:
        describe_data.print_nan_counts(df)

    # Return features data frame and dependent variable
    return df, y

# function implementing logistic regression on dataset
def logistic_regression(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    sc = StandardScaler(copy=True, with_mean=True, with_std=True)
    X_train = sc.fit_transform(X_train)
    X_test = sc.transform(X_test)

    clf = LogisticRegression(solver='sag', random_state=0, penalty='l2',
                             max_iter=1000, multi_class='multinomial')
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'Logistic regresssion test data', verbose=verbose)

    return clf, sc

# function implementing KNN on dataset
def k_nn(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    sc = StandardScaler()
    X_train = sc.fit_transform(X_train)
    X_test = sc.transform(X_test)

    clf = KNeighborsClassifier(n_neighbors=10, metric='minkowski', p=2)
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'KNN test data', verbose=verbose)

    return clf, sc

# function implementing SVM on dataset
def svm_clf(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    sc = StandardScaler()
    X_train = sc.fit_transform(X_train)
    X_test = sc.transform(X_test)

    clf = SVC(kernel='sigmoid', C=1.0, gamma=0.2,
              probability=True, tol=1e-6, random_state=0)
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'SCV test data',
                                         verbose=verbose)
    return clf, sc

# function implementing naive bayes on dataset
def naive_bayes_clf(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    clf = ComplementNB()
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'Naive bayes test data',
                                         verbose=verbose)

    return clf

# function implementing decision tree on dataset
def decision_tree_clf(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    clf = DecisionTreeClassifier(random_state=0)
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'Decision tree test data',
                                         verbose=verbose)

    return clf

# function implementing random forest on dataset
def random_forest_clf(X_df, y, verbose=False):
    X = X_df.values
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=0)

    clf = RandomForestClassifier(n_estimators=100, random_state=0)
    clf.fit(X_train, y_train)

    classification_metrics.print_metrics(y_test, clf.predict(X_test),
                                         'Random forest classifier test data',
                                         verbose=verbose)

    return clf

if __name__ == '__main__':
    modules = ['numpy', 'pandas', 'sklearn']
    test_env.versions(modules)

    students = read_data('data/students.xlsx')

    # show histograms
    students.hist()
    pyplot.show()

    # scatter Matrix Plot
    sm = scatter_matrix(students, alpha=0.2, figsize=(4, 4), diagonal='kde')

    # Change label rotation
    [s.xaxis.label.set_rotation(-20) for s in sm.reshape(-1)]
    [s.yaxis.label.set_rotation(0) for s in sm.reshape(-1)]

    # May need to offset label when rotating to prevent overlap of figure
    [s.get_yaxis().set_label_coords(-0.3, 0.5) for s in sm.reshape(-1)]

    # Hide all ticks
    [s.set_xticks(()) for s in sm.reshape(-1)]
    [s.set_yticks(()) for s in sm.reshape(-1)]
    # show scatter matrix
    pyplot.show()

    # Correlation Matrix Heatmap
    f, ax = pyplot.subplots()
    corr = students.corr()
    hm = sns.heatmap(corr, annot=True, cmap="coolwarm", xticklabels="", yticklabels="")
    t = f.suptitle('Students scores Correlation Heatmap', fontsize=14)
    # show heat map
    pyplot.show()

    # PRINT_OVERVIEW AND PRINT_CATEGORICAL FUNCTIONS
    # FILE NAME AS ARGUMENT
    describe_data.print_overview(students, file='results/students_overview.txt')
    describe_data.print_categorical(students, file='results/students_categorical_features.txt')

    students_X, students_y = preprocess_data(students)

    X_train, X_test, y_train, y_test = train_test_split(
        students_X, students_y, test_size=0.25, random_state=0)

    # CLASSIFIERS FUNCTIONS
    verbose = False

    log_reg_clf, log_reg_sc = logistic_regression(
        students_X, students_y, verbose=verbose)
    k_nn_clf, k_nn_sc = k_nn(students_X, students_y, verbose=verbose)
    svm_clf = svm_clf(students_X, students_y, verbose=verbose)
    nb_clf = naive_bayes_clf(students_X, students_y, verbose=verbose)
    dt_clf = decision_tree_clf(students_X, students_y, verbose=verbose)
    rf_clf = random_forest_clf(students_X, students_y, verbose=verbose)

    # Spot Check Algorithms
    models = [('LR', LogisticRegression(solver='sag', random_state=0, penalty='l2',
                                        max_iter=5000, multi_class='multinomial')),
              ('KNN', KNeighborsClassifier(n_neighbors=10, metric='minkowski', p=2)),
              ('SVM', SVC(kernel='sigmoid', C=1.0, gamma=0.2,
                          probability=True, tol=1e-6, random_state=0)),
              ('NB', ComplementNB()),
              ('CART', DecisionTreeClassifier(random_state=0)),
              ('RD', RandomForestClassifier(n_estimators=100, random_state=0))]

    # evaluate each model in turn
    results = []
    names = []
    for name, model in models:
        kfold = StratifiedKFold(n_splits=10, random_state=1, shuffle=True)
        cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring='accuracy')
        results.append(cv_results)
        names.append(name)
        print('%s: %f (%f)' % (name, cv_results.mean(), cv_results.std()))

    print('Done')


import logging
import socket
import threading
import argparse
import pickle

from rich.console import Console
from rich.logging import RichHandler

from hmg.word_utils import get_word_from_internet

FORMAT_LOGGING = "%(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT_LOGGING, handlers=[RichHandler()])

logger = logging.getLogger("hmg-server")

MAX_PLAYERS = 2

console = Console()

class Server:
    def __init__(self, host, port, secret_word=None):
        self.host = host
        self.port = port
        self.secret_word = secret_word
        self._using_word_arg = secret_word is not None
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._players = {}

    def _update_secret_word(self):
        with console.status("[blue]Searching for word, wait..."):
            self.secret_word = get_word_from_internet()
        logger.info("Secret word=%s", self.secret_word)

    def serve_forever(self):
        self._socket.bind((self.host, self.port))
        self._socket.listen(2)
        logger.info("Server ready and listening on %s:%s", self.host, self.port)
        self._listen_connection()

    def _listen_connection(self):
        is_listening = True
        while is_listening:
            if len(self._players) == MAX_PLAYERS:
                if not self._using_word_arg:
                    self._update_secret_word()

                for _, sock in self._players.items():
                    sock.send(pickle.dumps({"msg": self.secret_word}))

                player1, player2 = self._players.values()
                payload = {"turn": False, "win": False, "msg": ""}
                player1.send(pickle.dumps(payload))
                payload = {"turn": True, "win": False, "msg": ""}
                player2.send(pickle.dumps(payload))

            conn, addr = self._socket.accept()
            incoming_message = conn.recv(1024)
            if not incoming_message:
                continue
            data = pickle.loads(incoming_message)
            username = data["user"]

            if username in self._players.keys():
                username += "_x"

            logger.info("Username=%s with address=%s connected", username, addr)
            self._players[username] = conn

            client_thread = threading.Thread(
                target=self._handle_client, args=(conn, addr, username)
            )
            client_thread.start()

    def _handle_client(self, client, addr, username):
        while True:
            logger.info("Waiting for client message...")
            msg_client = client.recv(1024)
            if not msg_client:
                logger.warning("Client=%s disconnected", username)
                del self._players[username]
                break

            data = pickle.loads(msg_client)
            logger.info("Sending data=%s", data)

            for user, sock in self._players.items():
                if user != username:
                    data["turn"] = True
                else:
                    data["turn"] = False
                sock.send(pickle.dumps(data))

def get_cli_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("-H", "--host", default=socket.gethostname())
    parser.add_argument("-p", "--port", type=int, default=8889)
    parser.add_argument(
        "-w",
        "--word",
        default=None,
        help="Word to use instead of internet (usefull for testing)",
    )

    return parser

def main():
    args = get_cli_parser().parse_args()

    server = Server(host=args.host, port=args.port, secret_word=args.word)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        logger.info("Exiting...")

if __name__ == "__main__":
    main()


import sqlite3
import json
from tqdm import tqdm

class DatabaseConnection:
    def __init__(self, db_file, topic_json, heading_json, alltree_json):
        self.db_file = db_file
        self.connection = None
        self.topic_json = json.loads(open(topic_json, "r").read())
        self.heading_json = json.loads(open(heading_json, "r").read())
        self.alltree_json = json.loads(open(alltree_json, "r").read())
        self.connect()

    def connect(self):
        try:
            self.connection = sqlite3.connect(self.db_file)
            print("Connected to the database")
        except sqlite3.Error as e:
            print(f"Error connecting to the database: {e}")

    def disconnect(self):
        if self.connection:
            self.connection.close()
            print("Disconnected from the database")
        else:
            print("No active connection to disconnect")

    def __table(self):
        cursor = self.connection.cursor()
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
        print(cursor.fetchall())

    def insert_topic(self, table_name, json_topic):
        cursor = self.connection.cursor()
        for index, data_topic in enumerate(json_topic):
            cursor.execute(
                f"INSERT INTO {table_name} VALUES ('{data_topic['Value']}', '{data_topic['Text']}');")
        self.connection.commit()

    def insert_heading(self, table_name, json_heading):
        cursor = self.connection.cursor()
        for index, value in enumerate(json_heading):
            id_topic = value['ChuDe']
            id_heading = value['Value']
            rank = value['STT']
            data = value['Text']

            # get topic from id_topic
            cursor.execute(
                "SELECT * FROM app_topic WHERE topic_id=?", (id_topic,))
            topic = cursor.fetchone()

            # insert heading
            if topic:
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?);", (id_heading, data, rank, topic[0]))
            else:
                print(f"Topic with id {id_topic} not found.")

        self.connection.commit()

    def insert_article(self, table_name, json_article):
        cursor = self.connection.cursor()
        for index, value in enumerate(json_article):
            id_article = value['ID']
            rank = value['ChiMuc']
            mapc = value['MAPC']
            name_article = value['TEN']
            id_heading = value['DeMucID']

            # Retrieve heading object
            cursor.execute(
                "SELECT * FROM app_heading WHERE heading_id=?", (id_heading,))
            heading = cursor.fetchone()

            if heading:
                # Instead of passing the entire heading object, pass its primary key
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?, ?);",
                    (id_article, rank, mapc, name_article, heading[0]),
                )
            else:
                print(f"Heading with id {id_heading} not found.")

        self.connection.commit()

    def article(self, table_name, json_article, len_MAPC=20):
        print(f"INSERT DATA {len_MAPC}")
        cursor = self.connection.cursor()
        json_data = dict()
        for index, value in enumerate(json_article):
            if len(value["MAPC"]) not in json_data.keys():
                json_data[len(value["MAPC"])] = []
            json_data[len(value["MAPC"])].append(value)

        json_data = json_data[len_MAPC]
        for index, value in tqdm(enumerate(json_data)):
            id_article = value['ID']
            rank = value['ChiMuc']
            mapc = value['MAPC']
            name_article = value['TEN']
            id_topic = value['ChuDeID']
            id_heading = value['DeMucID']

            cursor.execute(
                "SELECT * FROM app_topic WHERE topic_id=?", (id_topic,))
            topic = cursor.fetchone()
            cursor.execute(
                "SELECT * FROM app_heading WHERE heading_id=?", (id_heading,))
            heading = cursor.fetchone()

            if topic and heading:

                cursor.execute(
                    f"SELECT * FROM {table_name} WHERE mapc = ?", (mapc[:-20],))
                mapc_parent = cursor.fetchone()
                if mapc_parent is not None:
                    mapc_parent = mapc_parent[0]
                cursor.execute(
                    f"INSERT INTO {table_name} VALUES (?, ?, ?, ?, ?, ?, ?);",
                    (id_article, rank, mapc, name_article,
                     heading[0], mapc_parent, topic[0]),
                )
        self.connection.commit()

database = DatabaseConnection("db.sqlite3",
                              "static/JSON/topics_json.json",
                              "static/JSON/headings_json.json",
                              "static/JSON/allTree_json.json")

database.insert_topic("app_topic", database.topic_json)
database.insert_heading("app_heading", database.heading_json)
database.article("app_article", database.alltree_json, 20)
database.article("app_article", database.alltree_json, 40)
database.article("app_article", database.alltree_json, 60)
database.article("app_article", database.alltree_json, 80)
database.article("app_article", database.alltree_json, 100)
database.article("app_article", database.alltree_json, 59)
database.article("app_article", database.alltree_json, 79)


import wx, wx.aui, wx.lib.hyperlink
import os
import utils

RELEASE_ARTIST_WIDTH=150
RELEASE_RELEASE_WIDTH=100
RELEASE_DATE_WIDTH=100
RELEASE_FONT_SIZE=8

RELATED_ARTIST_WIDTH=150

ALLOW_AUI_FLOATING = False
LEFT_WIDTH = 160

class ReleasePage( wx.Panel ):
    def __init__( self, parent, id, width, main ):

        wx.Panel.__init__( self, parent, id )

        self.main = main

        self.__uiInit( parent,id,width )

    def __uiInit( self, parent, id, width ):
        self.artistWidth = LEFT_WIDTH
        self.leftWidth = LEFT_WIDTH
        self.rightWidth = width - LEFT_WIDTH - 7
        self.sizer = wx.BoxSizer( wx.VERTICAL )

        self.lrSplitter = wx.SplitterWindow( self, wx.ID_ANY,
                style = wx.SP_LIVE_UPDATE, size=(-1, 480))
        self.lrSplitter.SetMinimumPaneSize( 160 )
        self.lrSplitter.Bind( wx.EVT_SPLITTER_SASH_POS_CHANGING,
                self.OnVertSash )
        self.lrSplitter.Bind( wx.EVT_SPLITTER_SASH_POS_CHANGED,
                self.OnVertSashEnd )
        self.tbSplitter = wx.SplitterWindow( self.lrSplitter, wx.ID_ANY,
                style = wx.SP_LIVE_UPDATE, size=(-1, 480))
        self.tbSplitter.SetMinimumPaneSize( 60 )

        # Create the  left panel with an artist list and a search box
        leftPanel = wx.Panel( self.lrSplitter, wx.ID_ANY,
                style = wx.CLIP_CHILDREN )
        self.searchItems = {}

        self.artistList = wx.ListCtrl( leftPanel, wx.ID_ANY, wx.DefaultPosition,
                                       wx.DefaultSize,
                                       wx.LC_REPORT|wx.LC_SINGLE_SEL )
        self.artistList.Bind( wx.EVT_LIST_COL_END_DRAG, self.OnArtistColDrag,
                self.artistList )

        self.filter = wx.SearchCtrl(leftPanel, style=wx.TE_PROCESS_ENTER)
        self.filter.ShowCancelButton(True)
        self.filter.Bind(wx.EVT_TEXT, self.RecreateArtists)
        self.filter.Bind(wx.EVT_SEARCHCTRL_CANCEL_BTN, self.OnSearchCancelBtn)
        self.filter.Bind(wx.EVT_TEXT_ENTER, self.OnSearch)

        searchMenu = wx.Menu()
        item = searchMenu.AppendRadioItem(-1, "Artist")
        self.Bind(wx.EVT_MENU, self.OnSearchMenu, item)
        item = searchMenu.AppendRadioItem(-1, "Year")
        self.Bind(wx.EVT_MENU, self.OnSearchMenu, item)
        self.filter.SetMenu(searchMenu)

        self.RecreateArtists()
        self.artistList.Bind( wx.EVT_LIST_ITEM_SELECTED, self.OnItemSelected)

        # Create the top pane with the list of releases
        topPanel = wx.Panel( self.tbSplitter, style=wx.CLIP_CHILDREN )
        self.releases = wx.ListCtrl( topPanel, wx.ID_ANY,
                style=wx.LC_REPORT|wx.LC_SINGLE_SEL )
        self.RecreateReleases()

        topBox = wx.BoxSizer( wx.VERTICAL )
        topBox.Add( self.releases, 1, wx.EXPAND )
        topPanel.SetSizer( topBox )
        topPanel.SetAutoLayout( True )

        # Create static text placeholder
        bottomPanel = wx.Panel( self.tbSplitter, style=wx.CLIP_CHILDREN )
        #self.placeholder = wx.StaticText( bottomPanel, wx.ID_ANY,
            #"\n\n Placeholder \n\n" )
        self.link = wx.lib.hyperlink.HyperLinkCtrl( parent=bottomPanel )
        self.whatLink = wx.lib.hyperlink.HyperLinkCtrl( parent=bottomPanel )

        bottomBox = wx.BoxSizer( wx.VERTICAL )
        #bottomBox.Add( self.placeholder, 1, wx.EXPAND )
        bottomBox.Add( self.link, 1, wx.EXPAND )
        bottomBox.Add( self.whatLink, 1, wx.EXPAND )
        bottomPanel.SetSizer( bottomBox )
        bottomPanel.SetAutoLayout( True )

        # Create top/bottom splitter

        self.tbSplitter.SplitHorizontally( topPanel, bottomPanel )

        # Create the left pane
        leftBox = wx.BoxSizer( wx.VERTICAL )
        leftBox.Add( self.artistList, 1, wx.EXPAND )
        leftBox.Add( wx.StaticText( leftPanel, label="Filter artists:" ), 0,
                wx.TOP|wx.LEFT, 5 )
        leftBox.Add( self.filter, 0, wx.EXPAND|wx.ALL, 5 )
        leftPanel.SetSizer( leftBox )
        leftPanel.SetAutoLayout( True )

        # Split left / right
        self.lrSplitter.SplitVertically( leftPanel, self.tbSplitter )

        self.sizer.Add( self.lrSplitter, 1, wx.EXPAND )
        self.SetSizer( self.sizer )
        self.SetAutoLayout( True )
        self.sizer.Fit( self )
        self.Show( True )

    def OnVertSash( self, event ):
        if self.artistList.GetColumnWidth( 0 ) < event.GetSashPosition():
            self.artistList.SetColumnWidth( 0, event.GetSashPosition() )
            self.artistWidth = event.GetSashPosition()

    def OnVertSashEnd( self, event ):
        self.Refresh()

    def OnSearchCancelBtn( self, event ):
        self.filter.SetValue('')
        self.OnSearch()
        self.RecreateReleases()

    def OnSearch( self, item=None ):
        self.RecreateArtists()

    def OnSearchMenu( self, item ):
        x = 1

    def OnItemSelected( self, event ):
        self.RecreateReleases( event )

    def RecreateArtists( self, event=None ):
        self.artistList.DeleteAllColumns()
        self.artistList.DeleteAllItems()
        self.artistList.InsertColumn( 0, "Artist" )
        self.artistList.SetColumnWidth( 0, self.artistWidth )

        index = 0
        for junk,artist in sorted( self.main.library.artists.items() ):
            if( artist['name'].lower().find( self.filter.GetValue().lower() ) > -1 ):
                self.artistList.InsertStringItem( index, artist['name'] )
                index+=1

    def RecreateReleases( self, event=None ):
        self.releases.DeleteAllColumns()
        self.releases.DeleteAllItems()
        self.releases.InsertColumn( 0, "Release" )
        self.releases.InsertColumn( 1, "Have" )
        self.releases.InsertColumn( 2, "Date" )
        self.releases.SetColumnWidth( 0, self.rightWidth * .70 )
        self.releases.SetColumnWidth( 1, self.rightWidth * .09 )
        self.releases.SetColumnWidth( 2, self.rightWidth * .21 )

        if event:
            index = 0
            for junk,artist in self.main.library.artists.items() :
                if( artist['name'] == event.GetItem().GetText() ) :
                    self.link.SetURL( str(artist['aid']) )
                    self.link.SetLabel( artist['name'] )
                    self.link.SetToolTipString( str(artist['aid']) )
                    self.link.SetVisited(False)
                    whatURL = "http://what.cd/artist.php?name=" + str(artist['name'])
                    self.whatLink.SetURL( str(whatURL) )
                    self.whatLink.SetLabel('Search on What.CD' )
                    self.whatLink.SetToolTipString( str(whatURL) )
                    self.whatLink.SetVisited(False)
                    if( artist['aid'] == 0 ):
                        # queue a high priority lookup of this artist
                        self.main.library.lookupReleases( junk, 6 )
                    for rel in artist['releases']:
                        self.releases.InsertStringItem( index, rel.title )
                        self.releases.SetStringItem( index, 1, rel.have and "Y" or "N" )
                        self.releases.SetStringItem( index, 2, rel.date )
                        index += 1

    def InitSize( self ):
        self.lrSplitter.SetSashPosition( LEFT_WIDTH )
        self.tbSplitter.SetSashPosition( -100 )
        self.artistList.SetColumnWidth( 0, LEFT_WIDTH )

    def OnArtistColDrag( self, event ):
        self.artistWidth = event.GetPosition()[0]

    def FormatItem( self, itemNum, list, bg=wx.WHITE, fg=wx.BLACK,
                    size=RELEASE_FONT_SIZE,family=wx.FONTFAMILY_DEFAULT,
                    weight=wx.FONTWEIGHT_NORMAL ):
        item =  list.GetItem( itemNum )
        font = wx.Font( size, family, wx.FONTSTYLE_NORMAL, weight )
        item.SetFont( font )
        item.SetBackgroundColour( bg )
        item.SetTextColour( fg )
        list.SetItem( item )

    def OnResize( self, event ):
        if self.GetAutoLayout():
            self.Layout()


# coding: utf-8

import xair
import gym
import xsim
import numpy as np
import xtools as xt
import pandas as pd
import argparse
import matplotlib.pyplot as plt
from tqdm import tqdm
from dataclasses import dataclass
from feles.controller import sfb
from feles.controller.se import SystemEstimator3
from feles.utils.buffer import ReplayBuffer, Experience

COLOR_ZERO = "black"
COLOR_DATA = ["gray", "forestgreen", "hotpink"]
WIDTH_ZERO = "1"

@dataclass
class Log:
    state: np.ndarray
    action: np.ndarray
    next_state: np.ndarray

    @staticmethod
    def format(buffer):
        state = np.vstack([exp.state for exp in buffer]).astype(np.float32)
        action = np.vstack([exp.action for exp in buffer]).astype(np.float32)
        next_state = np.vstack([exp.next_state for exp in buffer]).astype(np.float32)
        return Log(state, action, next_state)

def dev(args):
    xt.info("development mode of SystemEstimator2")
    cf = xt.Config(args.config)

    # plant
    ecf = cf.env
    env = build_env(cf)
    dim_state = 2
    dim_action = 1
    xt.info("env", env)

    # controller
    scf = cf.controller.sfb
    Q = np.diag(scf.Q).astype(float)
    R = np.diag(scf.R).astype(float)
    K = sfb.compute_sfb_gain(env, Q, R)
    K = np.squeeze(K)

    # estimator
    se3 = SystemEstimator3(dim_state, dim_action, cf.controller.se.lr)

    # logger
    logger = xsim.Logger()
    rb = ReplayBuffer(int(1.0 / ecf.dt))

    # prepare simulation
    loss = 0.0

    xt.info("run simulation")
    for time in tqdm(xsim.generate_step_time(ecf.due, ecf.dt)):
        obs = env.observation
        state = env.state
        action = compute_action(env, K)

        # simulation update
        env.step(action)

        rb.add(Log(state, action[1], env.state))
        if len(rb) >= 100:
            data = rb.buffer
            loss = se3.update(data.state, data.action, data.next_state)
        predict = se3.predict(state, action[1:])

        logger.store(time=time, obs=obs, act=action, pre=predict, loss=loss).flush()

    print(se3.A())
    exit()
    # retrieve result
    xt.info("make result")
    res = xsim.Retriever(logger)
    res = pd.DataFrame(dict(
        time=res.time(),
        pitch=xt.r2d(res.obs(env.IX_T)),
        reference=xt.r2d(res.obs(env.IX_C)),
        predict=xt.r2d(res.pre(env.IX_T)),
        elevator=xt.r2d(res.act(env.IX_de)),
        loss=res.loss()
    ))

    if not args.show_fig and not cf.result.fig.show:
        return args

    xt.info("plot result")
    delim = xt.round(xt.r2d([
        env.action_space.low[env.IX_de],
        env.action_space.high[env.IX_de],
    ]), 3)

    fig, axes = plt.subplots(nrows=3, sharex=True, figsize=(8, 8))
    # draw zero-lines
    axes[0].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)
    axes[1].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)

    res.plot(
        x="time", y=["reference", "pitch", "predict"], ax=axes[0],
        style=["--", "-", "-"], color=COLOR_DATA,
        xlim=[0, ecf.due], ylim=ecf.reference.range, grid=True,
        yticks=np.arange(ecf.reference.range[0], ecf.reference.range[1]+1, 1)
    )
    res.plot(
        x="time", y="elevator", ax=axes[1],
        style="-", color="orange",
        ylim=delim, grid=True,
        yticks=np.arange(delim[0], delim[1]+1, 1)
    )
    res.plot(
        x="time", y="loss", ax=axes[2],
        style="-", color="orange",
        ylim=[0, np.max(res.loss) * 1.2], grid=True
    )

    plt.show()

def build_env(cf):
    ecf = cf.env
    env = gym.make(
        ecf.name, dt=ecf.dt,
        target_range=xt.d2r(ecf.reference.range),
        target_period=ecf.reference.interval
    )
    env.reset()
    return env

def compute_action(env, K):
    x = env.state
    r = env.target[:2]
    e = r - x
    u = e.dot(K)
    u = np.array([0, u], dtype=env.dtype)
    u = np.clip(u, env.action_space.low, env.action_space.high)
    return u

def run(args):
    xt.info("run to dev system estimation")
    cf = xt.Config(args.config)

    # prepare env
    ecf = cf.env
    env = gym.make(
        ecf.name, dt=ecf.dt,
        target_range=xt.d2r(ecf.reference.range),
        target_period=ecf.reference.interval
    )
    env.reset()
    xt.info("env", env)

    # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    # controllers
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # state F/B
    #
    scf = cf.controller.sfb
    Q = np.diag(scf.Q).astype(float)
    R = np.diag(scf.R).astype(float)
    K = sfb.compute_sfb_gain(env, Q, R)
    K = np.squeeze(K)

    # logger
    logger = xsim.Logger()
    rb = ReplayBuffer(int(ecf.due / ecf.dt + 1))

    xt.info("run simulation")
    for time in tqdm(xsim.generate_step_time(ecf.due, ecf.dt)):
        state = env.state
        action = compute_action(env, K)
        logger.store(time=time, x=state, u=action).flush()

        # simulation update
        env.step(action)

        rb.add(state, action, env.state)

    # retrieve result
    xt.info("make result")
    res = xsim.Retriever(logger)
    res = pd.DataFrame(dict(
        time=res.time(),
        pitch=xt.r2d(res.state(env.IX_T)),
        reference=xt.r2d(res.state(env.IX_C)),
        elevator=xt.r2d(res.action(env.IX_de))
    ))

    if not args.show_fig and not cf.result.fig.show:
        return args

    xt.info("plot result")
    delim = xt.round(xt.r2d([
        env.action_space.low[env.IX_de],
        env.action_space.high[env.IX_de],
    ]), 3)

    fig, axes = plt.subplots(nrows=2, sharex=True)
    # draw zero-lines
    axes[0].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)
    axes[1].axhline(0, color=COLOR_ZERO, linewidth=WIDTH_ZERO)

    res.plot(
        x="time", y=["reference", "pitch"], ax=axes[0],
        style=["--", "-"], color=COLOR_DATA,
        xlim=[0, ecf.due], ylim=ecf.reference.range, grid=True,
        yticks=np.arange(ecf.reference.range[0], ecf.reference.range[1]+1, 1)
    )
    res.plot(
        x="time", y="elevator", ax=axes[1],
        style="-", color="orange",
        ylim=delim, grid=True,
        yticks=np.arange(delim[0], delim[1]+1, 1)
    )

    plt.show()

def prepare_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--dev", action="store_true")
    parser.add_argument("--config", default="configs/se_no_fail.yaml", type=str)
    # result export
    parser.add_argument("--show-fig", action="store_true")

    return parser.parse_args()

if __name__ == '__main__':
    args = prepare_args()
    if args.dev:
        dev(args)
        exit(0)
    run(args)


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 14 01:14:25 2020

@author: SHOBHIT SAINI
"""
import numpy as np
import cv2
# Import the image
img = cv2.imread('D:\IIT_BBS_CLG_WORK\CV_Practice\island.jpeg')
cv2.imshow('BGR_img_default', img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow('RGB_IMG', img)
cv2.waitKey(0)
cv2.destroyAllWindows()


import os
from shutil import copyfile

import lxml.etree as etree

class config:

    def __init__(self, os=None, device=None, h=None, w=None, output=None, num_update_steps=None,
                 num_level_per_agent=None, num_worker=None, multiagent=None, training_attempts_per_level=None,
                 memory_size=None, memory_type=None, singleagent=None, lr=None, train_time_per_ep=None,
                 train_time_rise=None, train_batch=None, gamma=None, eps_start=None, eps_test=None, network=None,
                 reward_type=None, eval_freq=None, train_template=None, test_template=None, simulation_speed=None,
                 online_training=None, test_steps=None):
        # operating system
        self.test_steps = test_steps
        self.online_training = online_training
        self.os = os

        # pytorch parameters
        self.device = device

        # image network parameters
        self.h = h
        self.w = w
        self.output = output

        # multiagent trainning parameters
        self.num_update_steps = num_update_steps
        self.num_level_per_agent = num_level_per_agent
        self.num_worker = num_worker
        self.multiagent = multiagent
        self.training_attempts_per_level = training_attempts_per_level
        self.memory_size = memory_size
        self.memory_type = memory_type

        # single agent training parameters
        self.singleagent = singleagent

        # general trainning parameters

        self.lr = lr
        self.train_time_per_ep = train_time_per_ep
        self.train_time_rise = train_time_rise
        self.train_batch = train_batch
        self.gamma = gamma
        self.eps_start = eps_start
        self.eps_test = eps_test  # randomeness in testing
        self.network = network
        self.reward_type = reward_type  # 'score' / 'passing'
        self.simulation_speed = simulation_speed
        self.eval_freq = eval_freq

        self.train_template = train_template  # ['1_1_1', '1_1_2'] level 1 capability 1 template 1 and 2 for training
        self.test_template = test_template  # ['1_1_3', '1_1_4'] level 1 capability 1 template 3 and 4 for testing

        self.train_level_list = []
        self.test_level_list = []
        self.total_level = 0
        self.target_level_path = '../tasks/generated_tasks/'
        self.origin_level_path = '../buildgame/{}/9001_Data/StreamingAssets/Levels/novelty_level_1/type1/Levels/'.format(
            self.os)
        self.game_level_path = '9001_Data/StreamingAssets/Levels/novelty_level_1/type1/Levels/'.format(self.os)
        self.game_config_path = '../buildgame/{}/config.xml'.format(self.os)
        self.update_level_index()

    def update_level_index(self):
        '''
        by taking the TRAIN_TEMPLATE and TEST_TEMPLATE,
        this function create the config list for training and testing

        :return:
        '''

        # remove all the levels in self.origin_level_path
        old_levels = os.listdir(self.origin_level_path)
        for old_level in old_levels:
            os.remove(os.path.join(self.origin_level_path, old_level))

        if len(self.train_template) == 1 and self.train_template == self.test_template:  # within template training and testing
            # by default using 80% of levels to train and 20% for testing
            train_percent = 0.8
            level, capability, template_idx = self.train_template[0].split('_')
            template_path = os.path.join(self.target_level_path, level, capability, template_idx)

            # copy all target levels to self.origin_level_path
            new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))
            total_template_level_path = []
            for new_level in new_levels:
                src_path = os.path.join(template_path, new_level)
                dst_path = os.path.join(self.origin_level_path, new_level)
                copyfile(src_path, dst_path)
                total_template_level_path.append(os.path.join(self.game_level_path, new_level))
            self.train_level_list = list(range(1, int(len(total_template_level_path) * train_percent) + 1))
            self.test_level_list = list(
                range(int(len(total_template_level_path) * train_percent) + 1, len(total_template_level_path) + 1))

            # write configs
            parser = etree.XMLParser(encoding='UTF-8')
            game_config = etree.parse(self.game_config_path, parser=parser)
            config_root = game_config.getroot()
            # remove old level path
            for level in list(config_root[1][0][0]):
                config_root[1][0][0].remove(level)
            # add new level path
            for l in total_template_level_path:
                new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
                new_level.set('level_path', l)

            # add a repeated level for the weird not loadding last level bug
            new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
            new_level.set('level_path', l)

            game_config.write(self.game_config_path)

        else:  # when more than one template is required to train
            # cross template testing
            total_template_level_path = []

            for template in self.train_template:
                level, capability, template_idx = template.split('_')
                template_path = os.path.join(self.target_level_path, level, capability, template_idx)

                # move all levels to folder
                new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))
                for new_level in new_levels:
                    src_path = os.path.join(template_path, new_level)
                    dst_path = os.path.join(self.origin_level_path, new_level)
                    copyfile(src_path, dst_path)
                    total_template_level_path.append(os.path.join(self.game_level_path, new_level))

            self.train_level_list = [i for i in range(1, len(total_template_level_path) + 1)]

            for template in self.test_template:
                level, capability, template_idx = template.split('_')
                template_path = os.path.join(self.target_level_path, level, capability, template_idx)

                # move all levels to folder
                new_levels = sorted(os.listdir(template_path), key=lambda x: int(x.split('.')[0].split("_")[-1]))

                # takes only the last 20 levels per each template for testing
                new_levels = new_levels[80:]

                for new_level in new_levels:
                    src_path = os.path.join(template_path, new_level)
                    dst_path = os.path.join(self.origin_level_path, new_level)
                    copyfile(src_path, dst_path)
                    total_template_level_path.append(os.path.join(self.game_level_path, new_level))

            self.test_level_list = [i for i in range(self.train_level_list[-1] + 1, len(total_template_level_path) + 1)]

            # write config
            # write configs
            parser = etree.XMLParser(encoding='UTF-8')
            game_config = etree.parse(self.game_config_path, parser=parser)
            config_root = game_config.getroot()
            # remove old level path
            for level in list(config_root[1][0][0]):
                config_root[1][0][0].remove(level)
            # add new level path
            for l in total_template_level_path:
                new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
                new_level.set('level_path', l)

            # add a repeated level for the weird not loadding last level bug
            new_level = etree.SubElement(config_root[1][0][0], 'game_levels')
            new_level.set('level_path', l)

            game_config.write(self.game_config_path)

if __name__ == '__main__':
    from LearningAgents.DQNImageAgent import DQNImageAgent
    from LearningAgents.RL.DQNImageDueling import DQNImageDueling

    test_config = {
        # operating system
        'os': "Linux",
        # pytorch parameters
        'device': "cuda:0",

        # image network parameters
        'h': 224,
        'w': 224,
        'output': 180,

        # multiagent trainning parameters
        'num_update_steps': 100,
        'num_level_per_agent': 20,
        'num_worker': 30,
        'multiagent': DQNImageAgent,

        # single agent training parameters
        'singleagent': None,

        # general trainning parameters

        'train_time_per_ep': 32,
        'train_time_rise': 1.5,
        'train_batch': 32,
        'gamma': 0.99,
        'eps_start': 0.95,
        'eps_decay': 0.99,

        'network': DQNImageDueling,

        'train_template': ['1_1_1'],  # ['1_1_1', '1_1_2'] level 1 capability 1 template 1 and 2 for training
        'test_template': ['1_1_3'],  # ['1_1_3', '1_1_4'] level 1 capability 1 template 3 and 4 for testing
    }
    c = config(**test_config)

    print(c.train_level_list)
    print(c.test_level_list)


from django.shortcuts import render
from django.urls import reverse_lazy,reverse
# from django.core.urlresolvers import reverse
# Create your views here.
from django.http import HttpResponse
from django.views.generic.edit import FormView
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from django.views.generic import DetailView,ListView
from Products.models import Product
from Products.forms import ProductForm

class ProductCreate(CreateView):
    form_class = ProductForm
    template_name="Products/product_create.html"
    success_url = '/'
    def form_valid(self, form):

        form.save()
        return super(ProductCreate, self).form_valid(form)

    def form_invalid(self, form):
        # print "form is invalid"
        return HttpResponse("form is invalid.. this is just an HttpResponse object")

    def get_success_url(self):
        return reverse("Product-list")

    def get_context_data(self, **kwargs):
        kwargs["object_list"] = Product.objects.all()
        return super(ProductCreate, self).get_context_data(**kwargs)

class ProductUpdate(UpdateView):
    model = Product
    form_class = ProductForm
    template_name_suffix = '_update_form'
    def get_success_url(self):
        return reverse("Product-list")

class ProductDelete(DeleteView):
    # form_class = ProductForm
    model= Product
    success_url = reverse_lazy('Product-list')
    items_to_delete = []

class ProductDetailView(DetailView):
    model= Product
class ProductListView(ListView):
    queryset= Product.objects.all()


from qwerqerqwerqweqwerqwerqwerqwerqwerwewqrwrerewre import get_date
from random import randint
from aiogram import Router, Dispatcher
from aiogram.types import Message,InlineKeyboardMarkup,InlineKeyboardButton,CallbackQuery
from aiogram import Bot
from aiogram.types.dice import DiceEmoji
from aiogram import F
from aiogram.types import ContentType
from aiogram.filters import Text
from aiogram.filters.command import Command
from asyncio import run, sleep
from aiogram.types import Poll, PollAnswer
from abc import ABC,abstractclassmethod
from xixi import my_token
#from asyncio import run
City=InlineKeyboardMarkup(
    inline_keyboard= [
    [
    InlineKeyboardButton(text="Tashkent",callback_data="Tashkent"),
    InlineKeyboardButton(text="Fergana",callback_data="Fergana"),
    InlineKeyboardButton(text="Andijon",callback_data="Andijon"),
    ],
    [
    InlineKeyboardButton(text="Namangan",callback_data="Namangan"),
    InlineKeyboardButton(text="Qashqadaryo",callback_data="Qashqadaryo"),
    InlineKeyboardButton(text="Sirdaryo",callback_data="Sirdaryo"),
    ],
    [
    InlineKeyboardButton(text="Surhondaryo",callback_data="Termez"),
    InlineKeyboardButton(text="Samarqand",callback_data="Samarqand"),
    InlineKeyboardButton(text="Xorazm",callback_data="Urgench"),
    ],
    [
    InlineKeyboardButton(text="Buxoro",callback_data="Buxoro"),
    InlineKeyboardButton(text="Navoiy",callback_data="Navoiy"),
    InlineKeyboardButton(text="Jizzax",callback_data="Jizzax"),
    ],
    [
    InlineKeyboardButton(text="Qoraqolpog'iston",callback_data="Karakalpakstan"),
    ],
    ],
)
great=Router()
@great.message(Command(commands=["start"]))
async def great_message(msg:Message,bot:Bot):
    await msg.answer("Bu 12 viloyat ob-havo ma'lumoti boti",reply_markup=City)

@great.callback_query()
async def get_weather(call:CallbackQuery):
    city=call.data
    print(city)
    weather=get_date(city)
    await call.answer(f"{weather} in C")

async def start():
    dp=Dispatcher()
    kb=Bot(my_token)#token
    dp.include_router(great)
    try:
        await dp.start_polling(kb)
    except Exception as f:
        await kb.session.close()
if __name__=="__main__":
    run(start())


import os
import json
import time
import flask
import openai

from flask import (
    request, jsonify
)
from datetime import datetime
from dotenv import load_dotenv

load_dotenv()

openai.api_key = os.getenv('OPENAI_API_KEY')

app = flask.Flask(__name__)
app.config["DEBUG"] = True

SP = """You are an AI assitant that takes in a piece of text and tries to summarize it in 2-5 sentences."""

TP = """You are an AI assitant that takes in a piece of text and extracts topics from the text."""

def complete(system_prompt: str, user_prompt: str) -> str:
    start = time.time()
    completion = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[
            {
                "role": "system",
                "content": system_prompt
            },
            {
                "role": "user",
                "content": user_prompt
            }
        ],
        max_tokens=400
    )
    time_to_complete = time.time() - start
    return completion.choices[0].message.content, time_to_complete

@app.route('/', methods=['POST'])
def home():
    return "
Hello from Azure Hackthon! :)
"

@app.route('/api/v1/summarize', methods=['POST'])
def summarize():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        data = json.loads(request.data)
        if data.get('context'):
            summary, ttc = complete(
                system_prompt=SP,
                user_prompt=data.get('context')
            )

            return {
                "status": "success",
                "message": summary,
                "timeToComplete": ttc,
                "timeStamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            }
        else:
          return {
            "status": "error",
            "message": "Context to summarize must be provided."
        }
    else:
        return {
            "status": "error",
            "message": "Unsupported content type."
        }

@app.route('/api/v1/topics', methods=['POST'])
def topcis():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        data = json.loads(request.data)
        if data.get('context'):
            if data.get('numberTopics'):
                topics, ttc = complete(
                    system_prompt=TP,
                    user_prompt=f"Generate {data.get('numberTopics')} topics, not numbered, seperated by a newline for the following text:\n{data.get('context')}\n\nTopics:\n"
                )

                return {
                    "status": "success",
                    "message": topics.split('\n'),
                    "timeToComplete": ttc,
                    "timeStamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                }
            else:
                return {
                    "status": "error",
                    "message": "Number of topics must be provided."
                }
        else:
          return {
            "status": "error",
            "message": "Context to summarize must be provided."
        }
    else:
        return {
            "status": "error",
            "message": "Unsupported content type."
        }

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=7860)


import unittest

import torch
from torch_quant.observer import HistogramObserver

class TestHistogramObserver(unittest.TestCase):
    def test_basic(self):
        obs = HistogramObserver()
        obs.fake_quant = False
        inp = torch.tensor([1, 2, 3, 4, 5], dtype=torch.float)
        oup = obs(inp)
        obs.calculate_qparams()
        self.assertTrue(torch.equal(oup, inp))
        target_scale = torch.tensor(0.01960018463432789)
        target_zp = torch.tensor(0, dtype=torch.int32)
        self.assertTrue(torch.equal(target_scale, obs.scale))
        self.assertTrue(torch.equal(target_zp, obs.zero_point))
        target_fake_quant_output = torch.fake_quantize_per_tensor_affine(
            inp, target_scale, target_zp, 0, 255
        )
        obs.fake_quant = True
        obs.observe = False
        oup = obs(inp)
        self.assertTrue(torch.equal(target_fake_quant_output, oup))

    def test_per_channel(self):
        with self.assertRaises(NotImplementedError):
            HistogramObserver(qscheme=torch.per_channel_symmetric)

if __name__ == "__main__":
    unittest.main()


from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql.functions import avg, min, max

spark = SparkSession.builder.appName('filter-example').getOrCreate()
sc = spark.sparkContext
sc.setLogLevel("ERROR")

def run():
    print()
    student = spark.createDataFrame([
        Row(name='Song', subject='English', term='Spring', score=80),
        Row(name='Song', subject='English', term='Fall', score=90),
        Row(name='Song', subject='Math', term='Spring', score=90),
        Row(name='Song', subject='Math', term='Fall', score=100),
        Row(name='Donald trump', subject='English', term='Spring', score=20),
        Row(name='Donald trump', subject='English', term='Fall', score=40),
        Row(name='Donald trump', subject='Math', term='Spring', score=30),
        Row(name='Donald trump', subject='Math', term='Fall', score=25),
    ])

    student.printSchema()
    student.show()

    print('Get the aggregations')
    df = student.groupBy('name', 'subject').agg(
        avg('score').alias("avg_score"),
        min('score').alias("min_score"),
        max('score').alias("max_score")
    )

    df.show()

if __name__ == '__main__':
    run()


from django.conf.urls import patterns, url

from . import views

urlpatterns = patterns('',

    url(r'^register/$',
        view=views.RegisterView.as_view(),
        name="register",
    ),

    url(r'^(?P<profile_id>[0-9]+)/$',
        view=views.ProfileDetailView.as_view(),
        name="detail",
    ),

    url(r'^(?P<profile_id>[0-9]+)/edit$',
        view=views.ProfileEditView.as_view(),
        name="edit",
    ),

)


from transliterate import translit
import re

def slugify(string: str) -> str:
    return re.sub(
        r'^-+|-+$', '', re.sub(
            r'[\s_-]+', '-', re.sub(
                r'[^\w\s-]', '', translit(
                    u"" + string, "ru", reversed=True
                )
                .lower()
                .strip())))


from django.contrib import admin
from import_export import resources
from import_export.admin import ImportExportModelAdmin
from django.conf import settings
from django.conf.urls import url
from django.contrib import messages
from django.shortcuts import redirect
from pleethai.models import Word, SysWordJapanese, SysWordConnector, WordClass, Example, Constituent, Tag, TaggedItem
from pleethai.common import Common
from taggit.utils import _parse_tags
from django.db import models

class WordResource(resources.ModelResource):

    class Meta:
        model = Word

class WordClassResource(resources.ModelResource):

    class Meta:
        model = WordClass

class ExampleResource(resources.ModelResource):

    class Meta:
        model = Example

class ConstituentResource(resources.ModelResource):

    class Meta:
        model = Constituent

class TagResource(resources.ModelResource):

    class Meta:
        model = Tag

class WordAdmin(ImportExportModelAdmin):
    resource_class = WordResource
    # Add "UPDATE SYS_WORD TABLES" button
    change_list_template = "admin/pleethai/Word/change_list.html"

    # URL for update system tables
    def get_urls(self):
        urls = super().get_urls()
        my_urls = [
            url(r'^updatesystables/$', self.admin_site.admin_view(self.update_system_tables)),
            url(r'^appendsystables/$', self.admin_site.admin_view(self.append_system_tables)),
        ]
        return my_urls + urls

    # Condition of search
    def exist_japanese(self, word1, word2) -> bool:
        return word1.japanese == word2.japanese and word1.hiragana == word2.hiragana

    def delete_all(self, model: models.Model):
        while model.objects.count():
            ids = model.objects.values_list('pk', flat=True)[:100]
            model.objects.filter(pk__in = ids).delete()

    # Create SysWordJapanse table and SysWordConnector table from Word table
    def update_system_tables(self, request):
        all_words = Word.objects.all()
        sys_japanese = []
        sys_word_con = []
        temp_constituent = []
        try:
            for word in all_words:
                # Add a Word record (japanese & hiragana unique) to SysWordJapanese table
                if not Common.contains(sys_japanese, word, self.exist_japanese):
                    sys_japanese.append(SysWordJapanese.create(word))
                # Add a Word record to SysWordConnector table
                tempConnect = SysWordConnector.create(word, sys_japanese)
                if tempConnect != None:
                    sys_word_con.append(tempConnect)
            # delete and recreate
            for con_dict in Constituent.objects.values():
                temp_constituent.append(Constituent(**con_dict))
            self.delete_all(Constituent)
            self.delete_all(SysWordConnector)
            self.delete_all(SysWordJapanese)
            SysWordJapanese.objects.bulk_create(sys_japanese)
            SysWordConnector.objects.bulk_create(sys_word_con)
            Constituent.objects.bulk_create(temp_constituent)
            self.update_tags()
            messages.info(request, "Succeeded to update system tables")
        except Exception as e:
            messages.error(request, "Failed to update system tables")
            messages.error(request, str(e))
        return redirect(request.META['HTTP_REFERER'])

    # Update tags
    def update_tags(self):
        all_tags = Tag.objects.all().values_list('name', flat=True)
        for word_con in SysWordConnector.objects.all():
            for tag in _parse_tags(Word.objects.filter(id=word_con.id).first().tags):
                # skip the invalid tag
                if tag not in all_tags:
                    continue
                word_con.tags.add(tag)

    # Append SysWordJapanse table and SysWordConnector table
    # (append words number max is defined in settings.py)
    def append_system_tables(self, request):
        all_words = Word.objects.all()
        exist_sys_japs = list(SysWordJapanese.objects.all())
        sys_japanese = []
        exist_word_ids = SysWordConnector.objects.all().values_list("id", flat=True).distinct()
        sys_word_con = []
        all_tags = Tag.objects.all().values_list('name', flat=True)
        try:
            for word in all_words:
                # skip existed word
                if word.id in exist_word_ids:
                    continue

                # check max append
                if len(sys_word_con) >= settings.APPEND_TABLES_MAX_NUM:
                    break

                # Add a Word record (japanese & hiragana unique) to SysWordJapanese table
                if not Common.contains(exist_sys_japs + sys_japanese, word, self.exist_japanese):
                    sys_japanese.append(SysWordJapanese.create(word))
                # Add a Word record to SysWordConnector table
                tempConnect = SysWordConnector.create(word, exist_sys_japs + sys_japanese)
                if tempConnect != None:
                    sys_word_con.append(tempConnect)
                    # create tag
                    for tag in _parse_tags(word.tags):
                        # skip the invalid tag
                        if tag not in all_tags:
                            continue
                        tempConnect.tags.add(tag)
            # append
            SysWordJapanese.objects.bulk_create(sys_japanese)
            SysWordConnector.objects.bulk_create(sys_word_con)

            if len(sys_japanese) == 0 and len(sys_word_con) == 0:
                messages.info(request, "Already all data appended.")
            else:
                messages.info(request, "Succeeded to append {0} system tables(id from {1} to {2})"
                    .format(len(sys_word_con), sys_word_con[0].id, sys_word_con[-1].id))
                messages.info(request, "Not-appended data may remain. Try to 'append system tables' again.")
        except Exception as e:
            messages.error(request, "Failed to append system tables")
            messages.error(request, str(e))
        return redirect(request.META['HTTP_REFERER'])

class WordClassAdmin(ImportExportModelAdmin):
    resource_class = WordClassResource

class ExampleAdmin(ImportExportModelAdmin):
    resource_class = ExampleResource

class ConstituentAdmin(ImportExportModelAdmin):
    resource_class = ConstituentResource

class TaggedItemInline(admin.StackedInline):
    model = TaggedItem

class TagAdmin(ImportExportModelAdmin):
    resource_class = TagResource
    inlines = [TaggedItemInline]

admin.site.register(Word, WordAdmin)
admin.site.register(SysWordJapanese)
admin.site.register(SysWordConnector)
admin.site.register(WordClass,WordClassAdmin)
admin.site.register(Example, ExampleAdmin)
admin.site.register(Constituent, ConstituentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(TaggedItem)


from itertools import combinations

def factor(n):
    ans = []
    d = 2
    while d * d <= n:
        if n % d == 0:
            ans.append(d)
            n //= d
        else:
            d += 1
    if n > 1:
        ans.append(n)
    return ans

def print_out(nums):
    for i in nums:
        print(i, end=' ')
    print()

n = int(input())
res = factor(abs(n))
if n > 0:
    print_out(res)

start = 2 if n > 0 else 1
for i in range(start, len(res) + 1, 2):
    r = set(combinations(res, i))
    for case in r:
        res_copy = res.copy()
        for value in case:
            res_copy.remove(value)
            res_copy.append(-value)
        print_out(res_copy)


#!/usr/bin/env python
# enccoding: utf-8
'''
@author: Yankai
@project:
'''
import xlrd,xlwt
from xlutils.copy import copy

#打开excel
readbook = xlrd.open_workbook(r'C:\Users\kmf\接口复习\test.xlsx')
#获取读入文件的sheet
sheet1 = readbook.sheet_by_index(0)
sheet2 = readbook.sheet_by_name('sheet2')
allsheetnames = readbook.sheet_names()
print(sheet1,sheet2,allsheetnames)
#获取sheet最大行数和列数
nrows = sheet1.nrows#行
ncols = sheet2.ncols#列
print(nrows,ncols)
#获取某个单元格的值
lng = sheet1.cell(1,1).value#获取1行1列的表格值，从0开始计数
print(lng)
#获取某行/某列的值
row_value = sheet1.row_values(2)#获取x行的值，从0开始计数
col_value = sheet1.col_values(3)#获取y列的值，从0开始计数
print(row_value,col_value)