import pickle

import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from sklearn import preprocessing
from sklearn.model_selection import TimeSeriesSplit, cross_validate
from sklearn.linear_model import LinearRegression,LogisticRegression
import matplotlib.pyplot as plt
import seaborn as sns

# settings
import warnings

warnings.filterwarnings("ignore")

# reading data files
sales_train = pd.read_csv('data/sales_train_formated.csv')
sales_train.date = pd.to_datetime(sales_train.date, format='%Y-%m-%d')
items = pd.read_csv('data/items.csv')
shops = pd.read_csv('data/shops.csv')
item_category = pd.read_csv('data/item_categories.csv')
sales_test = pd.read_csv('data/test.csv')

# data pre-processing
sales_train.insert(3, 'month', sales_train['date'].dt.month)
#sales_train.insert(4, 'year', sales_train['date'].dt.year)

#sales_train['item_cnt_day'] = sales_train.groupby(['shop_id', 'item_id'], as_index=False)['item_cnt_day'].transform(
 #   pd.Series.diff)

# mean sales of item for each month period
month_mean = sales_train.groupby(['date_block_num','shop_id', 'item_id'], as_index=False)[['item_cnt_day']].mean()
month_mean = month_mean.rename(columns={'item_cnt_day': 'month_mean'})

# mean sales of shop
mean_sales= sales_train.groupby(['date_block_num','item_id'], as_index=False)[['item_cnt_day']].mean()
month_mean = pd.merge(month_mean, mean_sales, how='left', on=['date_block_num','item_id'])
month_mean = month_mean.rename(columns={'item_cnt_day': 'mean_sales'})

# mean prices of shop
#mean_prices = sales_train.groupby(['shop_id'], as_index=False)[['item_price']].mean()
#month_mean = pd.merge(month_mean, mean_prices, how='left', on=['shop_id'])
#month_mean = month_mean.rename(columns={'item_price': 'mean_prices'})

sales_train =sales_train.drop(['item_cnt_day'], axis=1)
sales_train =  pd.merge(sales_train, month_mean, how='left', on=['date_block_num','shop_id', 'item_id'])

'''sales_train = sales_train.groupby([c for c in sales_train.columns if c not in ['item_cnt_day']], as_index=False)[
    ['item_cnt_day']].sum()
sales_train = sales_train.rename(columns={'item_cnt_day': 'item_cnt_month'})
shop_item_monthly_mean = sales_train[['shop_id', 'item_id', 'item_cnt_month']].groupby(['shop_id', 'item_id'], as_index=False)[
    ['item_cnt_month']].mean()

shop_item_monthly_mean = shop_item_monthly_mean.rename(columns={'item_cnt_month': 'item_cnt_month_mean'})
sales_train = pd.merge(sales_train, shop_item_monthly_mean, how='left', on=['shop_id', 'item_id'])

print(sales_train.head(5))
print(sales_train.shape)

'''

# Last Month in the data : Oct 2015
shop_item_prev_month = sales_train[sales_train['date_block_num'] == 33][['shop_id', 'item_id', 'month_mean']]
shop_item_prev_month = shop_item_prev_month.rename(columns={'month_mean': 'prev_month_mean'})
sales_train = pd.merge(sales_train, shop_item_prev_month, how='left', on=['shop_id', 'item_id'])
sales_train = sales_train.fillna(0.)

sales_train = pd.merge(sales_train, items, how='left', on='item_id')
sales_train = pd.merge(sales_train, item_category, how='left', on=['item_category_id'])
sales_train = pd.merge(sales_train, shops, how='left', on=['shop_id'])

sales_train = sales_train.drop(['item_name','item_category_name','shop_name','date','item_price'], axis=1)

'''
# Add more features to test set
# Manipulate test data
sales_test['month'] = 11
sales_test['year'] = 2015
sales_test['date_block_num'] = 34
# sales_test['date_block_num'] = 34
sales_test =  pd.merge(sales_test, month_mean, how='left', on=['date_block_num','shop_id', 'item_id'])
sales_test = pd.merge(sales_test, shop_item_prev_month, how='left', on=['shop_id', 'item_id'])
sales_test = sales_test.fillna(0.)
sales_test = pd.merge(sales_test, items, how='left', on='item_id')
sales_test = pd.merge(sales_test, item_category, how='left', on=['item_category_id'])
sales_test = pd.merge(sales_test, shops, how='left', on=['shop_id'])
sales_test = sales_test.drop(['item_name','item_category_name','shop_name'], axis=1)
print(sales_test.shape)
print(sales_test.columns)
'''

#train_test_df = pd.concat([sales_train, sales_test], axis=0, ignore_index=True)
#stores_hm = train_test_df.pivot_table(index='shop_id', columns='item_category_id', values='item_cnt_month',
#                                      aggfunc='count', fill_value=0)
#stores_hm.tail()

print(sales_train.columns)

feature_list = [c for c in sales_train.columns if c != 'month_mean']
print(feature_list)
x1 = sales_train[sales_train['date_block_num'] < 33]
#x1 = sales_train
#x1 = sales_train['item_price']
y1 = np.log1p(x1['month_mean'].clip(0., 20.))
x1 = x1[feature_list]
#x1['mean_prices'] = np.log1p(x1['mean_prices'].clip(0., 100.))

x2 = sales_train[sales_train['date_block_num'] == 33]
#y2 = np.log1p(x2['month_mean'].clip(0., 20.))
y2 = np.log1p(x2['month_mean'].clip(0., 20.))
x2 = x2[feature_list]
#x2['mean_prices'] = np.log1p(x2['mean_prices'].clip(0., 100.))

rf = RandomForestRegressor(n_estimators=25, oob_score=True, random_state=12, max_depth=3, n_jobs=-1)
rf.fit(x1, y1)
print(1-rf.oob_score_)

lr = LinearRegression()
lr.fit(x1, y1)

df = pd.DataFrame()

df['y'] = y2
df['reg']=lr.predict(x2)
df['forest']=rf.predict(x2)

print(df.columns)
df.to_csv('pol.csv', index=False)

'''
rf_file = 'rf_file_diff.sav'
lr_file = 'lr_file_diff.sav'

pickle.dump(rf,open(rf_file,'wb'))
pickle.dump(lr,open(lr_file,'wb'))
'''

importance = rf.feature_importances_
print(importance)

#importance = np.array([0.05844234,0.00764173,0.01998627,0.006654,0.02563119,0.80008461,0.02529188,0.02550973,0.01148034,0.01171568,0.00756224])

#plots
fig, ax = plt.subplots(figsize=(15, 15))
#sns.heatmap(stores_hm, ax=ax, cbar=False)

indices = np.argsort(importance)[::-1]
featurenames = [feature_list[i] for i in indices]
plt.title("Feature Significance")
plt.bar(range(len(indices)),importance[indices],color='b',align='center')
plt.xticks(range(len(indices)),featurenames,rotation=20,fontsize=18)
plt.title("Feature Significance")
plt.xlabel("Features")
plt.xlabel("Significance")
plt.show()

print("RMSE on Validation hold out month 33: {}".format(
    np.sqrt(mean_squared_error(np.log1p(y2.clip(0., 100.)), rf.predict(x2).clip(0., 20.)))))

from sklearn.tree import export_graphviz
#for tree in rf.estimators_:
tree = rf.estimators_[1]
export_graphviz(tree,out_file='ziii.dot',feature_names=feature_list,filled=True,rounded=True )

#from subprocess import call
#call(['dot','-Tpng','ziii.dot','-o','ziiiv.png','Gdpi=600'])


# app.py
import config
import os
from flask import Flask, jsonify
# from flask_sqlalchemy import SQLAlchemy
from model import db
from flask_cors import CORS
from flask.blueprints import Blueprint
from flask_restful import Api

# Initialize Flask APP
app = Flask(__name__)
app.url_map.strict_slashes = False

# Set CORS header
cors = CORS(app)

# Load the appropriate environment configuration based on the 'env' environment variable
try:
    if os.environ.get('env') == 'PROD':
        app.config.from_object(config.ProdConfig)
    elif os.environ.get('env') == 'DEV':
        app.config.from_object(config.DevConfig)
    else:
        app.config.from_object(config.LocalConfig)
except Exception as e:
    print(f"Error loading environment configuration: {e}")

# db = SQLAlchemy(app)
db.init_app(app)

# Define a custom error handler for 404 (Not Found) errors
@app.errorhandler(404)
def not_found(error):
    return jsonify({'error': 'Not found'}), 404

api_bp = Blueprint('api', __name__)
api = Api(api_bp)

class Routes:
    def __init__(self) -> None:
        pass

    # Load API routes from the 'route' module
    def load_api():
        from route import typeahead
        from route import search
        from route import export
        api.add_resource(typeahead.TypeAheadResult, '/typeahead')
        api.add_resource(search.SearchResult, '/search/<profile_id>')
        api.add_resource(export.ExportResult, '/export/<profile_id>')

# Load API routes and register the blueprint with the Flask app
Routes.load_api()
app.register_blueprint(api_bp)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=4000)


#!/usr/bin/env python3

import hashlib
import io
import itertools
import json
import logging
import pathlib
import queue
import threading
import time

import click
import cv2
import numpy as np
import requests

def calc_sha1_hash(path):
    with path.open("rb") as file:
        return hashlib.sha1(file.read()).hexdigest()

def make_video_meta(video_file_path, video_capture):
    return {
        "sha1": calc_sha1_hash(video_file_path),
        "size": video_file_path.stat().st_size,
        "width": int(video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)),
        "height": int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)),
        "fps": video_capture.get(cv2.CAP_PROP_FPS),
        "numberOfFrames": int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT)),
    }

def dump_json_compact(obj):
    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

def dump_numpy(ary):
    bio = io.BytesIO()
    np.save(bio, ary)
    return bio.getvalue()

def read_thread(video_file_path, video_capture, output_queue):
    logging.info("start")

    total_count = 0
    total_time_ns = 0

    for frame_index in itertools.count():
        start_ns = time.perf_counter_ns()
        result, frame = video_capture.read()
        time_ns = time.perf_counter_ns() - start_ns
        if not result:
            break

        output_queue.put(
            {
                "video_file_path": video_file_path,
                "frame_index": frame_index,
                "frame": frame,
            }
        )

        total_count += 1
        total_time_ns += time_ns

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def detection_thread(input_queue, output_queue, detector_base_url):
    logging.info("start")
    logging.info("detector_base_url=%s", detector_base_url)

    url = detector_base_url + "/detect"
    total_count = 0
    total_time_ns = 0

    while True:
        frame_info = input_queue.get()
        try:
            if frame_info is None:
                break
            logging.info("frame_index=%d", frame_info["frame_index"])

            start_ns = time.perf_counter_ns()

            files = {
                "file": (
                    frame_info["video_file_path"].name,
                    dump_numpy(frame_info["frame"]),
                    "application/octet-stream",
                ),
            }
            response = requests.post(url, files=files)
            assert response.status_code == 200
            result = response.json()

            time_ns = time.perf_counter_ns() - start_ns

            output_queue.put(
                {
                    "video_file_path": frame_info["video_file_path"],
                    "frame_index": frame_info["frame_index"],
                    "frame": frame_info["frame"],
                    "result": result,
                }
            )

            total_count += 1
            total_time_ns += time_ns
        finally:
            input_queue.task_done()

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def write_thread(input_queue, output_file_path, video_meta):
    logging.info("start")
    logging.info("output_file_path=%s", str(output_file_path))

    total_count = 0
    total_time_ns = 0

    with output_file_path.open("w") as output_file:
        output_file.write(dump_json_compact(video_meta) + "\n")
        while True:
            frame_info = input_queue.get()
            try:
                if frame_info is None:
                    break

                start_ns = time.perf_counter_ns()

                record = {
                    "frame_index": frame_info["frame_index"],
                    "result": frame_info["result"],
                }
                output_file.write(dump_json_compact(record) + "\n")

                time_ns = time.perf_counter_ns() - start_ns

                total_count += 1
                total_time_ns += time_ns
            finally:
                input_queue.task_done()

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def start_read_worker(**kwargs):
    worker = threading.Thread(
        name="read",
        target=read_thread,
        daemon=True,
        kwargs=kwargs,
    )
    worker.start()
    return worker

def start_detection_workers(detector_base_urls, input_queue, output_queue):
    workers = []

    for thread_index, detector_base_url in enumerate(detector_base_urls):
        worker = threading.Thread(
            name="detection#{}".format(thread_index),
            target=detection_thread,
            daemon=True,
            kwargs={
                "input_queue": input_queue,
                "output_queue": output_queue,
                "detector_base_url": detector_base_url,
            },
        )
        worker.start()
        workers.append(worker)

    return workers

def start_write_worker(**kwargs):
    worker = threading.Thread(
        name="write",
        target=write_thread,
        daemon=True,
        kwargs=kwargs,
    )
    worker.start()
    return worker

def join_all_workers(workers, input_queue):
    for _ in workers:
        input_queue.put(None)
    for worker in workers:
        worker.join()

@click.command()
@click.argument("video_file_path", type=click.Path(exists=True))
@click.argument("output_file_path", type=click.Path(exists=False))
def main(video_file_path, output_file_path):
    threading.current_thread().name = "main"
    logging.info("start")

    video_file_path = pathlib.Path(video_file_path)
    output_file_path = pathlib.Path(output_file_path)
    detector_base_urls = [
        "http://detector-insightface:8000",
        "http://detector-insightface:8000",
    ]

    video_capture = cv2.VideoCapture(str(video_file_path))
    assert video_capture.isOpened()
    video_meta = make_video_meta(video_file_path, video_capture)

    frame_queue = queue.Queue(maxsize=10)
    detection_queue = queue.Queue()

    read_worker = start_read_worker(
        video_file_path=video_file_path,
        video_capture=video_capture,
        output_queue=frame_queue,
    )
    detection_workers = start_detection_workers(
        detector_base_urls=detector_base_urls,
        input_queue=frame_queue,
        output_queue=detection_queue,
    )
    write_worker = start_write_worker(
        input_queue=detection_queue,
        output_file_path=output_file_path,
        video_meta=video_meta,
    )

    read_worker.join()
    join_all_workers(detection_workers, frame_queue)
    join_all_workers([write_worker], detection_queue)

    logging.info("end")

if __name__ == "__main__":
    logging.basicConfig(
        format="%(asctime)s %(levelname)s [%(threadName)s] %(message)s",
        level=logging.INFO,
    )
    main()


import tensorflow as tf
import time
from tensorflow import keras
from tensorflow.keras import layers, datasets, losses, optimizers, Sequential

batchSize = 250
total_words = 90000
max_review_len = 200
embedding_len = 100

(x_train, y_train), (x_test, y_test) = datasets.imdb.load_data(num_words=total_words)

# word_index=datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)

x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
train_db = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(5 * batchSize).batch(batchSize,drop_remainder=True)
test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(batchSize,drop_remainder=True)

class MyLSTM(keras.Model):
    def __init__(self, units):
        super(MyLSTM, self).__init__()
        # 初始化[c0,h0]
        self.state0 = [tf.zeros([batchSize, units]), tf.zeros([batchSize, units])]
        self.state1 = [tf.zeros([batchSize, units]), tf.zeros([batchSize, units])]
        # [b,200] => [b,200,100]
        self.embedding = layers.Embedding(input_dim=total_words, input_length=max_review_len, output_dim=embedding_len)
        # self.lstm_cell0 = layers.LSTMCell(units=units,dropout=0.5)
        # self.lstm_cell1 = layers.LSTMCell(units=units,dropout=0.5)
        self.rnn=Sequential([layers.LSTM(units,dropout=0.5,return_sequences=True),
                             layers.LSTM(units, dropout=0.5, return_sequences=True),
                             layers.LSTM(units, dropout=0.5)
                             ])
        self.outlayer = layers.Dense(1)

    def call(self, inputs, training=None):
        # out:[b,200,100]
        out = self.embedding(inputs)
        # [b,100]
        # state0 = self.state0
        # state1 = self.state1
        # word:[b,100]
        # out1=0
        #  在时间的维度上进行展开
        # for word in tf.unstack(out, axis=1):
        #     out0, state0 = self.lstm_cell0(word, state0, training)
        #     out1, state1 = self.lstm_cell1(out0, state1, training)

        out=self.rnn(out,training)
        # out1: [b,64] 不能使用relu激活函数
        prob = self.outlayer(out)
        prob = tf.sigmoid(prob)
        return prob

def main():
    epochs = 4
    units = 64
    t0 = time.time()
    model = MyLSTM(units=units)
    model.compile(optimizer=optimizers.Adam(0.0005),
                  loss=losses.BinaryCrossentropy(),
                  metrics=['accuracy'], experimental_run_tf_function=False)
    model.fit(train_db, epochs=epochs, validation_data=test_db)
    model.evaluate(test_db)
    t1 = time.time()
    # 395.76 seconds,84.76%
    print('total time cost:', t1 - t0)

if __name__ == '__main__':
    main()


from django.contrib.auth import get_user_model
from rest_framework import authentication
from rest_framework import exceptions

User = get_user_model()

class AppSecretTokenAuthentication(authentication.BaseAuthentication):
    def authenticate(self, request):
        user_token = request.META.get('CL-X-TOKEN')
        if not user_token:
            return None

        try:
            user = User.objects.get(app_secret_token=user_token)
        except User.DoesNotExist:
            raise exceptions.AuthenticationFailed('Un Authenticate')
        return (user, None)


from django.db import models

# Create your models here.
class Destinations(models.Model):
    name = models.CharField(max_length=100)
    img = models.ImageField(upload_to = 'pics')
    desc = models.TextField()
    price = models.IntegerField()
    offer = models.BooleanField(default = False)

class News(models.Model):
    date = models.DateTimeField(auto_now=True)
    name = models.CharField(max_length=100)
    desc = models.TextField()
    img = models.ImageField(upload_to = 'News/pics')


#!/usr/bin/python3
'''handlimng http errors using requests package'''
from requests import get
from sys import argv

if __name__ == "__main__":
    url = argv[1]
    with get(url) as response:
        if response.status_code >= 400:
            print('Error code: {}'.format(response.status_code))
        else:
            print(response.text)


import gzip
import json

import matplotlib
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

matplotlib.use('TkAgg')
# import matplotlib.pyplot as plt
# from nltk.corpus import stopwords

with gzip.open("goemotions.json.gz", "rb") as f:
    fullData = json.loads(f.read().decode("ascii"))

length = len(fullData)
emotions = [None] * length
posts = [None] * length
sentiments = [None] * length

for i in range(length):
    emotions[i] = fullData[i][1]
    posts[i] = fullData[i][0]
    sentiments[i] = fullData[i][2]

# distribution= [sentiments.count('positive')/length*100, sentiments.count('neutral')/length*100, sentiments.count('negative')/length*100, sentiments.count('ambiguous')/length*100]
# labels = 'Positive', 'Neutral', 'Negative', 'Ambiguous'
# explode = (0, 0.1, 0,0)  # only "explode" the 2nd slice (i.e. 'Hogs')
#
# plt.pie(distribution, explode=explode, labels=labels, autopct='%1.1f%%',
#         shadow=True, startangle=90)
# plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
#
# plt.savefig("sentiments.png")
# plt.rcParams.update({'font.size': 3})
#
# plt.hist(emotions, bins=28, alpha=0.5)
# plt.title('Emotions')
# plt.xlabel('X')
# plt.ylabel('Y')
#
# plt.show()

# Basic Naive Bayes

print('--------------BASIC NAIVE BAYES SENTIMENTS------------------------')
vectorizer = CountVectorizer(stop_words='english')
sentiments_encoded = sentiments
posts_encoded = vectorizer.fit_transform(posts)
print(vectorizer.vocabulary_)
print("The length of the vocabulary is " + str(len(vectorizer.vocabulary_)))
X_train, X_test, y_train, y_test = train_test_split(posts_encoded, sentiments_encoded, stratify=sentiments_encoded,
                                                    test_size=0.2, random_state=0)
classifier = MultinomialNB()
modelBE = classifier.fit(X_train, y_train)
predictions = classifier.predict(X_test)
f = open("performance.txt", "a")
f.write("Bayes Setiments Confusion Matrix")
f.write("\n")
f.write(str(confusion_matrix(y_test, predictions)))
f.write("\n")
f.write(str(classification_report(y_test, predictions)))
f.write("\n")
f.close()
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
print(accuracy_score(y_test, predictions))
print('------------------------------------------------------------------')


import os
import time

class Log:
    def __init__(self,save_path):
        time_str = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
        file_name = os.path.join(save_path,time_str)
        self.f = open(file_name,'w+')
        print('open file success')

    def log(self,txt):
        print(txt)
        self.f.write(txt)
        self.f.write('\n')

    def close(self):
        self.f.close()
        print('close file success')


import cv2
import numpy as np
import os

class FileStorage(object):
    def __init__(self, filename, isWrite=False):
        version = cv2.__version__
        self.major_version = int(version.split('.')[0])
        self.second_version = int(version.split('.')[1])

        if isWrite:
            os.makedirs(os.path.dirname(filename), exist_ok=True)
            self.fs = cv2.FileStorage(filename, cv2.FILE_STORAGE_WRITE)
        else:
            self.fs = cv2.FileStorage(filename, cv2.FILE_STORAGE_READ)

    def __del__(self):
        cv2.FileStorage.release(self.fs)

    def write(self, key, value, dt='mat'):
        if dt == 'mat':
            cv2.FileStorage.write(self.fs, key, value)
        elif dt == 'list':
            if self.major_version == 4: # 4.4
                self.fs.startWriteStruct(key, cv2.FileNode_SEQ)
                for elem in value:
                    self.fs.write('', elem)
                self.fs.endWriteStruct()
            else: # 3.4
                self.fs.write(key, '[')
                for elem in value:
                    self.fs.write('none', elem)
                self.fs.write('none', ']')

    def read(self, key, dt='mat'):
        if dt == 'mat':
            output = self.fs.getNode(key).mat()
        elif dt == 'list':
            results = []
            n = self.fs.getNode(key)
            for i in range(n.size()):
                val = n.at(i).string()
                if val == '':
                    val = str(int(n.at(i).real()))
                if val != 'none':
                    results.append(val)
            output = results
        else:
            raise NotImplementedError
        return output

    def close(self):
        self.__del__(self)

def read_intri(intri_name):
    assert os.path.exists(intri_name), intri_name
    intri = FileStorage(intri_name)
    camnames = intri.read('names', dt='list')
    cameras = {}
    for key in camnames:
        cam = {}
        cam['K'] = intri.read('K_{}'.format(key))
        cam['invK'] = np.linalg.inv(cam['K'])
        cam['dist'] = intri.read('dist_{}'.format(key))
        cameras[key] = cam
    return cameras

def write_intri(intri_name, cameras):
    intri = FileStorage(intri_name, True)
    results = {}
    camnames = list(cameras.keys())
    intri.write('names', camnames, 'list')
    for key_, val in cameras.items():
        key = key_.split('.')[0]
        K, dist = val['K'], val['dist']
        assert K.shape == (3, 3), K.shape
        assert dist.shape == (1, 5) or dist.shape == (5, 1), dist.shape
        intri.write('K_{}'.format(key), K)
        intri.write('dist_{}'.format(key), dist.reshape(1, 5))

def write_extri(extri_name, cameras):
    extri = FileStorage(extri_name, True)
    results = {}
    camnames = list(cameras.keys())
    extri.write('names', camnames, 'list')
    for key_, val in cameras.items():
        key = key_.split('.')[0]
        extri.write('R_{}'.format(key), val['Rvec'])
        extri.write('Rot_{}'.format(key), val['R'])
        extri.write('T_{}'.format(key), val['T'])
    return 0

def read_camera(intri_name, extri_name, cam_names=[]):
    assert os.path.exists(intri_name), intri_name
    assert os.path.exists(extri_name), extri_name

    intri = FileStorage(intri_name)
    extri = FileStorage(extri_name)
    cams, P = {}, {}
    cam_names = intri.read('names', dt='list')
    for cam in cam_names:
        # 内参只读子码流的
        cams[cam] = {}
        cams[cam]['K'] = intri.read('K_{}'.format( cam))
        cams[cam]['invK'] = np.linalg.inv(cams[cam]['K'])
        Rvec = extri.read('R_{}'.format(cam))
        Tvec = extri.read('T_{}'.format(cam))
        R = cv2.Rodrigues(Rvec)[0]
        RT = np.hstack((R, Tvec))

        cams[cam]['RT'] = RT
        cams[cam]['R'] = R
        cams[cam]['T'] = Tvec
        P[cam] = cams[cam]['K'] @ cams[cam]['RT']
        cams[cam]['P'] = P[cam]

        cams[cam]['dist'] = intri.read('dist_{}'.format(cam))
    cams['basenames'] = cam_names
    return cams

def write_camera(camera, path):
    from os.path import join
    intri_name = join(path, 'intri.yml')
    extri_name = join(path, 'extri.yml')
    intri = FileStorage(intri_name, True)
    extri = FileStorage(extri_name, True)
    results = {}
    camnames = [key_.split('.')[0] for key_ in camera.keys()]
    intri.write('names', camnames, 'list')
    extri.write('names', camnames, 'list')
    for key_, val in camera.items():
        if key_ == 'basenames':
            continue
        key = key_.split('.')[0]
        intri.write('K_{}'.format(key), val['K'])
        intri.write('dist_{}'.format(key), val['dist'])
        if 'Rvec' not in val.keys():
            val['Rvec'] = cv2.Rodrigues(val['R'])[0]
        extri.write('R_{}'.format(key), val['Rvec'])
        extri.write('Rot_{}'.format(key), val['R'])
        extri.write('T_{}'.format(key), val['T'])

def camera_from_img(img):
    height, width = img.shape[0], img.shape[1]
    # focal = 1.2*max(height, width) # as colmap
    focal = 1.2*min(height, width) # as colmap
    K = np.array([focal, 0., width/2, 0., focal, height/2, 0. ,0., 1.]).reshape(3, 3)
    camera = {'K':K ,'R': np.eye(3), 'T': np.zeros((3, 1)), 'dist': np.zeros((1, 5))}
    return camera

class Undistort:
    @staticmethod
    def image(frame, K, dist):
        return cv2.undistort(frame, K, dist, None)

    @staticmethod
    def points(keypoints, K, dist):
        # keypoints: (N, 3)
        assert len(keypoints.shape) == 2, keypoints.shape
        kpts = keypoints[:, None, :2]
        kpts = np.ascontiguousarray(kpts)
        kpts = cv2.undistortPoints(kpts, K, dist, P=K)
        keypoints[:, :2] = kpts[:, 0]
        return keypoints

    @staticmethod
    def bbox(bbox, K, dist):
        keypoints = np.array([[bbox[0], bbox[1], 1], [bbox[2], bbox[3], 1]])
        kpts = Undistort.points(keypoints, K, dist)
        bbox = np.array([kpts[0, 0], kpts[0, 1], kpts[1, 0], kpts[1, 1], bbox[4]])
        return bbox

def undistort(camera, frame=None, keypoints=None, output=None, bbox=None):
    # bbox: 1, 7
    print('This function is deprecated')
    raise NotImplementedError

def get_fundamental_matrix(cameras, basenames):
    skew_op = lambda x: np.array([[0, -x[2], x[1]], [x[2], 0, -x[0]], [-x[1], x[0], 0]])
    fundamental_op = lambda K_0, R_0, T_0, K_1, R_1, T_1: np.linalg.inv(K_0).T @ (
            R_0 @ R_1.T) @ K_1.T @ skew_op(K_1 @ R_1 @ R_0.T @ (T_0 - R_0 @ R_1.T @ T_1))
    fundamental_RT_op = lambda K_0, RT_0, K_1, RT_1: fundamental_op (K_0, RT_0[:, :3], RT_0[:, 3], K_1,
                                                                          RT_1[:, :3], RT_1[:, 3] )
    F = np.zeros((len(basenames), len(basenames), 3, 3))  # N x N x 3 x 3 matrix
    F = {(icam, jcam): np.zeros((3, 3)) for jcam in basenames for icam in basenames}
    for icam in basenames:
        for jcam in basenames:
            F[(icam, jcam)] += fundamental_RT_op(cameras[icam]['K'], cameras[icam]['RT'], cameras[jcam]['K'], cameras[jcam]['RT'])
            if F[(icam, jcam)].sum() == 0:
                F[(icam, jcam)] += 1e-12  # to avoid nan
    return F


user = input("Enter your name: ")
age = int(input("Enter your age: "))
print(f"The users' name is {user}, and they are {age} years old")
year2020 = 2020

while age < 100:
    print("You are not 100 years yet!")
    age += 1
    currentage = age
    year2020 += 1
    print(f"Your current age is {currentage}.")
    year100 = year2020
    if age >= 100:
        print(f"You are 100 years old in the year {year100}")


# Author: Daniele Rondina, geaaru@sabayonlinux.org
# Description:
# Note: Use PEP-0008 for indentation with max 79 chars on line.

import asyncio
import os
import traceback2
import uuid
import json
import datetime
import pytz

from sys import exit as start_shuttle
from concurrent.futures import CancelledError

from web_keepaliver.common import AppSeed, KafkaClientConfigurator
from web_keepaliver.db.postgres import PostgresConnector, PostgresDatabase

KEEPALIVER_KAFKA_CONSUMER_CONFIG = '/etc/web-keepaliver/kafka-consumer.yaml'
KEEPALIVER_KAFKA_CONSUMER_DEF_NAME = 'web-keepaliver-kafka-consumer'
KEEPALIVER_KAFKA_CONSUMER_DESC = """
Copyright (c) 2021 Daniele Rondina

Web Keepaliver Kafka Consumer.
"""

class KeepaliverKafkaConsumer(AppSeed,
                              KafkaClientConfigurator,
                              PostgresConnector):

    def __init__(self):
        self.kafka_brokers = []
        self.kafka_consumer = None
        self.msgs_counter = 0
        self.in_shutdown = False

        config_def_file = os.getenv(
            'KEEPALIVER_KAFKA_CONSUMER_CONFIG',
            KEEPALIVER_KAFKA_CONSUMER_CONFIG,
        )

        AppSeed.__init__(self, logger_name='kafka-consumer',
                         config_default_file=config_def_file,
                         app_description=KEEPALIVER_KAFKA_CONSUMER_DESC)
        KafkaClientConfigurator.__init__(self)
        PostgresConnector.__init__(self)

        # Avoid cancel tasks when kafka consumer is running.
        self._cancel_tasks = False

        self.get_parser().add_argument(
            '--broker',
            action='store',
            default=[],
            nargs='*',
            type=str,
            dest='kafka_brokers',
            help='Override kafka brokers defined in the configuration file.'
        )

    def get_postgres_conn_options(self):
        return self.get_config_category('postgres', {})

    def get_kafka_configuration(self, producer=True, admin=False):
        opts = self.get_config_category('kafka', {})

        if len(self.kafka_brokers) > 0:
            opts['bootstrap_servers'] = self.kafka_brokers

        opts['client_id'] = self.name
        return opts

    async def _init_consumer(self):
        self.kafka_consumer = self.create_kafka_consumer()

        brokers = 'localhost:9092'
        if len(self.kafka_brokers) > 0:
            brokers = self.kafka_brokers
        else:
            brokers = self.get_config_param(
                'kafka', 'bootstrap_servers', brokers
            )

        self.logger.info(
            'Starting instance %s with kafka brokers %s.',
            self.name, brokers,
        )

        await self.kafka_consumer.start()

        # Subscripte topics
        self.kafka_consumer.subscribe(pattern=self.get_config_param(
            None, 'subscription_regex', "web-keepaliver"
        ))
        # NOTE: when the group coardinator is not yet present in the
        # kafka broker aiokafka library wrote this error:
        # Group Coordinator Request failed:
        # [Error 15] CoordinatorNotAvailableError
        #
        # This error is automatically handled by the library with
        # retries and after 2/3 seconds all works correctly.

    async def _initialize(self):
        self.name = self.get_config_param(
            None, 'name', KEEPALIVER_KAFKA_CONSUMER_DEF_NAME
        )

        # New aiokafka package has deprecated loop argument.
        # It's needed to ensure the is used the right loop object.
        asyncio.set_event_loop(loop=self.loop)

        await self._init_consumer()
        await self.create_pool()

    def signal_handler(self, signame):
        self.in_shutdown = True

    async def _process_site_events(self, site, topic, messages):
        last_event = messages[len(messages)-1]
        site_probes = []
        status = {}

        self.logger.info(
            "[%s] Processing %d events from topic %s.", site,
            len(messages), topic,
        )

        try:
            for event in messages:
                if 'probes' not in event:
                    self.logger.warning(
                        "[%s] Received invalid event without probes from "
                        "topic %s", site, topic,
                    )
                    continue

                probes = event['probes']

                all_events_ok = True
                for p in probes:
                    if 'resp_ts' in p:
                        ts = datetime.datetime.fromtimestamp(
                            int(p['resp_ts'])/1000.0,
                        )
                    else:
                        # If resp_ts is not present then
                        # we use the kafka message timestamp.
                        ts = datetime.datetime.fromtimestamp(
                            p.timestamp/1000.0,
                        )

                    ts = datetime.datetime(ts.year, ts.month, ts.day,
                                           hour=ts.hour,
                                           minute=ts.minute,
                                           second=ts.second,
                                           tzinfo=pytz.utc)

                    site_probes.append((
                        ts, site,
                        p['resource'],
                        p['method'],
                        p['url'],
                        p['resp_http_code'],
                        p['resp_time_ms'],
                        p['expected_http_code'],
                        p['result'],
                        p['error_desc'] if 'error_desc' in p else None,
                    ))

                    if not p['result']:
                        all_events_ok = False

                if event == last_event:
                    last_update = datetime.datetime.utcnow()
                    # Prepare status entry
                    status = {
                        'site': site,
                        'last_update': last_update,
                        'status': all_events_ok,
                        'n_resources': len(event['probes']),
                        'err_counter': 0 if all_events_ok else False,
                    }
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or prepare records on db: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )

        con = await self.acquire()
        try:
            # Register probes
            await PostgresDatabase.register_probes(con, site_probes)

            # Update site status
            await PostgresDatabase.register_site_status(con, status)
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or register records on db: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )
        finally:
            await self.release(con)

    async def _parse_topic_messages(self, tp, messages):

        try:
            site_events_map = {}
            for message in messages:

                jmsg = json.loads(str(message.value.decode('ascii')))

                self.msgs_counter += 1
                self.logger.info(
                    "[%s/%d/%d] [%s] Received message %s",
                    tp.topic, tp.partition,
                    message.offset,
                    str(uuid.UUID(bytes=message.key)),
                    str(message.value.decode('ascii')) if self.debug
                    else 'with %d events' % len(jmsg['probes'])
                )

                # Organize messages for site
                if jmsg['site'] not in site_events_map:
                    site_events_map[jmsg['site']] = [jmsg]
                else:
                    site_events_map[jmsg['site']].append(jmsg)

            futures = []
            for site in site_events_map:
                events = site_events_map[site]
                futures.append(asyncio.ensure_future(
                    self._process_site_events(site, tp.topic, events)
                ))

            await asyncio.gather(
                *futures,
                return_exceptions=True
            )
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or parse topic messages: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )

    async def _async_main(self):
        await self._initialize()

        self.logger.info(
            "Instance %s started.", self.name
        )

        try:
            while not self.in_shutdown:
                self.logger.debug("Waiting for messages...")

                data = await self.kafka_consumer.getmany(
                    timeout_ms=self.get_config_param(
                        None, 'timeout_waiting_data',
                        10000,
                    )
                )
                futures = []
                for tp, messages in data.items():
                    future = asyncio.ensure_future(
                        self._parse_topic_messages(
                            tp, messages,
                        )
                    )
                    futures.append(future)

                await asyncio.ensure_future(
                    asyncio.gather(
                        *futures,
                        return_exceptions=True,
                        loop=self.loop,
                    )
                )

        except CancelledError:
            pass

        await self.kafka_consumer.commit()

        # TODO: Verify why with the unsubscribe i receive error on stop
        #  consumer.
        # Error: Commit offset before unsubscribe to avoid
        # UnknownMemberIdError
        # self.kafka_consumer.unsubscribe()

    async def _cleanup(self):
        if self.kafka_consumer:
            await self.kafka_consumer.stop()
        await self.postgres_cleanup()

    def main(self, parse_cmdline_opts=True):
        ans = 0
        if parse_cmdline_opts:
            self.parse_cmd_line()

        try:
            self.init_thread_pool(size=self.get_config_param(
                'general', 'thread_pool_size', 20,
            ))
            self.add_signal_handler()
            self.loop.run_until_complete(
                asyncio.ensure_future(self._async_main())
            )
        except KeyboardInterrupt:
            pass
        except CancelledError:
            pass
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines()) if self.debug
                else ''
            )
            ans = 1

        # Running closing operation of the producer
        self.loop.run_until_complete(self._cleanup())

        self.logger.info(
            "Instance %s stopped.", self.name
        )
        self.loop.close()

        return ans

def main():
    in_space = KeepaliverKafkaConsumer()
    start_shuttle(in_space.main())

if __name__ == '__main__':
    main()

# vim: ts=4 sw=4 expandtab


import re
import sys
from google.oauth2 import service_account
from google.auth.transport import requests
import base64
from pathlib import Path
import argparse
from typing import Optional
from typing import Sequence
import json
import os

API_BASE_URL='https://backstory.googleapis.com'
SCOPES = ['https://www.googleapis.com/auth/chronicle-backstory']

def get_http_client_from_file(service_account_file):
  # Create a credential using Google Developer Service Account Credential and Chronicle API scope.
  credentials = service_account.Credentials.from_service_account_file(service_account_file, scopes=SCOPES)
  # Build an HTTP client to make authorized OAuth requests.
  return requests.AuthorizedSession(credentials)

def get_http_client_from_sa_info(service_account_info):
  service_account_info = os.environ.get("SA_CREDENTIAL", "Not Found")
  # Create a credential using Google Developer Service Account Credential and Chronicle API scope.
  credentials = service_account.Credentials.from_service_account_info(json.loads(service_account_info), scopes=SCOPES)
  # Build an HTTP client to make authorized OAuth requests.
  return requests.AuthorizedSession(credentials)

def get_http_client_from_sa_env(env_var_name):
  service_account_info = os.environ.get(env_var_name)
  # Create a credential using Google Developer Service Account Credential and Chronicle API scope.
  credentials = service_account.Credentials.from_service_account_info(json.loads(service_account_info), scopes=SCOPES)
  # Build an HTTP client to make authorized OAuth requests.
  return requests.AuthorizedSession(credentials)

def initialize_command_line_args(
    args: Optional[Sequence[str]] = None) -> Optional[argparse.Namespace]:
  """Initializes and checks all the command-line arguments."""
  parser = argparse.ArgumentParser()
  parser.add_argument(
    "-c", "--credentials_file", type=str, help="path to credentials file")
  parser.add_argument(
    "-i", "--credentials_info", type=str, help="service account credentials info (alternative to file)")
  parser.add_argument(
    "-e", "--credentials_env", type=str, help="service account credentials info stored in environment variable")
  parser.add_argument(
    "-l", "--local_path", type=str, required=True, help="local rules path")
  parser.add_argument(
    "-m", "--make_changes", action='store_true',  help="fix any differences, if omitted then just report differences" )
  parser.add_argument(
    "-s", "--silent", action='store_true',  help="supress error messages" )
  parser.add_argument(
    "-r", "--region", required=False, default="us", choices=("asia-southeast1", "europe", "us"), help="Chronicle instance region (leave blank for US)" )

  parser.set_defaults(make_changes=False)
  return parser.parse_args(args)

def get_all_rules(session):
  try:
    rules_response = []
    url=f'{API_BASE_URL}/v2/detect/rules'
    response = session.request(
      method="GET",
      url=url,
    )
    if response.json():
      rules_array = response.json()
      for rule in rules_array['rules']:
        rules_response.append({
          'name': rule['ruleName'],
          'rule_id': rule['ruleId'],
          'text': base64.b64encode(rule['ruleText'].strip().encode('utf-8')).decode('utf-8'),
          'text_raw': rule['ruleText'].strip(),
        })
    return rules_response
  except:
    print(f'Received error code: {response.status_code}')

def load_local_files(path):
  rules_response = []
  file_list = list(Path(path).rglob("*.[yY][aA][rR][aA][lL]"))
  for file in file_list:
    file_content = open(file, "r").read()
    rules_response.append({
      'name': parse_rule_name(file_content), #str(file.stem),
      'path': str(file),
      'text': base64.b64encode(file_content.strip().encode('utf-8')).decode('utf-8'),
      'text_raw': file_content.strip(),
    })
  return rules_response

def parse_rule_name(rule_text):
  rule_name = re.search("rule ([A-Za-z0-9_]+)[\r\n\s]?{", rule_text)
  if rule_name:
    return rule_name.group(1)
  else:
    return False

def compare_rules(session, local_rules_path, silent=False):
  count_non_existent_rules = 0
  count_matched_rules = 0
  count_rules_to_update = 0
  local_rules = load_local_files(local_rules_path)
  remote_rules = get_all_rules(session)
  for local_rule in local_rules:
    if remote_rules:
      matched_remote_rules = [x for x in remote_rules if x['name'] == local_rule['name']]
      if len(matched_remote_rules) > 0:
        if len(matched_remote_rules) != 1:
          print(f'Matched rules was not equal to 1 for {local_rule["name"]}')
        else:
          if matched_remote_rules[0]['text'] == local_rule['text']:
            count_matched_rules += 1
          else:
            count_rules_to_update += 1
      else:
        count_non_existent_rules += 1
    else:
      count_non_existent_rules += 1
  if remote_rules:
    count_remote_rules = len(remote_rules)
  else:
    count_remote_rules = 0
  if local_rules:
    count_local_rules = len(local_rules)
  else:
    count_local_rules = 0
  return {
    "matched_rules": count_matched_rules,
    "rules_to_update": count_rules_to_update,
    "non_existent_rules": count_non_existent_rules,
    "remote_rules_total": count_remote_rules,
    "local_rules_total": count_local_rules,
  }

def upload_missing_rules(session, local_rules_path, silent=False):
  rules_updated = 0
  rules_added = 0
  error_count = 0
  errors = []
  local_rules = load_local_files(local_rules_path)
  remote_rules = get_all_rules(session)
  for local_rule in local_rules:
    if local_rule['name']:
      matched_remote_rules = [x for x in remote_rules if x['name'] == local_rule['name']]
      if len(matched_remote_rules) > 0:
        if len(matched_remote_rules) != 1:
          print(f'upload_missing_rules / Matched rules was not equal to 1 for {local_rule["name"]}')
        else:
          try:
            if matched_remote_rules[0]['text'] != local_rule['text']:
              if update_rule(session,local_rule['text_raw'],matched_remote_rules[0]['rule_id'],local_rule['name'], silent):
                rules_updated += 1
              else:
                error_count += 1
          except:
            if not silent: print(f"upload_missing_rules / Update of rule {local_rule['name']} failed")
            errors.append(f"upload_missing_rules / Update of rule {local_rule['name']} failed")
            error_count += 1
      else:
        try:
          if create_rule(session,local_rule['text_raw'],local_rule['name'],silent):
            rules_added += 1
          else:
            error_count += 1
        except:
          if not silent: print(f"upload_missing_rules / Creation of rule {local_rule['name']} failed")
          errors.append(f"upload_missing_rules / Creation of rule {local_rule['name']} failed")
          error_count += 1
    else:
      if not silent: print(f"upload_missing_rules / rule with path {local_rule['path']} has an issue")
      errors.append(f"upload_missing_rules / Creation of rule {local_rule['path']} failed")
      error_count += 1

  return {
    "rules_added": rules_added,
    "rules_updated": rules_updated,
    "errors": error_count,
    "error_details": errors,
  }

def verify_rule(session,rule_text,rule_name,silent=False):
  try:
    url=f'{API_BASE_URL}/v2/detect/rules:verifyRule'
    body={"ruleText":rule_text}
    response = session.request(
      method="POST",
      url=url,
      json=body
    )
    response.raise_for_status()
    return True
  except:
    if not silent: print(f'verify_rule [{rule_name}]/ Received error code: {response.status_code}')
    return False

def create_rule(session,rule_text,rule_name, silent=False):
  if verify_rule(session,rule_text,rule_name,silent):
    try:
      url=f'{API_BASE_URL}/v2/detect/rules'
      body={"ruleText":rule_text}
      response = session.request(
        method="POST",
        url=url,
        json=body
      )
      response.raise_for_status()
      return True
    except:
      if not silent: print(f'create_rule / Received error code: {response.status_code}')
      return False
  else:
    return False

def update_rule(session,rule_text,rule_id,rule_name,silent=False):
  if verify_rule(session,rule_text,rule_name,silent):
    try:
      url=f'{API_BASE_URL}/v2/detect/rules/{rule_id}:createVersion'
      body={"ruleText":rule_text}
      response = session.request(
        method="POST",
        url=url,
        json=body
      )
      response.raise_for_status()
      return True
    except:
      if not silent: print(f'update_rule / Received error code: {response.status_code}')
      return False
  else:
    return False

if __name__ == "__main__":
  cli = initialize_command_line_args()
  if not cli:
    sys.exit(1)  # A sanity check failed.
  if not cli.region == "us":
    API_BASE_URL=f'https://{cli.region}-backstory.googleapis.com'
  if cli.credentials_file:
    session = get_http_client_from_file(cli.credentials_file)
  elif cli.credentials_info:
    session = get_http_client_from_sa_info(cli.credentials_info)
  elif cli.credentials_env:
    session = get_http_client_from_sa_env(cli.credentials_env)
  else:
    print("No credential passed")
  if not cli.make_changes:
    print(json.dumps(compare_rules(session,cli.local_path, cli.silent),indent=2))
  else:
    print(json.dumps(upload_missing_rules(session,cli.local_path, cli.silent),indent=2))


import json
import os
import re
import shutil

import numpy as np

def replace_digits_zero():
    """ replace every digit in a string to zero"""

    if not os.path.exists('./data/no_digit'):
        os.mkdir('./data/no_digit')

    raw_paths = ['./data/raw/train.txt',
                 './data/raw/valid.txt', './data/raw/test.txt']
    processed_paths = ['./data/no_digit/train.txt',
                       './data/no_digit/valid.txt', './data/no_digit/test.txt']

    for raw_path, processed_path in zip(raw_paths, processed_paths):
        with open(raw_path, 'r') as f:
            l = f.readlines()
        tmp_l = [l[0], l[1]]
        for line in l[2:]:
            tmp_str = re.sub(r'\d+', '0', line)
            tmp_l.append(tmp_str)

        with open(processed_path, 'w') as f:
            for line in tmp_l:
                f.write(line)

def counter():
    """ count the number of words and tags """
    WORD, TAG = dict(), dict()
    with open('./data/no_digit/train.txt', 'r') as f:
        l = f.readlines()

    for line in l[2:]:  # ignore the redundant top 2 lines
        line = line
        if line == '\n':
            continue
        word, tag = line.split()[0].lower(), line.split()[3]
        try:
            WORD[word] += 1
        except KeyError:
            WORD[word] = 1
        try:
            TAG[tag] += 1
        except KeyError:
            TAG[tag] = 1

    # sort
    TAG = sorted(TAG.items(), key=lambda x: x[1], reverse=True)
    WORD = sorted(WORD.items(), key=lambda x: x[1], reverse=True)

    with open('./data/tag_cnt.txt', 'w') as f:
        for item in TAG:
            f.write(item[0] + '\t' + str(item[1]) + '\n')

    return list(map(lambda x: x[0], WORD))

def build_vocab(words):
    """ build vocabulary for words, chars and tags """
    word2id, char2id = dict(), dict()

    # word vocab
    word2id['pad'] = 0
    word2id['unk'] = 1
    idx = 2
    for word in words:
        if word not in word2id:
            word2id[word] = idx
            idx += 1

    # char vocab
    char2id['pad'] = 0
    char_list = ['!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/',
                 '0', ':', ';', '=', '?', '@', '[', ']', '`',
                 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
                 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
    idx = 1
    for c in char_list:
        char2id[c] = idx
        idx += 1

    # tag vocab
    tag2id = {'O': 0, 'B-ORG': 1, 'B-MISC': 2, 'B-PER': 3, 'B-LOC': 4,
              'I-ORG': 5, 'I-MISC': 6, 'I-PER': 7, 'I-LOC': 8}

    vocab = {'tag2id': tag2id, 'char2id': char2id, 'word2id': word2id}
    with open('./data/vocab.json', 'w') as f:
        json.dump(vocab, f)

    return vocab

def process_data():
    if not os.path.exists('./data/dataset'):
        os.mkdir('./data/dataset')

    corpus_paths = ['./data/no_digit/train.txt',
                    './data/no_digit/valid.txt', './data/no_digit/test.txt']
    dest_paths = ['./data/dataset/train.json',
                  './data/dataset/valid.json', './data/dataset/test.json']

    corpus = []
    for corpus_path in corpus_paths:
        with open(corpus_path, 'r') as f:
            l = f.readlines()
        sent_list, tags_list, sent_tmp, tag_tmp, samples = [], [], [], [], []
        for line in l[2:]:  # ignore the redundant top 2 lines
            if line == '\n':  # blank lines separate each sentence
                assert len(sent_tmp) == len(tag_tmp)  # sanity check
                sent_list.append(' '.join(sent_tmp))
                tags_list.append(' '.join(tag_tmp))
                sent_tmp.clear()
                tag_tmp.clear()
                continue
            word, tag = line.split()[0], line.split()[3]
            sent_tmp.append(word)
            tag_tmp.append(tag)

        # print({len(sent_list)})  # 14986, 3465, 3683

        for i in range(len(sent_list)):
            samples.append({'sentence': sent_list[i], 'tags': tags_list[i]})

        corpus.append(samples)

    for i, path in enumerate(dest_paths):
        with open(path, 'w') as f:
            json.dump(corpus[i], f)

def load_pretrained_emb(glove_path, embed_dim, vocab):
    words, vectors = [], []

    idx = 0
    word2id = dict()
    with open(glove_path, 'rb') as f:
        for l in f:
            line = l.decode().split()
            word = line[0]
            words.append(word)
            word2id[word] = idx
            vectors.append(np.array(line[1:]).astype(np.float))
            idx += 1

    glove = {w: vectors[word2id[w]] for w in words}

    id2word = {key: value for value, key in vocab['word2id'].items()}
    weights_matrix = np.zeros((len(id2word), embed_dim))

    for idx, word in id2word.items():
        idx = int(idx)
        try:
            weights_matrix[idx] = glove[word.lower()]
        except KeyError:
            weights_matrix[idx] = np.random.normal(
                scale=0.6, size=(embed_dim,))

    np.save('./data/weights_matrix.npy', weights_matrix)

if __name__ == "__main__":
    replace_digits_zero()
    words = counter()
    vocab = build_vocab(words)
    process_data()
    load_pretrained_emb(glove_path='./data/raw/glove.6B.100d.txt',
                        embed_dim=100,
                        vocab=vocab)
    shutil.rmtree('./data/no_digit')


import argparse
import os
import time
from datetime import datetime

import torch
from torch import nn
from torch.utils.data import TensorDataset, DataLoader
from tqdm import tqdm
from tst import Transformer

from seq2seq_transduction.data import load_dataset
from seq2seq_transduction.utils import time_since, plot_loss_history

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def train(dataset_path, batch_size, num_epochs, learning_rate):
    normalized_carrier_train, normalized_params_train, normalized_modulated_train, _, _ = load_dataset(dataset_path, kind="train")
    train_dataset = TensorDataset(torch.from_numpy(normalized_carrier_train).float(),
                            torch.from_numpy(normalized_params_train).float(),
                            torch.from_numpy(normalized_modulated_train).float())
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=1,
                             pin_memory=True, drop_last=True)

    normalized_carrier_validate, normalized_params_validate, normalized_modulated_validate, _, _ = load_dataset(dataset_path, kind="validate")
    validate_dataset = TensorDataset(torch.from_numpy(normalized_carrier_validate).float(),
                            torch.from_numpy(normalized_params_validate).float(),
                            torch.from_numpy(normalized_modulated_validate).float())
    validate_loader = DataLoader(validate_dataset, batch_size=batch_size, shuffle=True, num_workers=1,
                             pin_memory=True, drop_last=True)

    p = {
        "d_model": 64,  # Latent dim
        "q": 8,         # Query size
        "v": 8,         # Value size
        "h": 8,         # Number of heads
        "N": 3,         # Number of encoder and decoder to stack
        "attention_size": 12,  # Attention window size
        "dropout": 0.3,        # Dropout rate
        "pe": None,            # Positional encoding
        "chunk_mode": None,
        "d_input": normalized_carrier_train.shape[-1] + normalized_params_train.shape[-1],
        "d_output": normalized_modulated_train.shape[-1],
        "batch_size": batch_size,
        "seq_len": normalized_carrier_train.shape[1]
    }

    model = Transformer(p["d_input"], p["d_model"], p["d_output"], p["q"], p["v"], p["h"], p["N"], p["attention_size"],
                        p["dropout"], p["chunk_mode"], p["pe"]).to(device)
    output_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir, "output", "transformer")
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    loss_history = train_loop(model, train_loader, validate_loader, num_epochs, learning_rate)
    timestamp = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S")
    torch.save({"parameters": p, "state_dict": model.state_dict()}, os.path.join(output_dir, f"{timestamp}_model.pt"))
    plot_loss_history(loss_history, os.path.join(output_dir, "{}_loss.png".format(timestamp)), labels=["Train", "Validate"])

def train_loop(model, train_loader, val_loader, n_epochs, learning_rate):
    start = time.time()
    loss_history = []

    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-4)
    criterion = nn.MSELoss()

    for epoch in range(1, n_epochs + 1):
        train_loss = 0
        model.train()
        for carrier_sig, params, modulated_sig in tqdm(train_loader):
            param_seq_batch = params.unsqueeze(1).repeat(1, carrier_sig.size(1), 1)
            input_seq_batch = torch.cat((param_seq_batch, carrier_sig), dim=-1).to(device)
            target_seq_batch = modulated_sig.to(device)
            batch_loss = train_step(input_seq_batch, target_seq_batch, model, optimizer, criterion, validate=False)
            train_loss += batch_loss

        avg_train_loss = train_loss / len(train_loader)
        print('%s (%d %d%%) %.4f' % (time_since(start, epoch / n_epochs),
                                     epoch, epoch / n_epochs * 100, avg_train_loss))

        val_loss = 0
        model.eval()
        with torch.no_grad():
            for carrier_sig, params, modulated_sig in tqdm(val_loader):
                param_seq_batch = params.unsqueeze(1).repeat(1, carrier_sig.size(1), 1)
                input_seq_batch = torch.cat((param_seq_batch, carrier_sig), dim=-1).to(device)
                target_seq_batch = modulated_sig.to(device)
                batch_loss = train_step(input_seq_batch, target_seq_batch, model, optimizer, criterion, validate=True)
                val_loss += batch_loss
        avg_val_loss = val_loss / len(val_loader)

        loss_history.append([avg_train_loss, avg_val_loss])
    return loss_history

def train_step(input_tensor, target_tensor, model, optimizer, criterion, validate=False):
    output = model(input_tensor)
    loss = criterion(output, target_tensor)
    if not validate:
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return loss.item()

def main(args):
    batch_size = 32
    learning_rate = 5e-4
    num_epochs = 100
    train(args.dataset_path, batch_size, num_epochs, learning_rate)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("dataset_path", type=str)
    main(parser.parse_args())


import threading
import time
from HelloThread import thQueue
from HelloThread import dataQueue

class myThread(threading.Thread):
    def __init__(self, threadId, name):
        threading.Thread.__init__(self)
        self.threadId = threadId
        self.name = name

    def setQueue(self, dataQueue):
        self.dataQueue = dataQueue

    def setLock(self, lock):
        self.lock = lock

    def run(self):
        global thQueue
        global lock
        print('start myThread')
        print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        thQueue.acquire()
        lock.put(self.name)
        thQueue.release()
        time.sleep(2)
        print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        print('end myThread')


from typing import List

# See details here https://wenshengchen.com/2020/01/21/221-maximal-square.html
class Solution:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        maximal = 0
        for row in range(len(matrix)):
            for col in range(len(matrix[0])):
                maximal = max(maximal, int(matrix[row][col]))

                if row > 0 and col > 0:
                    square = min(int(matrix[row-1][col]), int(matrix[row-1][col-1]), int(matrix[row][col-1]))
                    if square > 0 and matrix[row][col] == "1":
                        matrix[row][col] = str(square + 1)
                        maximal = max(maximal, int(matrix[row][col]))

        return maximal**2

## TEST CASES
test = Solution()
assert test.maximalSquare([["1","0","1","0","0"],["1","0","1","1","1"],["1","1","1","1","1"],["1","0","0","1","0"]]) == 4
assert test.maximalSquare([["1","0","1","1","1"],["1","0","1","1","1"],["1","1","1","1","1"],["1","0","0","1","0"]]) == 9
assert test.maximalSquare([["0","0","0","0","0"],["0","0","0","0","0"],["0","0","0","0","0"],["0","0","0","1","0"]]) == 1
assert test.maximalSquare([["1"]]) == 1
assert test.maximalSquare([[]]) == 0
assert test.maximalSquare(None) == 0
assert test.maximalSquare([["1","1"],["1","1"]]) == 4
print('All Passed!')


import unittest
from app import app
from unittest.mock import patch, MagicMock

class ChatAppTestCase(unittest.TestCase):

    def setUp(self):
        self.app = app.test_client()
        self.app.testing = True

    @patch('app.pusher_client.authenticate')
    def test_pusher_authentication(self, mock_authenticate):
        mock_authenticate.return_value = {'auth': 'test_auth_token'}

        response = self.app.post('/api/chat/auth/', data={
            'email': 'user@example.com',
            'channel_name': 'test_channel',
            'socket_id': '12345'
        })
        self.assertEqual(response.status_code, 200)
        self.assertIn('test_auth_token', response.json['auth'])

    def test_get_pusher_config(self):
        response = self.app.get('/api/chat/config/')
        self.assertEqual(response.status_code, 200)
        self.assertIn('key', response.json)
        self.assertIn('cluster', response.json)

    @patch('app.pusher.trigger')
    @patch('app.DynamoMessageRepo.create')
    def test_send_messages(self, mock_repo_create, mock_pusher_trigger):
        mock_pusher_trigger.return_value = None
        mock_repo_create.return_value = None

        response = self.app.post('/api/chat/message/', json={
            'message': 'Hello',
            'userEmail': 'sender@example.com',
            'receiverEmail': 'receiver@example.com',
            'userInfo': {'name': 'Sender'},
            'receiverInfo': {'name': 'Receiver',}
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json['status'], 'success')

    @patch('app.DynamoMessageRepo.get_all')
    def test_get_history(self, mock_get_all):
        mock_get_all.return_value = {"results": []}

        response = self.app.post('/api/chat/history/', json={
            'userEmail': 'user@example.com',
            'receiverEmail': 'receiver@example.com'
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, [])

    @patch('app.FriendRepo.get_all_friends')
    def test_get_friend_list(self, mock_get_all_friends):
        mock_get_all_friends.return_value = {"friends": []}

        response = self.app.post('/api/chat/friendlist/', json={'userEmail': 'user@example.com'})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, [])

    @patch('app.pusher.trigger')
    @patch('app.FriendRepo.add_friend')
    def test_add_friend(self, mock_add_friend, mock_pusher_trigger):
        mock_add_friend.return_value = None
        mock_pusher_trigger.return_value = None

        response = self.app.post('/api/chat/addfriend/', json={
            'userEmail': 'user@example.com',
            'userData': {'name': 'User'},
            'friendData': {'email': 'friend@example.com', 'name': 'Friend'}
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, [])

    @patch('app.pusher.trigger')
    @patch('app.FriendRepo.add_friend')
    def test_add_multi_friend(self, mock_add_friend, mock_pusher_trigger):
        mock_add_friend.return_value = None
        mock_pusher_trigger.return_value = None

        response = self.app.post('/api/chat/addfriend-multi/', json={
            'userEmail': 'user@example.com',
            'userData': {'name': 'User'},
            'friendData': [{'email': 'friend1@example.com', 'name': 'Friend1'}, {'email': 'friend2@example.com', 'name': 'Friend2'}]
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, [])

    @patch('app.FriendRepo.update_current_chat')
    def test_update_last_chat(self, mock_update_current_chat):
        mock_update_current_chat.return_value = None

        response = self.app.post('/api/chat/lastchat/', json={
            'userEmail': 'user@example.com',
            'receiverEmail': 'receiver@example.com'
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, [])

    @patch('app.FriendRepo.update_friend_data_for_others')
    def test_update_user_info(self, mock_update_friend_data_for_others):
        mock_update_friend_data_for_others.return_value = None

        response = self.app.post('/api/chat/updateuser/', json={
            'userEmail': 'user@example.com',
            'newData': {'name': 'Updated User'}
        })
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, {'status': 'success', 'message': 'User data updated successfully'})


"""==================================================================================================="""
### Basic Libraries
import warnings
warnings.filterwarnings("ignore")
import os, sys, numpy as np, argparse, imp, datetime, pandas as pd, copy
import time, pickle as pkl, random, json, collections

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
from tqdm import tqdm

### Repository-specific Libraries
import parameters    as par
import utilities.misc as misc

"""==================================================================================================="""
parser = argparse.ArgumentParser()

parser = par.basic_training_parameters(parser)
parser = par.batch_creation_parameters(parser)
parser = par.batchmining_specific_parameters(parser)
parser = par.loss_specific_parameters(parser)
parser = par.wandb_parameters(parser)
parser = par.s2sd_parameters(parser)
parser = par.maxentropy_parameters(parser)
parser.add_argument('--data_hardness', default=1, type=int, help='OOD-ness of datasets. Ranges/Default id are [CUB200-2011, (1-9), 3], [CARS196, (1-8), 1], [SOP, (1-8), 1]')
parser.add_argument('--new_test_dataset', default='None', type=str, help='Optionally evaluate on the test dataset of a different dataset.')
##### Read in parameters
opt = parser.parse_args()

"""==================================================================================================="""
### The following setting is useful when logging to wandb and running multiple seeds per setup:
### By setting the savename to <group_plus_seed>, the savename will instead comprise the group and the seed!
if opt.savename=='group_plus_seed':
    if opt.log_online:
        opt.savename = opt.group+'_s{}'.format(opt.seed)
    else:
        opt.savename = ''

### Easy checkpointing
start_from_checkpoint = False
if opt.checkpoint:
    init_save_path = copy.deepcopy(opt.save_path)
    checkpath      = opt.save_path+'/'+opt.dataset+'/'+opt.savename
    if os.path.exists(checkpath):
        if any(['checkpoint.pth.tar'==x for x in os.listdir(checkpath)]):
            load_opt = pkl.load(open(checkpath+'/hypa.pkl', 'rb'))
            new_run  = False
            for key,item in vars(opt).items():
                if key!='load_to_ram':
                    if item!=vars(load_opt)[key] and key not in ['source_path', 'save_path', 'completed']:
                        new_run = True

            if not new_run:
                opt             = load_opt
                opt.save_path   = init_save_path
                opt.source_path = '/'.join(opt.source_path.split('/')[:-1])
                start_from_checkpoint = True

if opt.completed:
    print('\n\nTraining Run already completed!')
    exit()

### If wandb-logging is turned on, initialize the wandb-run here:
if opt.log_online:
    import wandb
    _ = os.system('wandb login {}'.format(opt.wandb_key))
    os.environ['WANDB_API_KEY'] = opt.wandb_key
    if not start_from_checkpoint:
        opt.unique_run_id = wandb.util.generate_id()
    wandb.init(id=opt.unique_run_id, resume='allow', project=opt.project, group=opt.group, name=opt.savename, dir=opt.source_path)
    wandb.config.update(opt)

"""==================================================================================================="""
### Load Remaining Libraries that need to be loaded after wandb
import torch, torch.nn as nn, torch.nn.functional as F
import torch.multiprocessing
import torchvision
torch.multiprocessing.set_sharing_strategy('file_system')
import architectures as archs
import datasampler   as dsamplers
import datasets      as dataset_library
import criteria      as criteria
import metrics       as metrics
import batchminer    as bmine
import evaluation    as eval
from utilities import misc
from utilities import logger

"""==================================================================================================="""
full_training_start_time = time.time()

opt.source_path += '/'+opt.dataset
opt.save_path   += '/'+opt.dataset

#Assert that the construction of the batch makes sense, i.e. the division into class-subclusters.
assert not opt.bs%opt.samples_per_class, 'Batchsize needs to fit number of samples per class for distance sampling and margin/triplet loss!'

opt.pretrained      = not opt.not_pretrained
opt.evaluate_on_gpu = not opt.evaluate_on_cpu

os.environ["CUDA_DEVICE_ORDER"]   ="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]= str(opt.gpu[0])

if start_from_checkpoint:
    chkp_data = torch.load(opt.save_path+'/'+opt.savename+'/checkpoint.pth.tar')
misc.set_seed(opt.seed)

"""==================================================================================================="""
##################### NETWORK SETUP ##################
opt.device = torch.device('cuda')
model      = archs.select(opt.arch, opt)

if opt.fc_lr<0:
    to_optim   = [{'params':model.parameters(),'lr':opt.lr,'weight_decay':opt.decay}]
else:
    all_but_fc_params = [x[-1] for x in list(filter(lambda x: 'last_linear' not in x[0], model.named_parameters()))]
    fc_params         = model.model.last_linear.parameters()
    to_optim          = [{'params':all_but_fc_params,'lr':opt.lr,'weight_decay':opt.decay},
                         {'params':fc_params,'lr':opt.fc_lr,'weight_decay':opt.decay}]

if start_from_checkpoint:
    opt       = chkp_data['opt']
    _ = model.load_state_dict(chkp_data['state_dict'])

_  = model.to(opt.device)

"""============================================================================"""
# Code needed for Uniform-Prior regularization (Sinha et al., 2020)
class Discriminator(nn.Module):
    def __init__(self, in_dim, latent):
        super(Discriminator, self).__init__()
        self.in_dim = in_dim
        self.fc1 = nn.Linear(in_dim, latent)
        self.fc2 = nn.Linear(latent, latent)
        self.fc3 = nn.Linear(latent, 1)
    def forward(self, x):
        x = x.view(x.size(0), -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = torch.sigmoid(self.fc3(x))
        return x

if opt.with_entropy:
    discr = Discriminator(opt.maxentropy_chunksize, opt.maxentropy_latent)
    if start_from_checkpoint:
        _ = discr.load_state_dict(chkp_data['aux']['discr_state_dict'])
    _ = discr.to(opt.device)

    optimizer_discr     = torch.optim.Adam(discr.parameters(), lr=opt.maxentropy_lrmulti*opt.lr, betas=(0.5, 0.999))
    if start_from_checkpoint:
        optimizer_discr.load_state_dict(chkp_data['aux']['optimizer_discr_state_dict'])
    scheduler_discr = torch.optim.lr_scheduler.MultiStepLR(optimizer_discr, milestones=opt.maxentropy_tau, gamma=opt.maxentropy_gamma)
    if start_from_checkpoint:
        scheduler_discr.load_state_dict(chkp_data['aux']['scheduler_discr_state_dict'])

"""============================================================================"""
#################### DATALOADER SETUPS ##################
datasets    = dataset_library.select(opt.dataset, opt, opt.source_path)
if hasattr(model, 'specific_normalization'):
    for dataset_type in datasets.keys():
        if datasets[dataset_type] is not None:
            datasets[dataset_type].normal_transform_list[-1] = model.specific_normalization
            datasets[dataset_type].normal_transform = torchvision.transforms.Compose(
                datasets[dataset_type].normal_transform_list
            )

if opt.new_test_dataset != 'None':
    init_dataset = copy.deepcopy(opt.dataset)
    init_sourcepath = copy.deepcopy(opt.source_path)
    opt.source_path = opt.source_path.replace('/' + opt.dataset, '/' + opt.new_test_dataset)
    opt.dataset = opt.new_test_dataset
    datasets_temp = dataset_library.select(opt.new_test_dataset, opt, opt.source_path)
    datasets['testing'] = datasets_temp['testing']
    opt.dataset = init_dataset
    opt.source_path = init_sourcepath

dataloaders = {}
dataloaders['evaluation'] = torch.utils.data.DataLoader(datasets['evaluation'], num_workers=opt.kernels, batch_size=opt.bs, shuffle=False)
dataloaders['testing']    = torch.utils.data.DataLoader(datasets['testing'],    num_workers=opt.kernels, batch_size=opt.bs, shuffle=False)

if opt.use_tv_split:
    dataloaders['validation'] = torch.utils.data.DataLoader(datasets['validation'], num_workers=opt.kernels, batch_size=opt.bs,shuffle=False)

train_data_sampler      = dsamplers.select(opt.data_sampler, opt, datasets['training'].image_dict, datasets['training'].image_list)
if train_data_sampler.requires_storage:
    train_data_sampler.create_storage(dataloaders['evaluation'], model, opt.device)

dataloaders['training'] = torch.utils.data.DataLoader(datasets['training'], num_workers=opt.kernels, batch_sampler=train_data_sampler)

opt.n_classes  = len(dataloaders['training'].dataset.avail_classes)
opt.n_test_classes = len(dataloaders['testing'].dataset.avail_classes)

"""============================================================================"""
#################### CREATE LOGGING FILES ###############
sub_loggers = ['Train', 'Test', 'Model Grad']
if opt.use_tv_split: sub_loggers.append('Val')

if start_from_checkpoint:
    LOG = chkp_data['progress']
else:
    LOG = logger.LOGGER(opt, sub_loggers=sub_loggers, start_new=True, log_online=opt.log_online)

"""============================================================================"""
#################### Basic DML Loss Setup ####################
batchminer   = bmine.select(opt.batch_mining, opt)
criterion, to_optim = criteria.select(opt.loss, opt, to_optim, batchminer)

if start_from_checkpoint:
    _ = criterion.load_state_dict(chkp_data['aux']['criterion'][-1])

_ = criterion.to(opt.device)

if 'criterion' in train_data_sampler.name:
    train_data_sampler.internal_criterion = criterion

"""============================================================================"""
#################### OPTIM SETUP ####################
if opt.optim == 'adam':
    optimizer    = torch.optim.Adam(to_optim)
else:
    raise Exception('Optimizer <{}> not available!'.format(opt.optim))

if start_from_checkpoint:
    _ = optimizer.load_state_dict(chkp_data['aux']['optimizer'])

scheduler    = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=opt.tau, gamma=opt.gamma)
if start_from_checkpoint:
    scheduler.load_state_dict(chkp_data['aux']['scheduler'])

"""============================================================================"""
#################### METRIC COMPUTER ####################
metric_computer = metrics.MetricComputer(opt.evaluation_metrics, opt)

"""============================================================================"""
################### Summary #########################3
chkp_text  = 'RESTARTED FROM CHKP: {} {}\n'.format(start_from_checkpoint, '- at epoch {}'.format(opt.epoch+1) if 'epoch' in vars(opt) else '')
data_text  = 'Dataset:\t {}'.format(opt.dataset.upper())
setup_text = 'Objective:\t {}'.format(opt.loss.upper())
miner_text = 'Batchminer:\t {}'.format(opt.batch_mining if criterion.REQUIRES_BATCHMINER else 'N/A')
arch_text  = 'Backbone:\t {} (#weights: {})'.format(opt.arch.upper(), misc.gimme_params(model))
summary    = chkp_text+'\n'+data_text+'\n'+setup_text+'\n'+miner_text+'\n'+arch_text
print(summary)

"""============================================================================"""
################### SCRIPT MAIN ##########################
print('\n-----\n')

iter_count = 0
include_aux_augmentations = dataloaders['training'].dataset.include_aux_augmentations
loss_args  = {'batch':None, 'labels':None, 'batch_features':None, 'f_embed':None}
init_lrs = [param_group['lr'] for param_group in optimizer.param_groups[1:]]

if not start_from_checkpoint:
    opt.epoch  = 0
else:
    if 'epoch' in vars(opt):
        opt.epoch += 1
epochs = range(opt.epoch, opt.n_epochs)

for epoch in epochs:
    opt.epoch = epoch
    epoch_start_time = time.time()
    #Set seeds for each epoch - this ensures reproducibility after resumption.
    misc.set_seed(opt.n_epochs*opt.seed+epoch)

    ### Scheduling Changes specifically for cosine scheduling
    if opt.scheduler!='none':
        print('Running with learning rates {}...'.format(' | '.join('{}'.format(x['lr']) for x in optimizer.param_groups)))

    ### Train one epoch
    start = time.time()
    _ = model.train()

    loss_collect = []
    data_iterator = tqdm(dataloaders['training'], desc='Epoch {} Training...'.format(epoch))

    for i,out in enumerate(data_iterator):
        class_labels, input, input_indices = out

        input      = input.to(opt.device)
        model_args = {'x':input.to(opt.device), 'warmup':epoch<opt.warmup}

        out_dict  = model(**model_args)
        embeds, avg_features, features, extra_embeds = [out_dict[key] for key in ['embeds', 'avg_features', 'features', 'extra_embeds']]

        loss_args['input_batch']    = input
        loss_args['batch']          = embeds
        loss_args['extra_batches']  = extra_embeds
        loss_args['labels']         = class_labels
        loss_args['f_embed']        = model.model.last_linear
        loss_args['batch_features'] = features
        loss_args['avg_batch_features'] = avg_features
        loss_args['model']          = model
        loss_args['optim']          = optimizer

        loss      = criterion(**loss_args)

        if opt.with_entropy:
            # Apply adversarial uniformity regularization.
            if iter_count % opt.maxentropy_iter == 0:
                # DSC STEP
                embeddings = avg_features.view(-1, opt.maxentropy_chunksize).detach()
                z          = torch.rand(embeddings.size(0), opt.maxentropy_chunksize).to(opt.device) * 2. - 1.
                ones       = torch.ones(embeddings.size(0), 1).to(opt.device)
                preds_real = discr(z)
                real_loss  = F.binary_cross_entropy(preds_real, ones)

                zeros      = torch.zeros(embeddings.size(0), 1).to(opt.device)
                preds_fake = discr(embeddings.detach())
                fake_loss  = F.binary_cross_entropy(preds_fake, zeros)
                dsc_loss   = real_loss + fake_loss

                optimizer_discr.zero_grad()
                dsc_loss.backward()
                optimizer_discr.step()

            embeddings = avg_features.view(-1, opt.maxentropy_chunksize)
            ones       = torch.ones(embeddings.size(0), 1).to(opt.device)
            preds_fake = discr(embeddings)
            adv_loss   = F.binary_cross_entropy(preds_fake, ones)
            loss += opt.maxentropy_w * adv_loss

        """========================="""
        optimizer.zero_grad()
        loss.backward()

        ### Compute Model Gradients and log them!
        grads = np.concatenate([p.grad.detach().cpu().numpy().flatten() for p in model.parameters() if p.grad is not None])
        grad_l2, grad_max  = np.mean(np.sqrt(np.mean(np.square(grads)))), np.mean(np.max(np.abs(grads)))

        LOG.progress_saver['Model Grad'].log('Grad L2',  grad_l2,  group='L2')
        LOG.progress_saver['Model Grad'].log('Grad Max', grad_max, group='Max')

        ### Update network weights!
        optimizer.step()

        ###
        loss_collect.append(loss.item())

        ###
        iter_count += 1

        data_iterator.set_postfix_str('Loss: {0:.4f}'.format(np.mean(loss_collect)))
        if i==len(dataloaders['training'])-1: data_iterator.set_description('Epoch (Train) {0}: Mean Loss [{1:.4f}]'.format(epoch, np.mean(loss_collect)))

    ####
    result_metrics = {'loss': np.mean(loss_collect)}

    ####
    LOG.progress_saver['Train'].log('epochs', epoch)
    for metricname, metricval in result_metrics.items():
        LOG.progress_saver['Train'].log(metricname, metricval)
    LOG.progress_saver['Train'].log('time', np.round(time.time()-start, 4))

    ### Learning Rate Scheduling Step
    if opt.scheduler != 'none':
        scheduler.step()
    if opt.with_entropy:
        scheduler_discr.step()

    """======================================="""
    ### Evaluate Metric for Training & Test (& Validation)
    _ = model.eval()
    print('\nComputing Testing Metrics...')

    aux_store = {'optimizer':optimizer.state_dict(), 'criterion':(criterion, criterion.state_dict()),
                 'scheduler':scheduler.state_dict(), 'dataloaders':dataloaders,
                 'discr_state_dict':discr.state_dict() if opt.checkpoint and opt.with_entropy else None,
                 'optimizer_discr_state_dict':optimizer_discr.state_dict() if opt.checkpoint and opt.with_entropy else None,
                 'scheduler_discr_state_dict':scheduler_discr.state_dict() if opt.checkpoint and opt.with_entropy else None,
                 'datasets':datasets, 'train_data_sampler':train_data_sampler}

    eval.evaluate(opt.dataset, LOG, metric_computer, [dataloaders['testing']], model, opt, opt.evaltypes, opt.device, log_key='Test', aux_store=aux_store)
    if opt.use_tv_split:
        print('\nComputing Validation Metrics...')
        eval.evaluate(opt.dataset, LOG, metric_computer, [dataloaders['validation']], model, opt, opt.evaltypes, opt.device, log_key='Val', aux_store=aux_store)

    if not opt.no_train_metrics:
        print('\nComputing Training Metrics...')
        eval.evaluate(opt.dataset, LOG, metric_computer, [dataloaders['evaluation']], model, opt, opt.evaltypes, opt.device, log_key='Train', aux_store=aux_store)

    import wandb
    LOG.update(all=True)

    eval.set_checkpoint(model, opt, LOG, LOG.prop.save_path+'/checkpoint.pth.tar', aux=aux_store)

    print('\nTotal Epoch Runtime: {0:4.2f}s'.format(time.time()-epoch_start_time))
    print('\n-----\n')

"""======================================================="""
opt.completed = True
pkl.dump(opt,open(opt.save_path+"/hypa.pkl","wb"))


import requests

def find_long_url(url):
    try:
        response = requests.head(url, allow_redirects=True)
        long_url = response.url
        return long_url
    except requests.exceptions.RequestException as e:
        print("Error occurred:", e)
        return None


#!/usr/bin/env python
"""Simple analysis of pie QMC output files.

By default data will be aggregated into a single output file with analysed_
prefixed to input filename.
"""
import argparse
import json
import os
import sys

import pandas as pd

_script_dir = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(_script_dir, "analysis"))
import glob

from ipie.analysis.blocking import (
        analyse_estimates,
        reblock_minimal,
        average_fp
        )
from ipie.analysis.extraction import extract_data_sets

def parse_args(args):
    """Parse command-line arguments.

    Parameters
    ----------
    args : list of strings
        command-line arguments.

    Returns
    -------
    options : :class:`argparse.ArgumentParser`
        Command line arguments.
    """

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "-s",
        "--start",
        type=float,
        dest="start_time",
        default=0,
        help="Imaginary time in a.u. after which we " "gather statistics.  Default: 0",
    )
    parser.add_argument(
        "-b",
        "--block-start",
        type=int,
        dest="block_start",
        default=0,
        help="Simulation block to start blocking analysis from (equilibration "
        "time): Default 0",
    )
    parser.add_argument(
        "-m",
        "--multi-sim",
        action="store_true",
        dest="multi_sim",
        default=False,
        help="Average over multiple simulations. By default "
        "an attempt is made to group results by features.",
    )
    parser.add_argument(
        "--free-proj",
        action="store_true",
        default=False,
        dest="free_proj",
        help="Free projection error analysis using mulitple" " simulations",
    )
    parser.add_argument(
        "--legacy",
        action="store_true",
        default=False,
        dest="legacy",
        help="Perform legacy heavyweight analysis.",
    )
    parser.add_argument(
        "-t",
        "--average-tau",
        action="store_true",
        dest="av_tau",
        default=False,
        help="Compute average of energy as a function of tau.",
    )
    parser.add_argument(
        "-c",
        "--correlation",
        dest="cfunc",
        action="store_true",
        default=False,
        help="Extract correlation functions.",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        default=False,
        help="Analyse all all energy components." " Default False.",
    )
    parser.add_argument(
        "-f",
        nargs="+",
        dest="filenames",
        help="Space-separated list of files to analyse.",
    )

    options = parser.parse_args(args)

    if not options.filenames:
        parser.print_help()
        sys.exit(1)

    return options

def main(args):
    """Run reblocking and data analysis on pie output.

    Parameters
    ----------
    args : list of strings
        command-line arguments.

    Returns
    -------
    None.
    """

    options = parse_args(args)
    if "*" in options.filenames[0]:
        files = glob.glob(options.filenames[0])
    else:
        files = options.filenames
    if options.free_proj:
        data = extract_data_sets(files, "basic", "energies")
        res = average_fp(data)
        print(res.to_string(index=False))
    elif options.legacy:
        res = analyse_estimates(
            files,
            start_time=options.start_time,
            multi_sim=options.multi_sim,
            av_tau=options.av_tau,
            verbose=options.verbose,
        )
        fmt = lambda x: "{:13.8f}".format(x)
        print(res.to_string(index=False, float_format=fmt))
    else:
        results = reblock_minimal(
            files,
            start_block=options.block_start,
            verbose=options.verbose,
        )
        fmt = lambda x: "{:13.8f}".format(x)
        print(results.to_string(index=False, float_format=fmt))

if __name__ == "__main__":
    main(sys.argv[1:])


from bs4 import BeautifulSoup
import urllib.request
from urllib.parse import quote
import pandas

date = []

def main():
    query = input('검색어 : ')
    s_date = input('시작날짜 (2019.05.15) : ')
    e_date = input('끝날짜 (2019.05.15) : ')

    get_news(query, s_date, e_date)

def get_news(query, s_date, e_date):
    date_index = pandas.date_range(start=s_date, end=e_date)
    date = date_index.strftime("%Y.%m.%d").to_list()

    for i in date:
        t_query = quote(query)

        url = "https://search.naver.com/search.naver?where=news&query=" + t_query + "&sort=0&photo=0&field=0&reporter_article=&pd=3&ds=" + i + "&de=" + i

        news = []
        while True:
            news_keyword_url = urllib.request.urlopen(url).read()
            soup = BeautifulSoup(news_keyword_url, 'html.parser')

            link_all = soup.find_all('dd', {'class': 'txt_inline'})

            for link in link_all:
                link_find_a = link.find('a')
                link_find_href = link_find_a.get('href')

                # 네이버 뉴스 url이 없는 경우, 크롤링하지 않음
                if (link_find_href == '#'):
                    continue

                # 네이버 뉴스 url이 있는 경우, 뉴스 본문 크롤링
                else:
                    news_link = urllib.request.urlopen(link_find_href).read()
                    news_link_html = BeautifulSoup(news_link, 'html.parser')

                    for item in news_link_html.find_all('div', {'id': 'articleBodyContents'}):
                        result = item.get_text()
                        result = result.replace("// flash 오류를 우회하기 위한 함수 추가", "")
                        result = result.replace("function _flash_removeCallback() {}", "")
                        result = result.replace("\n", "")
                        result = result.replace("\t", "")

                        news.append(result)
                    news.append("\n\n")

            try :
                # paging에서 다음페이지 버튼에 연결된 url 찾음
                next = soup.find('a', {'class': 'next'})
                next_href = next.get('href')

            except AttributeError :
                # 맨 마지막장에 다음페이지 버튼이 없어서 에러
                # 이 때, loop를 break;
                break

            # 다음페이지에서 뽑은 url
            url = 'https://search.naver.com/search.naver' + next_href
            print(url)

        # 키워드를 통해 존재하는 모든 네이버 뉴스기사를 txt파일로 저장
        f = open('C:/Users/gpqls/OneDrive/바탕 화면/4/cap/' + query + '(' + i.replace(".","") + ')' +'.txt', 'a', -1, 'utf-8')
        for text in news :
            f.write(text)
        f.close()

main()


# Your first task at your new job is to create a table of the stock in a bakery, and you really don't want to fail on
# your first day at work.
# You will receive a single line containing some food (keys) and quantities (values).
# They will be separated by a single space (the first element is the key, the second – the value, and so on).
# Create a dictionary with all the keys and values and print it on the console.

# Version 1:

# elements_list = input().split()
# bakery = {}
# keys = []
# values = []
# for index in range(0, len(elements_list)):
#     if index % 2 == 0:
#         keys.append(elements_list[index])
#     else:
#         values.append(int(elements_list[index]))
#
# bakery = dict(zip(keys, values))
#
# print(bakery)

# Version 2:

elements_list = input().split()
bakery = {}

for index in range(0, len(elements_list), 2):
    key = elements_list[index]
    value = elements_list[index + 1]

    bakery[key] = int(value)

print(bakery)


from menu import Menu, MenuItem
from coffee_maker import CoffeeMaker
from money_machine import MoneyMachine

menu = Menu()
money_things = MoneyMachine()
coffee_maker = CoffeeMaker()

should_continue = True

while should_continue:
    user_input = input(f"What would you like? {menu.get_items()}:")
    if user_input == "off":
        print("Turning off the machine.")
        should_continue = False
    elif user_input == "report":
        coffee_maker.report()
        money_things.report()
    else:
        order = menu.find_drink(user_input)
        cost = menu.find_drink(user_input).cost
        if coffee_maker.is_resource_sufficient(order):
            if money_things.make_payment(cost):
                coffee_maker.make_coffee(order)