import math
from numpy.linalg import inv
from scipy.stats import multivariate_normal

class BiGauss(object):
    """
    Implementation of Measuring the component overlapping in the Gaussian mixture model by Sun and Wang (2011)
    Modified code from https://github.com/SJinping/GMM-overlap-rate
    """
    def __init__(self, mu1, mu2, Sigma1, Sigma2, pi1, pi2, steps = 100):
        super(BiGauss, self).__init__()
        self.mu1      = mu1
        self.mu2      = mu2
        self.Sigma1   = Sigma1
        self.Sigma2   = Sigma2
        self.pi1      = pi1
        self.pi2      = pi2

        self.biGauss1 = multivariate_normal(mean = self.mu1, cov = self.Sigma1, allow_singular = True)
        self.biGauss2 = multivariate_normal(mean = self.mu2, cov = self.Sigma2, allow_singular = True)

        self.inv_Sig1 = -inv(self.Sigma1)
        self.inv_Sig2 = -inv(self.Sigma2)

        self.steps    = steps

        # variables to calculate RC
        self.A_1 = self.inv_Sig1[0][0]
        self.B_1 = self.inv_Sig1[0][1]
        self.C_1 = self.inv_Sig1[1][0]
        self.D_1 = self.inv_Sig1[1][1]
        self.A_2 = self.inv_Sig2[0][0]
        self.B_2 = self.inv_Sig2[0][1]
        self.C_2 = self.inv_Sig2[1][0]
        self.D_2 = self.inv_Sig2[1][1]

    def pdf(self, x):
        return self.pi1 * self.biGauss1.pdf(x) + self.pi2 * self.biGauss2.pdf(x)

    def overlap_rate(self):
        e      = math.sqrt((self.mu1[0] - self.mu2[0])**2 + (self.mu1[1] - self.mu2[1])**2) / float(self.steps)
        x_step = e*(self.mu1[0]-self.mu2[0]) # each step for x
        y_step = e*(self.mu1[1]-self.mu2[1]) # each step for y
        p_x    = self.mu1[0] - x_step

        while self.RC(p_x) == None:
            p_x = p_x - x_step

        p_y   = self.RC(p_x)
        p     = [p_x, p_y]
        p_pre = self.mu1
        p_min = min(self.pdf(p), self.pdf(p_pre))
        p_max = max(self.pdf(p), self.pdf(p_pre))
        index = 0
        while index < self.steps:
            if self.RC(p[0] - x_step) != None:
                p_next = [p[0] - x_step, self.RC(p[0] - x_step)] # next point on ridge curve
                if self.pdf(p) > self.pdf(p_pre) and self.pdf(p) > self.pdf(p_next):
                    p_max = self.pdf(p)
                if self.pdf(p) < self.pdf(p_pre) and self.pdf(p) < self.pdf(p_next):
                    p_min = self.pdf(p)
            p_pre = p
            p     = p_next
            index += 1

        pdf_mu1 = self.pdf(self.mu1)
        pdf_mu2 = self.pdf(self.mu2)
        return p_min / min(pdf_mu1, pdf_mu2) if p_min < min(pdf_mu1, pdf_mu2) else 1.0

    # get y given x, satisfying (x,y) is on ridge curve (RC)
    def RC(self, x):
        E = self.A_1 * (x - self.mu1[0])
        F = self.C_1 * (x - self.mu1[0])
        G = self.A_2 * (x - self.mu2[0])
        H = self.C_2 * (x - self.mu2[0])

        I = E * self.D_2 - F * self.B_2
        J = H * self.B_1 - G * self.D_1
        K = self.B_1 * self.D_2 - self.B_2 * self.D_1
        M = F * G - E * H

        P = K
        Q = I + J - K * (self.mu2[1] + self.mu1[1])
        S = -(M + I * self.mu2[1] + J * self.mu1[1])

        if Q**2 - 4*P*S < 0:
            return None

        y = max((-Q + math.sqrt(Q**2 - 4*P*S)) / (2*P), (-Q - math.sqrt(Q**2 - 4*P*S)) / (2*P))

        return y

def calc_overlap_rate(mu1, mu2, Sigma1, Sigma2, pi1, pi2):
    bg = BiGauss(mu1, mu2, Sigma1, Sigma2, pi1, pi2)
    return bg.overlap_rate()

# def calc_overlap_rate(pi1 = 0.5, mu1 = [0, 0], Sigma1 = [[1, 0], [0, 1]], pi2=0.5, mu2 = [3, 0], Sigma2 = [[2.17, 1.82], [1.82, 2.17]]):
#
#     Bi = BiGauss(mu1, mu2, Sigma1, Sigma2, pi1, pi2)
#     steps = Bi.steps
#
#     e = math.sqrt((mu1[0] - mu2[0]) ** 2 + (mu1[1] - mu2[1]) ** 2) / float(steps)
#     x_step = e * (mu1[0] - mu2[0])  # each step for x
#     y_step = e * (mu1[1] - mu2[1])  # each step for y
#     p_x = mu1[0] - x_step
#     while Bi.RC(p_x) == None:
#         p_x = p_x - x_step
#     p_y = Bi.RC(p_x)
#     p = [p_x, p_y]
#     p_pre = mu1
#     p_min = min(Bi.pdf(p), Bi.pdf(p_pre))
#     p_max = max(Bi.pdf(p), Bi.pdf(p_pre))
#     index = 0
#     while index < steps:
#         if Bi.RC(p[0] - x_step) != None:
#             p_next = [p[0] - x_step, Bi.RC(p[0] - x_step)]  # next point on ridge curve
#             if Bi.pdf(p) > Bi.pdf(p_pre) and Bi.pdf(p) > Bi.pdf(p_next):
#                 p_max = Bi.pdf(p)
#             if Bi.pdf(p) < Bi.pdf(p_pre) and Bi.pdf(p) < Bi.pdf(p_next):
#                 p_min = Bi.pdf(p)
#         p_pre = p
#         p = p_next
#         index += 1
#     pdf_mu1 = Bi.pdf(mu1)
#     pdf_mu2 = Bi.pdf(mu2)
#     print('the OLR of this mixture Gaussian: {}'.format(
#         p_min / min(pdf_mu1, pdf_mu2) if p_min < min(pdf_mu1, pdf_mu2) else 1.0))

# if __name__ == '__main__':
#     import matplotlib.pyplot as plt
#     # Example from Eq. 20
#     pi1 = 0.5
#     pi2 = 1.0 - pi1
#     mu1 = [0,0]
#     mu2 = [3,0]
#     Sigma1 = [[1,0],[0,1]]
#     Sigma2 = [[2.17,1.82], [1.82,2.17]]
#     Bi = BiGauss(mu1, mu2, Sigma1, Sigma2, pi1, pi2)
#     print('the OLR of this mixture Gaussian: {}'.format(Bi.overlap_rate()))
#
#     exit(0)
#
#     # OLR's changing over distance between two means (Fig. 7)
#     step = 0.1
#     pi1 = 0.5
#     OLRs = []
#     x = []
#     mu1 = [0,0]
#     mu2 = [0,0]
#     Sigma1 = [[1,0],[0,1]]
#     Sigma2 = [[2.17,1.82], [1.82,2.17]]
#     while mu2[0] <= 8.0:
#         pi2 = 1.0-pi1
#         Bi = BiGauss(mu1, mu2, Sigma1, Sigma2, pi1, pi2)
#         OLRs.append(Bi.overlap_rate())
#         x.append(mu2[0])
#         mu2[0] += step
#     plt.plot(x, OLRs)
#     plt.xlabel('Distance between two means')
#     plt.show()
#
#     # OLR's changing over pi1 (Fig. 8)
#     step = 0.05
#     pi1 = 0.0
#     OLRs = []
#     x = []
#     mu1 = [0,0]
#     mu2 = [3,0]
#     Sigma1 = [[1,0],[0,1]]
#     Sigma2 = [[2.17,1.82], [1.82,2.17]]
#     while pi1 <= 1.0:
#         pi2 = 1.0-pi1
#         Bi = BiGauss(mu1, mu2, Sigma1, Sigma2, pi1, pi2)
#         OLRs.append(Bi.overlap_rate())
#         x.append(pi1)
#         pi1 += step
#     print(OLRs)
#     plt.plot(x, OLRs)
#     plt.xlabel('Coefficient(%) of {}'.format(r"$\pi_1$"))
#     plt.show()


from flask import Flask, request, Response, redirect
import base64
import jsonpickle
from datetime import datetime
from dateExt import dateExtractor
from textExt import TesseractTxtExtractor
from preProc import preProcessor
import pandas as pd
import os
from PIL import Image
import cv2 as cv

app = Flask(__name__)
pp = preProcessor()
tx = TesseractTxtExtractor()
dx = dateExtractor()

@app.route("/extract_date", methods=['GET','POST'])
def index():
    if request.method == "POST":
        try:

            r = request
            image_64_decode = base64.decodestring(r.data)
            image_result = open('recieved_img.jpeg', 'wb')
            image_result.write(image_64_decode)
            mtx = cv.imread('recieved_img.jpeg')

            # Writ logic here to perform manipulation on image
            mtx = pp.rescale(mtx)
            content = tx.extractTxt(mtx)
            date = dx.extractDate(content)

            # Return response:
            response = {'date' : date }
            # encode response using jsonpickle
            response_pickled = jsonpickle.encode(response)

        except Exception as e:
            response = {'date': 'null'}
            response_pickled = jsonpickle.encode(response)
            return Response(response=response_pickled, status=200, mimetype="application/json")
        # print(r.data)

    elif request.method=="GET":
        response_pickled = "Make a POST request and see the result on your console"

    return Response(response=response_pickled, status=200, mimetype="application/json")

    # else:
    #     return Response(response="No Data", status=200, mimetype="application/json")

if __name__ == "__main__":
    app.run(debug = True, host="0.0.0.0")


from matplotlib import pyplot as plt
from sklearn.metrics import precision_score, recall_score, accuracy_score, classification_report, confusion_matrix
import numpy as np

label = ['mel', 'akiec', 'bcc', 'nv', 'bkl']

def plot_confusion_matrix(y_true, y_pred, classes, normalize=True,
                          title=None,
                          cmap=plt.cm.Blues):
    cm = confusion_matrix(y_true, y_pred)
    cm2 = cm.astype('float') / cm.sum(axis=0)[:, np.newaxis]
    cm2 = cm2 * 100
    print(cm)

    fig, ax = plt.subplots(figsize=(10, 10))
    im = ax.imshow(cm2, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    ax.set(xticks=np.arange(cm2.shape[1]),
           yticks=np.arange(cm2.shape[0]),
           xticklabels=label, yticklabels=label,
           title=title,
           ylabel='Predicted label',
           xlabel='True label')

    plt.setp(ax.get_xticklabels(), rotation=0, ha="right",
             rotation_mode="anchor")

    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            z = cm[i, j]
            z = np.around(z)

            ax.text(j, i, format(z),
                    ha="center", va="center",
                    color="white" if i == j else "black")
    fig.tight_layout()
    return ax


import os

from ansiblelint import AnsibleLintRule
try:
    from ansible.module_utils.parsing.convert_bool import boolean
except ImportError:
    try:
        from ansible.utils.boolean import boolean
    except ImportError:
        try:
            from ansible.utils import boolean
        except ImportError:
            from ansible import constants
            boolean = constants.mk_boolean

class CommandsInsteadOfModulesRule(AnsibleLintRule):
    id = '303'
    shortdesc = 'Using command rather than module'
    description = 'Executing a command when there is an Ansible module ' + \
                  'is generally a bad idea'
    tags = ['command-shell', 'resources', 'ANSIBLE0006']

    _commands = ['command', 'shell']
    _modules = {
        'apt-get': 'apt-get',
        'chkconfig': 'service',
        'curl': 'get_url or uri',
        'git': 'git',
        'hg': 'hg',
        'mount': 'mount',
        'patch': 'patch',
        'rpm': 'yum or rpm_key',
        'rsync': 'synchronize',
        'sed': 'template, replace or lineinfile',
        'service': 'service',
        'supervisorctl': 'supervisorctl',
        'svn': 'subversion',
        'systemctl': 'systemd',
        'tar': 'unarchive',
        'unzip': 'unarchive',
        'wget': 'get_url or uri',
        'yum': 'yum',
    }

    def matchtask(self, file, task):
        if task['action']['__ansible_module__'] not in self._commands:
            return

        if 'cmd' in task['action']:
            first_cmd_arg = task['action']['cmd'].split()[0]
        else:
            first_cmd_arg = task['action']['__ansible_arguments__'][0]
        if not first_cmd_arg:
            return

        executable = os.path.basename(first_cmd_arg)
        if executable in self._modules and \
                boolean(task['action'].get('warn', True)):
            message = '{0} used in place of {1} module'
            return message.format(executable, self._modules[executable])


#!/usr/bin/env python
# coding: utf-8

# In[0]:

import pandas as pd
import re
import itertools
import sys
import flag

# In[1]:
# read list of stopwords from text file
def readStopWords(fileName='stopwords.txt'):
    fd = open(fileName)
    line = fd.read()
    stopwords = line.splitlines()
    fd.close()
    return stopwords

# In[2]:
# preprocess input text
def removePuncs(text, stopwords):
    # convert to lower case
    text = text.lower()

    # convert text to a list of words
    listWord = text.split()

    # remove punctuations and digits
    puncs = r"[!\(\)\-\[\]\{\};'\":\\,<>\./\?@#\$%^&\*_~0-9]"
    listWord = [re.sub(puncs,'',word) for word in listWord]
    listWord = [word for word in listWord if word not in stopwords and len(word)>0]

    return listWord

# In[3]:
# generate "n grams" (nC3, nC2, nC1) for given list of terms
def n_grams(terms):
    three_grams = [' '.join(i) for i in itertools.combinations(terms, 3)]
    two_grams = [' '.join(i) for i in itertools.combinations(terms, 2)]
    one_grams = [' '.join(i) for i in itertools.combinations(terms, 1)]
    return three_grams, two_grams, one_grams

# In[4]:
# get a list of locations for each n gram combination, if any
def get_locations(terms, cities='worldcities.csv'):
	# read cities file
    df = pd.read_csv(cities)
    cols = ['city', 'city_ascii','state', 'country', 'iso2', 'iso3']

    # convert columns to lowercase
    for c in cols:
        df[c] = df[c].str.lower()
    locations = pd.DataFrame(columns=cols)

    # iterate through columns and "n gram" combinations
    for c in cols:
        for t in terms:
            if t in df[c].values and t not in locations[c].values:
                data = df[df[c] == t][cols].iloc[0]
                loc = {}
                # if "n gram" is a city, return city, state, country and iso codes
                if c == 'city':
                    loc['city'] = data['city']
                    loc['city_ascii'] = data['city_ascii']
                    loc['state'] = data['state']
                    loc['country'] = data['country']
                    loc['iso2'] = data['iso2']
                    loc['iso3'] = data['iso3']
                # if "n gram" is a state, return state, country and iso codes
                elif c == 'state':
                    loc['state'] = t
                    loc['country'] = data['country']
                    loc['iso2'] = data['iso2']
                    loc['iso3'] = data['iso3']
                # if "n gram" is a country, return country and iso codes
                elif c == 'country':
                    loc['country'] = t
                    loc['iso2'] = data['iso2']
                    loc['iso3'] = data['iso3']
                # if "n gram" is an ISO code, return country and iso codes
                elif c == 'iso2' or c == 'iso3':
                    loc['country'] = data['country']
                    loc['iso2'] = data['iso2']
                    loc['iso3'] = data['iso3']

                locations = locations.append(loc, ignore_index=True)

    # remove duplicates, if any and tidy up results
    locations = locations.drop_duplicates()
    for c in cols:
        if c == 'iso2' or c == 'iso3':
            locations[c] = locations[c].str.upper()
        else:
            locations[c] = locations[c].str.title()

    # return list of locations
    return locations.to_dict(orient='records')

# In[5]:
def main():
    text = sys.argv[1]

    # handle flag emojis in the text if any
    place = flag.dflagize(text)
    text_size = len(place)
    start_index = place.find(':')
    end_index = -1
    start_prev, end_prev = 0, 0
    rep = {}

    while(start_index > -1):
        start_index += 1
        end_index = start_index + 1 + place[start_index+1:].find(':')
        emoji_loc = place[start_index:end_index]
        rep[emoji_loc] = (start_index+start_prev-1, end_index+end_prev)
        start_prev, end_prev = start_index, end_index
        place = place[end_index+1:]
        start_index = place.find(':')

    for key in rep.keys():
        text = text.replace(text[rep[key][0]:rep[key][1]], key)

    # remove stopwords
    stopwords = readStopWords(fileName='stopwords.txt')

    # generate location terms
    terms = removePuncs(text, stopwords)

    # generate n_grams from these terms (combination of terms basically)
    three_grams, two_grams, one_grams = n_grams(terms)

    # get unique locations from these n_grams
    locations = get_locations(three_grams)
    if len(locations) == 0:
        locations = get_locations(two_grams)
        if len(locations) == 0:
            locations = get_locations(one_grams)

    # return locations as an array of dictionary
    return locations

print(main())


#!/usr/bin/env python3
import os
import json
import datetime

data = open('/Users/james/Desktop/WeChatDB/db8/Chat_c361fce587751cedffb3e34d40fddb6b.json').read()
data = json.loads(data)

dict = {}
for item in data:
    # 获取消息发送时间
    unixtime = (item['msgCreateTime'])
    if isinstance(unixtime, int):
        value = datetime.datetime.fromtimestamp(unixtime).strftime('%H:%M')
        # 如果分钟数小于 30 则把后两位变成 00
    if int(value[3:]) < 30:
        value = value[:2]+'00'
        # 如果分钟数大于 30 则把后两位变成 30
    else:
        value = value[:2]+'30'
        # 如果存在这个时间点就把计数 +1
    if dict.get(value) is not None:
        dict[value] = dict.get(value)+1
        # 如果不存在这个时间点就初始化计数 = 1
    else:
        dict[value] = 1

# 开始对 dict 时刻分区结果排序
sort_result = sorted(dict.items(), key=lambda x:x[0])
# 存储排序之后的结果
result = {}
for item in sort_result:
    result[item[0]] = item[1]

import matplotlib.pyplot as plt

# 导入字体管理模块
from mplfonts import use_font
#指定中文字体
use_font('Noto Serif CJK SC')

axis_x=[i[:2]+'\n点\n'+i[2:] for i in result]
axis_y=[result[i] for i in result]

plt.bar(axis_x,axis_y)
plt.xlabel("时间段")
plt.ylabel("消息数量")
plt.title("蓝翼运动大众群")
plt.show()


# app.py
import config
import os
from flask import Flask, jsonify
# from flask_sqlalchemy import SQLAlchemy
from model import db
from flask_cors import CORS
from flask.blueprints import Blueprint
from flask_restful import Api

# Initialize Flask APP
app = Flask(__name__)
app.url_map.strict_slashes = False

# Set CORS header
cors = CORS(app)

# Load the appropriate environment configuration based on the 'env' environment variable
try:
    if os.environ.get('env') == 'PROD':
        app.config.from_object(config.ProdConfig)
    elif os.environ.get('env') == 'DEV':
        app.config.from_object(config.DevConfig)
    else:
        app.config.from_object(config.LocalConfig)
except Exception as e:
    print(f"Error loading environment configuration: {e}")

# db = SQLAlchemy(app)
db.init_app(app)

# Define a custom error handler for 404 (Not Found) errors
@app.errorhandler(404)
def not_found(error):
    return jsonify({'error': 'Not found'}), 404

api_bp = Blueprint('api', __name__)
api = Api(api_bp)

class Routes:
    def __init__(self) -> None:
        pass

    # Load API routes from the 'route' module
    def load_api():
        from route import typeahead
        from route import search
        from route import export
        api.add_resource(typeahead.TypeAheadResult, '/typeahead')
        api.add_resource(search.SearchResult, '/search/<profile_id>')
        api.add_resource(export.ExportResult, '/export/<profile_id>')

# Load API routes and register the blueprint with the Flask app
Routes.load_api()
app.register_blueprint(api_bp)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=4000)

{'typeahead': 'route.typeahead', 'search': 'route.search', 'export': 'route.export'}.load_api


"""
Placeholder end-to-end tests for generating linkml translations and pydantic models.

Should be replaced with more specific unit and integration tests, but in place for now
to ensure that the basics of the whole thing operate -- not doing any actual data validation
here.
"""

import pdb
from pathlib import Path
from typing import Dict

import pytest
import warnings

from .fixtures import nwb_core_fixture, tmp_output_dir
from linkml_runtime.dumpers import yaml_dumper
from linkml_runtime.linkml_model import SchemaDefinition
from nwb_linkml.generators.pydantic import NWBPydanticGenerator
from linkml_runtime.loaders.yaml_loader import YAMLLoader

from nwb_linkml.lang_elements import NwbLangSchema

def test_generate_nwblang(tmp_output_dir):
    output_file = (tmp_output_dir / NwbLangSchema.name).with_suffix('.yml')
    yaml_dumper.dump(NwbLangSchema, output_file)

def test_generate_core(nwb_core_fixture, tmp_output_dir):
    schemas = nwb_core_fixture.build().schemas

    (tmp_output_dir / 'schema').mkdir(exist_ok=True)

    for schema in schemas:
        output_file = tmp_output_dir / 'schema' / (schema.name + '.yaml')
        yaml_dumper.dump(schema, output_file)

def load_schema_files(path: Path) -> Dict[str, SchemaDefinition]:
    yaml_loader = YAMLLoader()
    sch: SchemaDefinition
    preloaded_schema = {}
    for schema_path in (path / 'schema').glob('*.yaml'):
        sch = yaml_loader.load(str(schema_path), target_class=SchemaDefinition)
        preloaded_schema[sch.name] = sch
    return preloaded_schema

@pytest.mark.depends(on=['test_generate_core'])
def test_generate_pydantic(tmp_output_dir):

    (tmp_output_dir / 'models').mkdir(exist_ok=True)

    preloaded_schema = load_schema_files(tmp_output_dir)

    for schema_path in (tmp_output_dir / 'schema').glob('*.yaml'):
        if not schema_path.exists():
            continue
        # python friendly name
        python_name = schema_path.stem.replace('.', '_').replace('-','_')

        pydantic_file = (schema_path.parent.parent / 'models' / python_name).with_suffix('.py')

        generator = NWBPydanticGenerator(
            str(schema_path),
            pydantic_version='2',
            emit_metadata=True,
            gen_classvars=True,
            gen_slots=True,
            schema_map=preloaded_schema
        )
        gen_pydantic = generator.serialize()

        with open(pydantic_file, 'w') as pfile:
            pfile.write(gen_pydantic)

    # make __init__.py
    with open(tmp_output_dir / 'models' / '__init__.py', 'w') as initfile:
        initfile.write('# Autogenerated module indicator')


from django.contrib.auth import get_user_model
from rest_framework import authentication
from rest_framework import exceptions

User = get_user_model()

class AppSecretTokenAuthentication(authentication.BaseAuthentication):
    def authenticate(self, request):
        user_token = request.META.get('CL-X-TOKEN')
        if not user_token:
            return None

        try:
            user = User.objects.get(app_secret_token=user_token)
        except User.DoesNotExist:
            raise exceptions.AuthenticationFailed('Un Authenticate')
        return (user, None)


from django.db import models

# Create your models here.
class Destinations(models.Model):
    name = models.CharField(max_length=100)
    img = models.ImageField(upload_to = 'pics')
    desc = models.TextField()
    price = models.IntegerField()
    offer = models.BooleanField(default = False)

class News(models.Model):
    date = models.DateTimeField(auto_now=True)
    name = models.CharField(max_length=100)
    desc = models.TextField()
    img = models.ImageField(upload_to = 'News/pics')


#!/usr/bin/env python3
# -*-coding:utf-8 -*-
# =============================================================================
"""
@Author        :   Yujie He
@File          :   gen_lidar_from_rosbags.py
@Date created  :   2021/10/19
@Maintainer    :   Yujie He
@Email         :   yujie.he@epfl.ch
"""
# =============================================================================
"""
The module provides workflow to extract pointcloud and corresponding timestamps
from rosbag.
"""
# =============================================================================
"""
TODO:
1. test support with pcd
"""
# =============================================================================

import os
import fnmatch
import argparse
import numpy as np

import rospy
import rosbag
import tf2_py as tf2
import ros_numpy

from qolo.core.crowdbot_data import bag_file_filter, CrowdBotData

# sudo apt-get install ros-$ROS_DISTRO-tf2-sensor-msgs
from tf2_sensor_msgs.tf2_sensor_msgs import do_transform_cloud

def get_tf_tree(bag):
    tf_buffer = tf2.BufferCore(rospy.Duration(1e9))
    for topic, msg, _ in bag.read_messages(topics=["/tf", "/tf_static"]):
        for msg_tf in msg.transforms:
            if topic == "/tf_static":
                tf_buffer.set_transform_static(msg_tf, "default_authority")
            else:
                tf_buffer.set_transform(msg_tf, "default_authority")

    return tf_buffer

def load_lidar(bag, topic, tf_buffer, target_frame="tf_qolo"):
    msgs, ts = [], []
    src_frame = None
    failed_counter = 0
    for _, msg, t in bag.read_messages(topics=[topic]):
        src_frame = msg.header.frame_id

        try:
            trans = tf_buffer.lookup_transform_core(
                target_frame, src_frame, msg.header.stamp
            )
            msg = do_transform_cloud(msg, trans)
        except tf2.ExtrapolationException as e:  # noqa
            # print(e)
            failed_counter += 1
            continue

        pc_xyz = ros_numpy.point_cloud2.pointcloud2_to_xyz_array(msg)
        msgs.append(pc_xyz)
        ts.append(msg.header.stamp.to_sec())
        # ts.append(t.to_sec())

    ts = np.array(ts, dtype=np.float64)  # warning: float32 is not enough

    s = (
        "Summary\n"
        "topic: {}\n"
        "count: {}\n"
        "min timestamp: {}\n"
        "max timestamp: {}\n"
        "average time between frames: {}\n"
        "source frame: {}\n"
        "target frame: {}\n"
        "tf failed count: {}\n"
    ).format(
        topic,
        len(ts),
        ts[0],
        ts[-1],
        (ts[-1] - ts[0]) / len(ts),
        src_frame,
        target_frame,
        failed_counter,
    )

    print(s)

    return msgs, ts

def save_lidar(filename, pc, library="numpy", write_ascii=False, compressed=True):
    # save with numpy
    if library == "numpy":
        with open(filename + ".nby", "wb") as f:
            np.save(f, pc)

    # save with open3d
    elif library == "open3d":
        import open3d as o3d

        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(pc)
        # open3d.io.write_point_cloud(filename, pointcloud, write_ascii=False, compressed=False, print_progress=False)
        o3d.io.write_point_cloud(
            filename + ".pcd", pcd, write_ascii=write_ascii, compressed=compressed
        )

def extract_lidar_from_rosbag(bag_path, out_dir, args):
    """Extract and save combined laser scan from rosbag. Existing files will be overwritten"""

    # print("rosbag: {}".format(bag_path))

    # load lidar in a unified coordinate frame
    with rosbag.Bag(bag_path) as bag:
        tf_buffer = get_tf_tree(bag)
        front_msgs, front_ts = load_lidar(
            bag,
            args.front_topic,
            tf_buffer,  # args.front_topic = "/front_lidar/velodyne_points"
        )
        rear_msgs, rear_ts = load_lidar(
            bag, args.rear_topic, tf_buffer
        )  # args.rear_topic = "/rear_lidar/velodyne_points"

    # sync lidar
    offset = min(front_ts.min(), rear_ts.min())
    front_ts -= offset  # only for easy viewing
    rear_ts -= offset

    front_t0, front_t1 = front_ts.min(), front_ts.max()
    front_dt = (front_t1 - front_t0) / float(len(front_ts))

    rear_t0, rear_t1 = rear_ts.min(), rear_ts.max()
    rear_dt = (rear_t1 - rear_t0) / float(len(rear_ts))

    sync_dt = max(front_dt, rear_dt)  # sync to the slower one
    sync_t0 = max(front_t0, rear_t0)
    sync_t1 = min(front_t1, rear_t1)
    sync_ts = np.arange(start=sync_t0, step=sync_dt, stop=sync_t1, dtype=np.float64)

    def get_sync_inds(ts, sync_ts):
        d = np.abs(sync_ts.reshape(-1, 1) - ts.reshape(1, -1))
        return np.argmin(d, axis=1)

    front_inds = get_sync_inds(front_ts, sync_ts)
    rear_inds = get_sync_inds(rear_ts, sync_ts)

    # write frame to file
    for frame_id, (idx_front, idx_rear) in enumerate(zip(front_inds, rear_inds)):
        pc = np.concatenate((front_msgs[idx_front], rear_msgs[idx_rear]), axis=0)
        file_path = os.path.join(out_dir, "{0:05d}".format(frame_id))
        if args.compressed:
            save_lidar(file_path, pc, library="open3d")
        else:
            save_lidar(file_path, pc)
        """
        # save with numpy
        with open(file_path+".nby", "wb") as f:
            np.save(f, pc)

        # save with open3d
        import open3d as o3d

        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(pc)
        o3d.io.write_point_cloud(file_path+".pcd", pcd, write_ascii=False, compressed=True)
        """

    # yujie: save sync frame id and timestamp
    id_list, ts_list = [], []
    for frame_id, ts in enumerate(sync_ts):
        id_list.append(frame_id)
        ts_list.append(ts + offset)  # adding offset

    lidar_stamped_dict = {
        "timestamp": np.asarray(ts_list, dtype=np.float64),
        "id": np.array(frame_id),
    }
    # bag_name = out_dir.split("/")[-1]
    # stamp_file_path = os.path.join(out_dir, "..", bag_name + "_stamped.npy")
    # np.save(stamp_file_path, lidar_stamped_dict)

    s = (
        "Summary\n"
        "topic: synced frames\n"
        "count: {}\n"
        "average time between frames: {}\n"
    ).format(
        len(sync_ts),
        sync_dt,
    )
    print(s)

    return lidar_stamped_dict

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="convert data from rosbag")

    parser.add_argument(
        "-f",
        "--folder",
        default="1203_test",
        type=str,
        help="different subfolder in rosbag/ dir",
    )
    parser.add_argument(
        "--front_topic",
        default="/front_lidar/velodyne_points",
        type=str,
        help="topic for the front lidar",
    )
    parser.add_argument(
        "--rear_topic",
        default="/rear_lidar/velodyne_points",
        type=str,
        help="topic for the rear lidar",
    )
    parser.add_argument(
        "--overwrite",
        dest="overwrite",
        action="store_true",
        help="Overwrite existing rosbags (default: False)",
    )
    parser.set_defaults(overwrite=False)
    parser.add_argument(
        "--compressed",
        dest="compressed",
        action="store_true",
        help="Save pointcloud with compressed format (default: True)",
    )
    parser.set_defaults(compressed=False)
    args = parser.parse_args()

    qolo_dataset = CrowdBotData()

    base_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')

    # source: rosbag data in data/rosbag/xxxx
    # rosbag_dir = os.path.join(base_folder, "data", "rosbag", args.folder)
    rosbag_dir = os.path.join(qolo_dataset.bagbase_dir, args.folder)
    bag_files = list(filter(bag_file_filter, os.listdir(rosbag_dir)))

    # destination: lidar data in xxxx_processed/lidars
    data_processed = args.folder + "_processed"
    data_processed_dir = os.path.join(qolo_dataset.outbase_dir, data_processed)
    if not os.path.exists(data_processed_dir):
        os.makedirs(data_processed_dir)
    lidar_file_dir = os.path.join(data_processed_dir, "lidars")
    lidar_stamp_dir = os.path.join(data_processed_dir, "source_data", "timestamp")
    if not os.path.exists(lidar_stamp_dir):
        os.makedirs(lidar_stamp_dir)

    print("Starting extracting lidar files from {} rosbags!".format(len(bag_files)))

    for idx, bf in enumerate(bag_files):
        bag_path = os.path.join(rosbag_dir, bf)
        bag_name = bf.split(".")[0]
        out_dir = os.path.join(lidar_file_dir, bag_name)
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        out_lidar_np = fnmatch.filter(os.listdir(out_dir), "*.nby")

        print("({}/{}): {}".format(idx + 1, len(bag_files), bag_path))

        if args.overwrite:
            lidar_stamped_dict = extract_lidar_from_rosbag(bag_path, out_dir, args)
            print(
                "lidar_stamped_dict with {} frames".format(
                    len(lidar_stamped_dict['timestamp'])
                )
            )
            stamp_file_path = os.path.join(lidar_stamp_dir, bag_name + "_stamped.npy")
            np.save(stamp_file_path, lidar_stamped_dict)
        else:
            print("{} already extracted!!!".format(bf))
            continue


import pickle

import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from sklearn import preprocessing
from sklearn.model_selection import TimeSeriesSplit, cross_validate
from sklearn.linear_model import LinearRegression,LogisticRegression
import matplotlib.pyplot as plt
import seaborn as sns

# settings
import warnings

warnings.filterwarnings("ignore")

# reading data files
sales_train = pd.read_csv('data/sales_train_formated.csv')
sales_train.date = pd.to_datetime(sales_train.date, format='%Y-%m-%d')
items = pd.read_csv('data/items.csv')
shops = pd.read_csv('data/shops.csv')
item_category = pd.read_csv('data/item_categories.csv')
sales_test = pd.read_csv('data/test.csv')

# data pre-processing
sales_train.insert(3, 'month', sales_train['date'].dt.month)
#sales_train.insert(4, 'year', sales_train['date'].dt.year)

#sales_train['item_cnt_day'] = sales_train.groupby(['shop_id', 'item_id'], as_index=False)['item_cnt_day'].transform(
 #   pd.Series.diff)

# mean sales of item for each month period
month_mean = sales_train.groupby(['date_block_num','shop_id', 'item_id'], as_index=False)[['item_cnt_day']].mean()
month_mean = month_mean.rename(columns={'item_cnt_day': 'month_mean'})

# mean sales of shop
mean_sales= sales_train.groupby(['date_block_num','item_id'], as_index=False)[['item_cnt_day']].mean()
month_mean = pd.merge(month_mean, mean_sales, how='left', on=['date_block_num','item_id'])
month_mean = month_mean.rename(columns={'item_cnt_day': 'mean_sales'})

# mean prices of shop
#mean_prices = sales_train.groupby(['shop_id'], as_index=False)[['item_price']].mean()
#month_mean = pd.merge(month_mean, mean_prices, how='left', on=['shop_id'])
#month_mean = month_mean.rename(columns={'item_price': 'mean_prices'})

sales_train =sales_train.drop(['item_cnt_day'], axis=1)
sales_train =  pd.merge(sales_train, month_mean, how='left', on=['date_block_num','shop_id', 'item_id'])

'''sales_train = sales_train.groupby([c for c in sales_train.columns if c not in ['item_cnt_day']], as_index=False)[
    ['item_cnt_day']].sum()
sales_train = sales_train.rename(columns={'item_cnt_day': 'item_cnt_month'})
shop_item_monthly_mean = sales_train[['shop_id', 'item_id', 'item_cnt_month']].groupby(['shop_id', 'item_id'], as_index=False)[
    ['item_cnt_month']].mean()

shop_item_monthly_mean = shop_item_monthly_mean.rename(columns={'item_cnt_month': 'item_cnt_month_mean'})
sales_train = pd.merge(sales_train, shop_item_monthly_mean, how='left', on=['shop_id', 'item_id'])

print(sales_train.head(5))
print(sales_train.shape)

'''

# Last Month in the data : Oct 2015
shop_item_prev_month = sales_train[sales_train['date_block_num'] == 33][['shop_id', 'item_id', 'month_mean']]
shop_item_prev_month = shop_item_prev_month.rename(columns={'month_mean': 'prev_month_mean'})
sales_train = pd.merge(sales_train, shop_item_prev_month, how='left', on=['shop_id', 'item_id'])
sales_train = sales_train.fillna(0.)

sales_train = pd.merge(sales_train, items, how='left', on='item_id')
sales_train = pd.merge(sales_train, item_category, how='left', on=['item_category_id'])
sales_train = pd.merge(sales_train, shops, how='left', on=['shop_id'])

sales_train = sales_train.drop(['item_name','item_category_name','shop_name','date','item_price'], axis=1)

'''
# Add more features to test set
# Manipulate test data
sales_test['month'] = 11
sales_test['year'] = 2015
sales_test['date_block_num'] = 34
# sales_test['date_block_num'] = 34
sales_test =  pd.merge(sales_test, month_mean, how='left', on=['date_block_num','shop_id', 'item_id'])
sales_test = pd.merge(sales_test, shop_item_prev_month, how='left', on=['shop_id', 'item_id'])
sales_test = sales_test.fillna(0.)
sales_test = pd.merge(sales_test, items, how='left', on='item_id')
sales_test = pd.merge(sales_test, item_category, how='left', on=['item_category_id'])
sales_test = pd.merge(sales_test, shops, how='left', on=['shop_id'])
sales_test = sales_test.drop(['item_name','item_category_name','shop_name'], axis=1)
print(sales_test.shape)
print(sales_test.columns)
'''

#train_test_df = pd.concat([sales_train, sales_test], axis=0, ignore_index=True)
#stores_hm = train_test_df.pivot_table(index='shop_id', columns='item_category_id', values='item_cnt_month',
#                                      aggfunc='count', fill_value=0)
#stores_hm.tail()

print(sales_train.columns)

feature_list = [c for c in sales_train.columns if c != 'month_mean']
print(feature_list)
x1 = sales_train[sales_train['date_block_num'] < 33]
#x1 = sales_train
#x1 = sales_train['item_price']
y1 = np.log1p(x1['month_mean'].clip(0., 20.))
x1 = x1[feature_list]
#x1['mean_prices'] = np.log1p(x1['mean_prices'].clip(0., 100.))

x2 = sales_train[sales_train['date_block_num'] == 33]
#y2 = np.log1p(x2['month_mean'].clip(0., 20.))
y2 = np.log1p(x2['month_mean'].clip(0., 20.))
x2 = x2[feature_list]
#x2['mean_prices'] = np.log1p(x2['mean_prices'].clip(0., 100.))

rf = RandomForestRegressor(n_estimators=25, oob_score=True, random_state=12, max_depth=3, n_jobs=-1)
rf.fit(x1, y1)
print(1-rf.oob_score_)

lr = LinearRegression()
lr.fit(x1, y1)

df = pd.DataFrame()

df['y'] = y2
df['reg']=lr.predict(x2)
df['forest']=rf.predict(x2)

print(df.columns)
df.to_csv('pol.csv', index=False)

'''
rf_file = 'rf_file_diff.sav'
lr_file = 'lr_file_diff.sav'

pickle.dump(rf,open(rf_file,'wb'))
pickle.dump(lr,open(lr_file,'wb'))
'''

importance = rf.feature_importances_
print(importance)

#importance = np.array([0.05844234,0.00764173,0.01998627,0.006654,0.02563119,0.80008461,0.02529188,0.02550973,0.01148034,0.01171568,0.00756224])

#plots
fig, ax = plt.subplots(figsize=(15, 15))
#sns.heatmap(stores_hm, ax=ax, cbar=False)

indices = np.argsort(importance)[::-1]
featurenames = [feature_list[i] for i in indices]
plt.title("Feature Significance")
plt.bar(range(len(indices)),importance[indices],color='b',align='center')
plt.xticks(range(len(indices)),featurenames,rotation=20,fontsize=18)
plt.title("Feature Significance")
plt.xlabel("Features")
plt.xlabel("Significance")
plt.show()

print("RMSE on Validation hold out month 33: {}".format(
    np.sqrt(mean_squared_error(np.log1p(y2.clip(0., 100.)), rf.predict(x2).clip(0., 20.)))))

from sklearn.tree import export_graphviz
#for tree in rf.estimators_:
tree = rf.estimators_[1]
export_graphviz(tree,out_file='ziii.dot',feature_names=feature_list,filled=True,rounded=True )

#from subprocess import call
#call(['dot','-Tpng','ziii.dot','-o','ziiiv.png','Gdpi=600'])


import gzip
import json

import matplotlib
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

matplotlib.use('TkAgg')
# import matplotlib.pyplot as plt
# from nltk.corpus import stopwords

with gzip.open("goemotions.json.gz", "rb") as f:
    fullData = json.loads(f.read().decode("ascii"))

length = len(fullData)
emotions = [None] * length
posts = [None] * length
sentiments = [None] * length

for i in range(length):
    emotions[i] = fullData[i][1]
    posts[i] = fullData[i][0]
    sentiments[i] = fullData[i][2]

# distribution= [sentiments.count('positive')/length*100, sentiments.count('neutral')/length*100, sentiments.count('negative')/length*100, sentiments.count('ambiguous')/length*100]
# labels = 'Positive', 'Neutral', 'Negative', 'Ambiguous'
# explode = (0, 0.1, 0,0)  # only "explode" the 2nd slice (i.e. 'Hogs')
#
# plt.pie(distribution, explode=explode, labels=labels, autopct='%1.1f%%',
#         shadow=True, startangle=90)
# plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
#
# plt.savefig("sentiments.png")
# plt.rcParams.update({'font.size': 3})
#
# plt.hist(emotions, bins=28, alpha=0.5)
# plt.title('Emotions')
# plt.xlabel('X')
# plt.ylabel('Y')
#
# plt.show()

# Basic Naive Bayes

print('--------------BASIC NAIVE BAYES SENTIMENTS------------------------')
vectorizer = CountVectorizer(stop_words='english')
sentiments_encoded = sentiments
posts_encoded = vectorizer.fit_transform(posts)
print(vectorizer.vocabulary_)
print("The length of the vocabulary is " + str(len(vectorizer.vocabulary_)))
X_train, X_test, y_train, y_test = train_test_split(posts_encoded, sentiments_encoded, stratify=sentiments_encoded,
                                                    test_size=0.2, random_state=0)
classifier = MultinomialNB()
modelBE = classifier.fit(X_train, y_train)
predictions = classifier.predict(X_test)
f = open("performance.txt", "a")
f.write("Bayes Setiments Confusion Matrix")
f.write("\n")
f.write(str(confusion_matrix(y_test, predictions)))
f.write("\n")
f.write(str(classification_report(y_test, predictions)))
f.write("\n")
f.close()
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
print(accuracy_score(y_test, predictions))
print('------------------------------------------------------------------')


#!/usr/bin/env python3

import hashlib
import io
import itertools
import json
import logging
import pathlib
import queue
import threading
import time

import click
import cv2
import numpy as np
import requests

def calc_sha1_hash(path):
    with path.open("rb") as file:
        return hashlib.sha1(file.read()).hexdigest()

def make_video_meta(video_file_path, video_capture):
    return {
        "sha1": calc_sha1_hash(video_file_path),
        "size": video_file_path.stat().st_size,
        "width": int(video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)),
        "height": int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)),
        "fps": video_capture.get(cv2.CAP_PROP_FPS),
        "numberOfFrames": int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT)),
    }

def dump_json_compact(obj):
    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

def dump_numpy(ary):
    bio = io.BytesIO()
    np.save(bio, ary)
    return bio.getvalue()

def read_thread(video_file_path, video_capture, output_queue):
    logging.info("start")

    total_count = 0
    total_time_ns = 0

    for frame_index in itertools.count():
        start_ns = time.perf_counter_ns()
        result, frame = video_capture.read()
        time_ns = time.perf_counter_ns() - start_ns
        if not result:
            break

        output_queue.put(
            {
                "video_file_path": video_file_path,
                "frame_index": frame_index,
                "frame": frame,
            }
        )

        total_count += 1
        total_time_ns += time_ns

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def detection_thread(input_queue, output_queue, detector_base_url):
    logging.info("start")
    logging.info("detector_base_url=%s", detector_base_url)

    url = detector_base_url + "/detect"
    total_count = 0
    total_time_ns = 0

    while True:
        frame_info = input_queue.get()
        try:
            if frame_info is None:
                break
            logging.info("frame_index=%d", frame_info["frame_index"])

            start_ns = time.perf_counter_ns()

            files = {
                "file": (
                    frame_info["video_file_path"].name,
                    dump_numpy(frame_info["frame"]),
                    "application/octet-stream",
                ),
            }
            response = requests.post(url, files=files)
            assert response.status_code == 200
            result = response.json()

            time_ns = time.perf_counter_ns() - start_ns

            output_queue.put(
                {
                    "video_file_path": frame_info["video_file_path"],
                    "frame_index": frame_info["frame_index"],
                    "frame": frame_info["frame"],
                    "result": result,
                }
            )

            total_count += 1
            total_time_ns += time_ns
        finally:
            input_queue.task_done()

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def write_thread(input_queue, output_file_path, video_meta):
    logging.info("start")
    logging.info("output_file_path=%s", str(output_file_path))

    total_count = 0
    total_time_ns = 0

    with output_file_path.open("w") as output_file:
        output_file.write(dump_json_compact(video_meta) + "\n")
        while True:
            frame_info = input_queue.get()
            try:
                if frame_info is None:
                    break

                start_ns = time.perf_counter_ns()

                record = {
                    "frame_index": frame_info["frame_index"],
                    "result": frame_info["result"],
                }
                output_file.write(dump_json_compact(record) + "\n")

                time_ns = time.perf_counter_ns() - start_ns

                total_count += 1
                total_time_ns += time_ns
            finally:
                input_queue.task_done()

    logging.info("total_count=%d", total_count)
    logging.info("total_time[ms]=%d", total_time_ns / 1000 / 1000)
    logging.info("mean_time[ms]=%f", total_time_ns / total_count / 1000 / 1000)
    logging.info("throughput/sec=%f", 1_000_000_000 / (total_time_ns / total_count))
    logging.info("end")

def start_read_worker(**kwargs):
    worker = threading.Thread(
        name="read",
        target=read_thread,
        daemon=True,
        kwargs=kwargs,
    )
    worker.start()
    return worker

def start_detection_workers(detector_base_urls, input_queue, output_queue):
    workers = []

    for thread_index, detector_base_url in enumerate(detector_base_urls):
        worker = threading.Thread(
            name="detection#{}".format(thread_index),
            target=detection_thread,
            daemon=True,
            kwargs={
                "input_queue": input_queue,
                "output_queue": output_queue,
                "detector_base_url": detector_base_url,
            },
        )
        worker.start()
        workers.append(worker)

    return workers

def start_write_worker(**kwargs):
    worker = threading.Thread(
        name="write",
        target=write_thread,
        daemon=True,
        kwargs=kwargs,
    )
    worker.start()
    return worker

def join_all_workers(workers, input_queue):
    for _ in workers:
        input_queue.put(None)
    for worker in workers:
        worker.join()

@click.command()
@click.argument("video_file_path", type=click.Path(exists=True))
@click.argument("output_file_path", type=click.Path(exists=False))
def main(video_file_path, output_file_path):
    threading.current_thread().name = "main"
    logging.info("start")

    video_file_path = pathlib.Path(video_file_path)
    output_file_path = pathlib.Path(output_file_path)
    detector_base_urls = [
        "http://detector-insightface:8000",
        "http://detector-insightface:8000",
    ]

    video_capture = cv2.VideoCapture(str(video_file_path))
    assert video_capture.isOpened()
    video_meta = make_video_meta(video_file_path, video_capture)

    frame_queue = queue.Queue(maxsize=10)
    detection_queue = queue.Queue()

    read_worker = start_read_worker(
        video_file_path=video_file_path,
        video_capture=video_capture,
        output_queue=frame_queue,
    )
    detection_workers = start_detection_workers(
        detector_base_urls=detector_base_urls,
        input_queue=frame_queue,
        output_queue=detection_queue,
    )
    write_worker = start_write_worker(
        input_queue=detection_queue,
        output_file_path=output_file_path,
        video_meta=video_meta,
    )

    read_worker.join()
    join_all_workers(detection_workers, frame_queue)
    join_all_workers([write_worker], detection_queue)

    logging.info("end")

if __name__ == "__main__":
    logging.basicConfig(
        format="%(asctime)s %(levelname)s [%(threadName)s] %(message)s",
        level=logging.INFO,
    )
    main()


from django.shortcuts import render
from .controllers.NLPInputProcessor import NLPInputProcessor
from .controllers.database import Database
from .controllers.ExploreStructure import ExploreStructure
from .controllers.ConvertToSQL import ConvertToSQL
from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse
import json

# Create your views here.
new_db = None
@csrf_exempt
def get_query(request):
    global new_db
    query = request.GET.get('query')
    # new_db = request.session.get("db")
    nlp_processor = NLPInputProcessor(query, new_db.flatten_dimension())
    mapped_query, token_to_tag = nlp_processor.map_query()
    print(mapped_query)
    mapped_types = nlp_processor.mapped_types(mapped_query)
    print("Sentence: ", query)
    # print(">> Mapped query: ", mapped_query)
    print(">> Mapped with tags: ", token_to_tag)
    # print(">> Mapped types: ", mapped_types)
    explore_struct = ExploreStructure(mapped_query, new_db.fetch_metadata())
    print(explore_struct.explore_struct)

    convertToSQL = ConvertToSQL(new_db.fetch_metadata(), query)
    aggregate, group_by = convertToSQL.aggregate_template(explore_struct.explore_struct["measures"])
    select_clause, from_clause =convertToSQL.checkJoinCondition(explore_struct.explore_struct["dimensions"], explore_struct.explore_struct["measures"])
    where_clause = convertToSQL.where_template(explore_struct.explore_struct["filters"])

    if (aggregate != ""):
        if select_clause == "SELECT * " or select_clause == "SELECT DISTINCT ":
            sql_query = select_clause + aggregate + from_clause + where_clause + group_by
        else:
            sql_query = select_clause + ", " + aggregate + from_clause + where_clause + group_by
    else:
        sql_query = select_clause + aggregate + from_clause + where_clause + group_by

    print(sql_query)
    queryResult = new_db.run_query(sql_query)
    print(queryResult)
    result = {"result": queryResult.to_dict(), "rowCount": queryResult.shape[0], "sqlQuery": sql_query, "expStruct": explore_struct.explore_struct}
    print(result)
    return JsonResponse(result, safe = False)

def connect_database(request):
    db_name = request.GET.get('name')
    db_type = request.GET.get('type')
    db_host = request.GET.get('host')
    db_port = request.GET.get('port')
    db_user = request.GET.get('user')
    db_pass = request.GET.get('password')
    global new_db
    new_db = Database(db_name, db_type, db_host, db_port, db_user, db_pass)
    if new_db.get_connection() != None:
        return JsonResponse({"message": "Success"}, safe=False)
    else:
        new_db = None
        return JsonResponse({"message": "Failed"}, safe=False)

def disconnect_database(request):
    global new_db
    if new_db != None:
        new_db.close_connection()
        new_db = None
    return JsonResponse({"message": "Done"}, safe=False)

def get_metadata(request):
    global new_db
    if new_db != None:
        return JsonResponse({"metadata": new_db.fetch_dimension(), "db_name": new_db.get_name()}, safe=False)
    else:
        return JsonResponse({"metadata": None, "db_name": None}, safe=False)

def get_keyword_type(request):
    global new_db
    if new_db != None:
        query = request.GET.get('query')
        nlp_processor = NLPInputProcessor(query, new_db.flatten_dimension())
        mapped_query, token_to_tag = nlp_processor.map_query()
        return JsonResponse({"mapTag": token_to_tag}, safe=False)
    else:
        return JsonResponse({"mapTag": None}, safe=False)


from typing import List

# See details here https://wenshengchen.com/2020/01/21/221-maximal-square.html
class Solution:
    def maximalSquare(self, matrix: List[List[str]]) -> int:
        maximal = 0
        for row in range(len(matrix)):
            for col in range(len(matrix[0])):
                maximal = max(maximal, int(matrix[row][col]))

                if row > 0 and col > 0:
                    square = min(int(matrix[row-1][col]), int(matrix[row-1][col-1]), int(matrix[row][col-1]))
                    if square > 0 and matrix[row][col] == "1":
                        matrix[row][col] = str(square + 1)
                        maximal = max(maximal, int(matrix[row][col]))

        return maximal**2

## TEST CASES
test = Solution()
assert test.maximalSquare([["1","0","1","0","0"],["1","0","1","1","1"],["1","1","1","1","1"],["1","0","0","1","0"]]) == 4
assert test.maximalSquare([["1","0","1","1","1"],["1","0","1","1","1"],["1","1","1","1","1"],["1","0","0","1","0"]]) == 9
assert test.maximalSquare([["0","0","0","0","0"],["0","0","0","0","0"],["0","0","0","0","0"],["0","0","0","1","0"]]) == 1
assert test.maximalSquare([["1"]]) == 1
assert test.maximalSquare([[]]) == 0
assert test.maximalSquare(None) == 0
assert test.maximalSquare([["1","1"],["1","1"]]) == 4
print('All Passed!')


from bs4 import BeautifulSoup
import urllib.request
from urllib.parse import quote
import pandas

date = []

def main():
    query = input('검색어 : ')
    s_date = input('시작날짜 (2019.05.15) : ')
    e_date = input('끝날짜 (2019.05.15) : ')

    get_news(query, s_date, e_date)

def get_news(query, s_date, e_date):
    date_index = pandas.date_range(start=s_date, end=e_date)
    date = date_index.strftime("%Y.%m.%d").to_list()

    for i in date:
        t_query = quote(query)

        url = "https://search.naver.com/search.naver?where=news&query=" + t_query + "&sort=0&photo=0&field=0&reporter_article=&pd=3&ds=" + i + "&de=" + i

        news = []
        while True:
            news_keyword_url = urllib.request.urlopen(url).read()
            soup = BeautifulSoup(news_keyword_url, 'html.parser')

            link_all = soup.find_all('dd', {'class': 'txt_inline'})

            for link in link_all:
                link_find_a = link.find('a')
                link_find_href = link_find_a.get('href')

                # 네이버 뉴스 url이 없는 경우, 크롤링하지 않음
                if (link_find_href == '#'):
                    continue

                # 네이버 뉴스 url이 있는 경우, 뉴스 본문 크롤링
                else:
                    news_link = urllib.request.urlopen(link_find_href).read()
                    news_link_html = BeautifulSoup(news_link, 'html.parser')

                    for item in news_link_html.find_all('div', {'id': 'articleBodyContents'}):
                        result = item.get_text()
                        result = result.replace("// flash 오류를 우회하기 위한 함수 추가", "")
                        result = result.replace("function _flash_removeCallback() {}", "")
                        result = result.replace("\n", "")
                        result = result.replace("\t", "")

                        news.append(result)
                    news.append("\n\n")

            try :
                # paging에서 다음페이지 버튼에 연결된 url 찾음
                next = soup.find('a', {'class': 'next'})
                next_href = next.get('href')

            except AttributeError :
                # 맨 마지막장에 다음페이지 버튼이 없어서 에러
                # 이 때, loop를 break;
                break

            # 다음페이지에서 뽑은 url
            url = 'https://search.naver.com/search.naver' + next_href
            print(url)

        # 키워드를 통해 존재하는 모든 네이버 뉴스기사를 txt파일로 저장
        f = open('C:/Users/gpqls/OneDrive/바탕 화면/4/cap/' + query + '(' + i.replace(".","") + ')' +'.txt', 'a', -1, 'utf-8')
        for text in news :
            f.write(text)
        f.close()

main()


# Author: Daniele Rondina, geaaru@sabayonlinux.org
# Description:
# Note: Use PEP-0008 for indentation with max 79 chars on line.

import asyncio
import os
import traceback2
import uuid
import json
import datetime
import pytz

from sys import exit as start_shuttle
from concurrent.futures import CancelledError

from web_keepaliver.common import AppSeed, KafkaClientConfigurator
from web_keepaliver.db.postgres import PostgresConnector, PostgresDatabase

KEEPALIVER_KAFKA_CONSUMER_CONFIG = '/etc/web-keepaliver/kafka-consumer.yaml'
KEEPALIVER_KAFKA_CONSUMER_DEF_NAME = 'web-keepaliver-kafka-consumer'
KEEPALIVER_KAFKA_CONSUMER_DESC = """
Copyright (c) 2021 Daniele Rondina

Web Keepaliver Kafka Consumer.
"""

class KeepaliverKafkaConsumer(AppSeed,
                              KafkaClientConfigurator,
                              PostgresConnector):

    def __init__(self):
        self.kafka_brokers = []
        self.kafka_consumer = None
        self.msgs_counter = 0
        self.in_shutdown = False

        config_def_file = os.getenv(
            'KEEPALIVER_KAFKA_CONSUMER_CONFIG',
            KEEPALIVER_KAFKA_CONSUMER_CONFIG,
        )

        AppSeed.__init__(self, logger_name='kafka-consumer',
                         config_default_file=config_def_file,
                         app_description=KEEPALIVER_KAFKA_CONSUMER_DESC)
        KafkaClientConfigurator.__init__(self)
        PostgresConnector.__init__(self)

        # Avoid cancel tasks when kafka consumer is running.
        self._cancel_tasks = False

        self.get_parser().add_argument(
            '--broker',
            action='store',
            default=[],
            nargs='*',
            type=str,
            dest='kafka_brokers',
            help='Override kafka brokers defined in the configuration file.'
        )

    def get_postgres_conn_options(self):
        return self.get_config_category('postgres', {})

    def get_kafka_configuration(self, producer=True, admin=False):
        opts = self.get_config_category('kafka', {})

        if len(self.kafka_brokers) > 0:
            opts['bootstrap_servers'] = self.kafka_brokers

        opts['client_id'] = self.name
        return opts

    async def _init_consumer(self):
        self.kafka_consumer = self.create_kafka_consumer()

        brokers = 'localhost:9092'
        if len(self.kafka_brokers) > 0:
            brokers = self.kafka_brokers
        else:
            brokers = self.get_config_param(
                'kafka', 'bootstrap_servers', brokers
            )

        self.logger.info(
            'Starting instance %s with kafka brokers %s.',
            self.name, brokers,
        )

        await self.kafka_consumer.start()

        # Subscripte topics
        self.kafka_consumer.subscribe(pattern=self.get_config_param(
            None, 'subscription_regex', "web-keepaliver"
        ))
        # NOTE: when the group coardinator is not yet present in the
        # kafka broker aiokafka library wrote this error:
        # Group Coordinator Request failed:
        # [Error 15] CoordinatorNotAvailableError
        #
        # This error is automatically handled by the library with
        # retries and after 2/3 seconds all works correctly.

    async def _initialize(self):
        self.name = self.get_config_param(
            None, 'name', KEEPALIVER_KAFKA_CONSUMER_DEF_NAME
        )

        # New aiokafka package has deprecated loop argument.
        # It's needed to ensure the is used the right loop object.
        asyncio.set_event_loop(loop=self.loop)

        await self._init_consumer()
        await self.create_pool()

    def signal_handler(self, signame):
        self.in_shutdown = True

    async def _process_site_events(self, site, topic, messages):
        last_event = messages[len(messages)-1]
        site_probes = []
        status = {}

        self.logger.info(
            "[%s] Processing %d events from topic %s.", site,
            len(messages), topic,
        )

        try:
            for event in messages:
                if 'probes' not in event:
                    self.logger.warning(
                        "[%s] Received invalid event without probes from "
                        "topic %s", site, topic,
                    )
                    continue

                probes = event['probes']

                all_events_ok = True
                for p in probes:
                    if 'resp_ts' in p:
                        ts = datetime.datetime.fromtimestamp(
                            int(p['resp_ts'])/1000.0,
                        )
                    else:
                        # If resp_ts is not present then
                        # we use the kafka message timestamp.
                        ts = datetime.datetime.fromtimestamp(
                            p.timestamp/1000.0,
                        )

                    ts = datetime.datetime(ts.year, ts.month, ts.day,
                                           hour=ts.hour,
                                           minute=ts.minute,
                                           second=ts.second,
                                           tzinfo=pytz.utc)

                    site_probes.append((
                        ts, site,
                        p['resource'],
                        p['method'],
                        p['url'],
                        p['resp_http_code'],
                        p['resp_time_ms'],
                        p['expected_http_code'],
                        p['result'],
                        p['error_desc'] if 'error_desc' in p else None,
                    ))

                    if not p['result']:
                        all_events_ok = False

                if event == last_event:
                    last_update = datetime.datetime.utcnow()
                    # Prepare status entry
                    status = {
                        'site': site,
                        'last_update': last_update,
                        'status': all_events_ok,
                        'n_resources': len(event['probes']),
                        'err_counter': 0 if all_events_ok else False,
                    }
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or prepare records on db: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )

        con = await self.acquire()
        try:
            # Register probes
            await PostgresDatabase.register_probes(con, site_probes)

            # Update site status
            await PostgresDatabase.register_site_status(con, status)
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or register records on db: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )
        finally:
            await self.release(con)

    async def _parse_topic_messages(self, tp, messages):

        try:
            site_events_map = {}
            for message in messages:

                jmsg = json.loads(str(message.value.decode('ascii')))

                self.msgs_counter += 1
                self.logger.info(
                    "[%s/%d/%d] [%s] Received message %s",
                    tp.topic, tp.partition,
                    message.offset,
                    str(uuid.UUID(bytes=message.key)),
                    str(message.value.decode('ascii')) if self.debug
                    else 'with %d events' % len(jmsg['probes'])
                )

                # Organize messages for site
                if jmsg['site'] not in site_events_map:
                    site_events_map[jmsg['site']] = [jmsg]
                else:
                    site_events_map[jmsg['site']].append(jmsg)

            futures = []
            for site in site_events_map:
                events = site_events_map[site]
                futures.append(asyncio.ensure_future(
                    self._process_site_events(site, tp.topic, events)
                ))

            await asyncio.gather(
                *futures,
                return_exceptions=True
            )
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error or parse topic messages: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines())
                if self.debug
                else ''
            )

    async def _async_main(self):
        await self._initialize()

        self.logger.info(
            "Instance %s started.", self.name
        )

        try:
            while not self.in_shutdown:
                self.logger.debug("Waiting for messages...")

                data = await self.kafka_consumer.getmany(
                    timeout_ms=self.get_config_param(
                        None, 'timeout_waiting_data',
                        10000,
                    )
                )
                futures = []
                for tp, messages in data.items():
                    future = asyncio.ensure_future(
                        self._parse_topic_messages(
                            tp, messages,
                        )
                    )
                    futures.append(future)

                await asyncio.ensure_future(
                    asyncio.gather(
                        *futures,
                        return_exceptions=True,
                        loop=self.loop,
                    )
                )

        except CancelledError:
            pass

        await self.kafka_consumer.commit()

        # TODO: Verify why with the unsubscribe i receive error on stop
        #  consumer.
        # Error: Commit offset before unsubscribe to avoid
        # UnknownMemberIdError
        # self.kafka_consumer.unsubscribe()

    async def _cleanup(self):
        if self.kafka_consumer:
            await self.kafka_consumer.stop()
        await self.postgres_cleanup()

    def main(self, parse_cmdline_opts=True):
        ans = 0
        if parse_cmdline_opts:
            self.parse_cmd_line()

        try:
            self.init_thread_pool(size=self.get_config_param(
                'general', 'thread_pool_size', 20,
            ))
            self.add_signal_handler()
            self.loop.run_until_complete(
                asyncio.ensure_future(self._async_main())
            )
        except KeyboardInterrupt:
            pass
        except CancelledError:
            pass
        except Exception as exc:  # pylint: disable=broad-except
            self.logger.error(
                'Unexpected error: %s\n%s', exc,
                '\n'.join(traceback2.format_exc().splitlines()) if self.debug
                else ''
            )
            ans = 1

        # Running closing operation of the producer
        self.loop.run_until_complete(self._cleanup())

        self.logger.info(
            "Instance %s stopped.", self.name
        )
        self.loop.close()

        return ans

def main():
    in_space = KeepaliverKafkaConsumer()
    start_shuttle(in_space.main())

if __name__ == '__main__':
    main()

# vim: ts=4 sw=4 expandtab


"""
    新增功能：
        完善音效类
        1.开场音乐
        2.爆炸音效
"""

# 导入模块
import pygame, random, time, os
from pygame.sprite import Sprite

SCREEN_WIDTH = 800  # 窗口的宽度
SCRENN_HEIGHT = 700 # 窗口的高度
BGCOLOR = pygame.Color(0, 0, 0) # 窗口的背景颜色
TEXT_COLOR = pygame.Color(0, 255, 255)    # 文字的字体颜色
os.environ["SDL_VEDIO_WINDOW_POS"] = "{0}, {1}".format(300, 300)    # 窗口的固定位置

# 定义一个基类，继承精灵类
class BaseItem(Sprite):
    def __init__(self, color, width, height):
        pygame.sprite.Sprite.__init__(self)

# 主类
class MainGame():
    window = None   # 窗口对象
    my_tank = None  # 我方坦克对象
    enemyTankList = []  # 敌方坦克对象列表
    enemyTankCount = 10  # 敌方坦克对象数量
    myBulletList = []   # 我方子弹对象列表
    enemyBulletList = []    # 敌方子弹对象列表
    explodeList = []    # 爆炸效果对象列表
    wallList = []       # 墙壁对象列表
    death_time = 0      # 我方坦克死亡次数
    def __init__(self):
        pass
    # 开始游戏方法
    def startGame(self):
        pygame.display.init()   # 初始化pygame
        Music("start").play()
        MainGame.window = pygame.display.set_mode([SCREEN_WIDTH, SCRENN_HEIGHT])    # 创建窗口
        os.environ["SDL_VEDIO_WINDOW_POS"] = "{0}, {1}".format(100, 100)    # 设置固定窗口位置
        pygame.display.set_caption("坦克大战1.0")   # 修改窗口标题
        # 创建我方坦克对象
        MainGame.my_tank = Tank(350, 250)   # 调用这个对象，则返回的也是一个对象，具有初始化中的属性
        # 创建坦克对象, 并添加到敌方坦d克列表
        self.createEnemyTank()
        # 创建墙壁对象
        self.createWall()
        # 使用一个死循环让程序一直运行，循环显示窗口
        while True:
            time.sleep(0.008)
            MainGame.window.fill(BGCOLOR)  # 填充窗口背景色
            self.getEvent() # 获取事件（退出和键盘按下）
            # 在窗口（20,20）位置显示文字 --》 将创建好的文字surface（表面）贴到主窗口
            MainGame.window.blit(self.getTextSurface("剩余敌方坦克数量：{0}".format(len(MainGame.enemyTankList)), 18), (20, 20))
            MainGame.window.blit(self.getTextSurface("死亡次数：{0}".format(MainGame.death_time), 18), (20, 40))
            if MainGame.my_tank and MainGame.my_tank.live:
                MainGame.my_tank.display()  # 显示我方坦克
            else:
                MainGame.my_tank = None
            self.blitEnemyTank()    # 显示敌方坦克
            self.blitMyBullet()     # 显示我方子弹
            self.blitEnemyBullet()  # 显示敌方子弹
            self.blitExplode()      # 显示爆炸效果
            self.blitWall()     # 显示墙壁
            # 只有移动开关为不关闭时，才能让坦克移动
            if MainGame.my_tank and MainGame.my_tank.live:
                if not MainGame.my_tank.stop:
                    MainGame.my_tank.move()  # 让我方坦克一直移动，直到按键松开
                    MainGame.my_tank.hitWall()  # 移动后，检测是否有和墙壁碰撞
                    MainGame.my_tank.myTank_hit_enemyTank()     # 检测我方坦克是否与敌方坦克碰撞
            else:
                MainGame.window.blit(self.getTextSurface("按Esc键复活坦克！", 28), (300, 200))
            self.updateEnemyTank()
            self.updateWall()
            pygame.display.update() # 更新窗口

    # 判断墙壁是否全部消灭
    def updateWall(self):
        if len(MainGame.wallList) <= 0:
            MainGame.window.blit(self.getTextSurface("按r键刷新墙壁！", 30), (300, 20))
    # 判断坦克是否全部消灭
    def updateEnemyTank(self):
        if len(MainGame.enemyTankList) <= 0:
            MainGame.window.blit(self.getTextSurface("按e键刷新敌方坦克！", 28), (300, 300))
    # 显示墙壁
    def blitWall(self):
        for wall in MainGame.wallList:
            wall.display()
    # 创建墙壁对象，并添加到墙壁列表中
    def createWall(self):
        for i in range(6):
            # 创建对象
            wall = Wall(250, 130*i)
            # 添加到墙壁列表中
            MainGame.wallList.append(wall)
            # wall.display()

    # 显示爆炸效果
    def blitExplode(self):
        for explode in MainGame.explodeList:
            if explode.live:
                explode.display()
                if not pygame.mixer.music.get_busy():
                    Music("fire").play()
            else:
                MainGame.explodeList.remove(explode)
    # 显示敌方子弹
    def blitEnemyBullet(self):
        for enemyBullet in MainGame.enemyBulletList:
            # 判断敌方子弹是否存活
            if enemyBullet.live:
                enemyBullet.display()   # 显示敌方子弹
                enemyBullet.move()      # 敌方子弹移动
                enemyBullet.enemyBullet_hit_myTank()
                enemyBullet.hitBullet()
                enemyBullet.hitWall()
                enemyBullet.enemyBullet_hit_enemyTank()
            else:
                MainGame.enemyBulletList.remove(enemyBullet)
    # 显示我方子弹
    def blitMyBullet(self):
        for myBullet in MainGame.myBulletList:
            # 只有子弹是存活的才显示和移动子弹
            if myBullet.live:
                myBullet.display()  # 显示子弹
                myBullet.move()     # 移动子弹
                # 我方坦克移动之后调用坚持碰撞方法
                myBullet.myBullet_hit_enemyTank()
                myBullet.hitBullet()
                myBullet.hitWall()
            else:
                MainGame.myBulletList.remove(myBullet)
    # 创建敌方坦克对象，并添加到对象中
    def createEnemyTank(self):
        top = 100
        for i in range(MainGame.enemyTankCount):
            left = random.randint(0, 600)
            speed = random.randint(10, 20)
            enemyTank = EnemyTank(top, left, speed)
            MainGame.enemyTankList.append(enemyTank)

    # 显示敌方坦克
    def blitEnemyTank(self):
        for enemyTank in MainGame.enemyTankList:
            # 只有敌方坦克存活时才显示
            if enemyTank.live:
                enemyTank.display()
                enemyTank.randMove()  # 敌方坦克随机移动
                enemyTank.hitWall()     # 检测敌方坦克是否撞墙
                enemyTank.enemyTank_hit_myTank()    # 检测敌方坦克是否与我方坦克相撞
                # enemyTank.enemyTank_hit_enemyTank()
                enemyBullet = enemyTank.shot()  # shot方法采用随机数处理，所以有大部分情况返回None
                # 只有敌方子弹不为空时才将敌方子弹对象添加到敌方子弹对象列表
                if enemyBullet:
                    MainGame.enemyBulletList.append(enemyBullet)
            else:
                MainGame.enemyTankList.remove(enemyTank)
    # 游戏结束方法
    def endGame(self):
        print("游戏结束！")
        exit()
    # 获取文字surface（表面）需要传递文字内容和文字大小
    def getTextSurface(self, text, size):
        self.text = text    # 文字内容
        self.size = size    # 文字大小
        pygame.font.init()  # 初始化字体
        # 获取font字体对象
        font = pygame.font.SysFont("kaiti", 18) # ,SysFont表示从系统字体中获取字体
        textSurface = font.render(text, True, TEXT_COLOR)   # 将文字贴在一个surface上
        return textSurface  # 调用结束后将返回一这个surface

    # 获取事件方法
    def getEvent(self):
        eventList = pygame.event.get()  # 获取事件列表
        # 循环遍历事件列表
        for event in eventList:
            # 如果事件类型为退出，则调用游戏结束方法
            if event.type == pygame.QUIT:
                self.endGame()
            # 如果事件类型为键盘按下，则判断按下了哪个键，并做相应的操作
            if event.type == pygame.KEYDOWN:
                if MainGame.my_tank and MainGame.my_tank.live:
                    if event.key == pygame.K_w or event.key == pygame.K_UP:
                        # print("按下了w键，坦克向上移动！")
                        MainGame.my_tank.direction = "U"
                        # MainGame.my_tank.move()
                        MainGame.my_tank.stop = False
                    elif event.key == pygame.K_a or event.key == pygame.K_LEFT:
                        # print("按下了a键，坦克向左移动！")
                        MainGame.my_tank.direction = "L"
                        # MainGame.my_tank.move()
                        MainGame.my_tank.stop = False
                    elif event.key == pygame.K_s or event.key == pygame.K_DOWN:
                        # print("按下了s键，坦克向下移动！")
                        MainGame.my_tank.direction = "D"
                        # MainGame.my_tank.move()
                        MainGame.my_tank.stop = False
                    elif event.key == pygame.K_d or event.key == pygame.K_RIGHT:
                        # print("按下了d键，坦克向右移动！")
                        MainGame.my_tank.direction = "R"
                        # MainGame.my_tank.move()
                        MainGame.my_tank.stop = False
                    if event.key == pygame.K_SPACE or event.key == pygame.K_q:
                        # print("发射子弹")
                        if len(MainGame.myBulletList) <= 2:
                            if not pygame.mixer.music.get_busy():
                                Music("hit").play()
                            myBullet = Bullet(MainGame.my_tank)  # 创建我方子弹
                            MainGame.myBulletList.append(myBullet)  # 将子弹添加到我方子弹列表
                # 按下e键创建敌方坦克
                if event.key == pygame.K_e:
                    self.createEnemyTank()
                if event.key == pygame.K_r:
                    self.createWall()
                # 我方坦克不存活的情况下检测是否有按下Esc键，有的话则让我方坦克复活
                elif event.key == pygame.K_ESCAPE:
                        # print("按下Esc键")
                        MainGame.my_tank = Tank(350, 250)

            # 如果松开是方向键，则让坦克停止
            if event.type == pygame.KEYUP:
                if MainGame.my_tank and MainGame.my_tank.live:
                    if event.key == pygame.K_w or event.key == pygame.K_a or event.key == pygame.K_s or event.key == pygame.K_d:
                        MainGame.my_tank.stop = True

class Tank(BaseItem):
    def __init__(self, top, left):
        self.images = {
            'U':pygame.image.load('img/p1tankU.gif'),
            'L':pygame.image.load('img/p1tankL.gif'),
            'D':pygame.image.load('img/p1tankD.gif'),
            'R':pygame.image.load('img/p1tankR.gif'),
        }
        # 方向
        self.direction = "U"
        self.image = self.images[self.direction]
        self.rect = self.image.get_rect()
        self.rect.top = top
        self.rect.left = left
        self.speed = 5      # 坦克的速度
        self.stop = True    # 坦克的移动情况
        self.live = True    # 坦克的存活情况
        # 坦克移动前的坐标
        self.oldLeft = 0
        self.oldTop = 0

    def display(self):
        # 获取展示的对象
        self.image = self.images[self.direction]
        MainGame.window.blit(self.image, self.rect)
    def move(self):
        # 保留每次移动前的坐标
        self.oldLeft = self.rect.left
        self.oldTop = self.rect.top
        if self.direction == "U":
            if self.rect.top - self.speed >= 0:
                self.rect.top -= self.speed
            else:
                self.rect.top = 0
        elif self.direction == "D":
            if self.rect.top + self.speed < SCRENN_HEIGHT - self.rect.height:
                self.rect.top += self.speed
            else:
                self.rect.top = SCRENN_HEIGHT - self.rect.height
        elif self.direction == "L":
            if self.rect.left - self.speed > 0:
                self.rect.left -= self.speed
            else:
                self.rect.left = 0
        elif self.direction == "R":
            if self.rect.left + self.speed < SCREEN_WIDTH - self.rect.height:
                self.rect.left += self.speed
            else:
                self.rect.left = SCREEN_WIDTH - self.rect.height

    # 坦克射击，返回一个子弹对象
    def shot(self):
        return Bullet(self)

    # 回到上一次的移动前的坐标
    def stay(self):
        self.rect.left = self.oldLeft
        self.rect.top = self.oldTop

    # 判断坦克是否有撞墙
    def hitWall(self):
        for wall in MainGame.wallList:
            # 如果撞墙，回到上一次移动前的坐标
            if pygame.sprite.collide_rect(self, wall):
                # self.stop = True
                self.stay()

    def myTank_hit_enemyTank(self):
        for enemyTank in MainGame.enemyTankList:
            if pygame.sprite.collide_rect(self, enemyTank):
                self.stay()

    def enemyTank_hit_myTank(self):
        for enemyTank in MainGame.enemyTankList:
            if MainGame.my_tank and MainGame.my_tank.live:
                if pygame.sprite.collide_rect(enemyTank, MainGame.my_tank):
                    enemyTank.stay()

    # 敌方坦克与敌方坦克碰撞
    def enemyTank_hit_enemyTank(self):
        for i in range(len(MainGame.enemyTankList)):
            if len(MainGame.enemyTankList) > 0 and i < len(MainGame.enemyTankList) - 1:
                if pygame.sprite.collide_rect(MainGame.enemyTankList[i], MainGame.enemyTankList[i+1]):
                    MainGame.enemyTankList[i].stay()
                    MainGame.enemyTankList[i+1].stay()
class Mytank(Tank):
    def __init__(self):
        pass
        # self.live = True

class EnemyTank(Tank):
    def __init__(self, top, left, speed):
        super(EnemyTank, self).__init__(top, left)
        # 将四个方向的坦克图片加载进来以字典的方式
        self.images = {
            'U': pygame.image.load('img/enemy1U.gif'),
            'L': pygame.image.load('img/enemy1L.gif'),
            'D': pygame.image.load('img/enemy1D.gif'),
            'R': pygame.image.load('img/enemy1R.gif'),
        }
        # 坦克坦克方向为随机获得
        self.direction = self.randDirection()
        # 根据方向获取图片对象
        self.image = self.images[self.direction]
        # 获取区域
        self.rect = self.image.get_rect()
        # 将坦克的位置放入rect内
        self.rect.left = left
        self.rect.top = top
        self.speed = 1
        self.stop = False
        self.step = random.randint(40, 100)  # 坦克移动方向前需要的步数
        self.live = True

    # 随机生成敌方坦克移动方向
    def randDirection(self):
        directions = ["U", "D", "L", "R"]
        num = random.randint(0, 3)
        return directions[num]

    def randMove(self):
        if self.step <= 0:
            self.direction = self.randDirection()
            self.step = random.randint(40, 100)
        else:
            self.move()
            self.step -= 1

    # 重写shot方法，只有随机数小于10时才发射子弹，其余情况返回None
    def shot(self):
        num = random.randint(0, 100)
        if num < 10:
            return Bullet(self)

class Bullet(BaseItem):
    def __init__(self, tank):
        self.image = pygame.image.load("img\enemymissile.gif")
        self.direction = tank.direction
        self.rect = self.image.get_rect()
        #子弹的left和top与方向有关
        if self.direction == 'U':
            self.rect.left = tank.rect.left + tank.rect.width / 2 - self.rect.width / 2
            self.rect.top = tank.rect.top - self.rect.height
        elif self.direction == 'D':
            self.rect.left = tank.rect.left + tank.rect.width / 2 - self.rect.width / 2
            self.rect.top = tank.rect.top + tank.rect.height
        elif self.direction == 'L':
            self.rect.left = tank.rect.left - self.rect.width / 2 - self.rect.width / 2
            self.rect.top = tank.rect.top + tank.rect.width / 2 - self.rect.width / 2
        elif self.direction == 'R':
            self.rect.left = tank.rect.left + tank.rect.width
            self.rect.top = tank.rect.top + tank.rect.width / 2 - self.rect.width / 2
        self.speed = 6  # 子弹的速度
        self.live = True    # 子弹的存活状态

    def display(self):
        MainGame.window.blit(self.image, self.rect)
    def move(self):
        if self.direction == "U":
            if self.rect.top - self.speed > 0:
                self.rect.top -= self.speed
            else:
                self.rect.top = 0
                self.live = False
        elif self.direction == "D":
            if self.rect.top + self.speed < SCRENN_HEIGHT - self.rect.height:
                self.rect.top += self.speed
            else:
                self.rect.top = SCRENN_HEIGHT - self.rect.height
                self.live = False
        elif self.direction == "L":
            if self.rect.left - self.speed > 0:
                self.rect.left -= self.speed
            else:
                self.rect.left = 0
                self.live = False
        elif self.direction == "R":
            if self.rect.left + self.speed < SCREEN_WIDTH - self.rect.height:
                self.rect.left += self.speed
            else:
                self.rect.left = SCREEN_WIDTH - self.rect.height
                self.live = False

    # 我方子弹与敌方坦克碰撞
    def myBullet_hit_enemyTank(self):
        for enemyTank in MainGame.enemyTankList:
            if pygame.sprite.collide_rect(self, enemyTank):
                self.live = False
                enemyTank.live = False
                # 发生碰撞之后创建爆炸对象
                explode = Explode(enemyTank)
                MainGame.explodeList.append(explode)
                # Music("fire").play()

    # 敌方子弹与我方坦克碰撞
    def enemyBullet_hit_myTank(self):
        if MainGame.my_tank and MainGame.my_tank.live:
            if pygame.sprite.collide_rect(self, MainGame.my_tank):
                self.live = False
                MainGame.my_tank.live = False
                MainGame.death_time += 1
                # 创建爆炸效果对象
                explode = Explode(MainGame.my_tank)
                MainGame.explodeList.append(explode)
                # Music("fire").play()

    # 检测子弹是否与墙壁碰撞
    def hitWall(self):
        for wall in MainGame.wallList:
            # 如果碰撞，则让子弹消失，墙壁生命值减少
            if pygame.sprite.collide_rect(self, wall):
                self.live = False
                wall.hp -= 1
                # 如墙壁生命值小于等于0，则让墙壁消失，并从墙壁列表中删除
                if wall.hp <= 0:
                    wall.live = False
                    MainGame.wallList.remove(wall)

    # 检测我方子弹与敌方子弹碰撞
    def hitBullet(self):
        for myBullet in MainGame.myBulletList:
            for enemyBullet in MainGame.enemyBulletList:
                if pygame.sprite.collide_rect(myBullet, enemyBullet):
                    myBullet.live = False
                    enemyBullet.live = False

    # 检测敌方子弹与敌方坦克碰撞
    def enemyBullet_hit_enemyTank(self):
        for enemyTank in MainGame.enemyTankList:
            for enemyBullet in MainGame.enemyBulletList:
                if pygame.sprite.collide_rect(enemyBullet, enemyTank):
                    enemyBullet.live = False
                    # enemyTank.live = False  # 敌方坦克自相残杀

class Wall():
    def __init__(self, top, left):
        # 加载墙壁图片
        self.image = pygame.image.load("img/steels.gif")
        # 获取区域
        self.rect = self.image.get_rect()
        # 给left和top赋值
        self.rect.left = left
        self.rect.top = top
        self.hp = 5     # 墙壁的生命值
        self.live = True    # 墙壁的存活状态
        # self.count = 6      # 墙壁的数量

    # 显示墙壁
    def display(self):
        if self.live:
            MainGame.window.blit(self.image, self.rect)

class Explode():
    def __init__(self, tank):
        # 加载爆炸图片集
        self.images = [
            pygame.image.load("img/blast0.gif"),
            pygame.image.load("img/blast1.gif"),
            pygame.image.load("img/blast2.gif"),
            pygame.image.load("img/blast3.gif"),
            pygame.image.load("img/blast4.gif")
        ]
        self.rect = tank.rect   # 爆炸的位置由坦克决定
        self.step = 0   # 步数，索引
        self.image = self.images[self.step]     # 根据索引获取爆炸对象
        self.live = True    # 爆炸对象的存活状态

    # 由于这个方法会在循环内持续调用，所以这里只需要设置好一个爆炸对象即可
    # 循环内系统会多次判断碰撞，之后将五张图片都存放在爆炸效果列表内
    def display(self):
        # 根据索引获取爆炸对象
        if self.step < len(self.images):
            self.image = self.images[self.step]
            self.step += 1
        else:
            self.live = False
            self.step = 0
        # 展示爆炸效果
        MainGame.window.blit(self.image, self.rect)
class Music():
    def __init__(self, music):
        # 初始化混合器
        pygame.mixer.init()
        if music == "start":
            file = pygame.mixer.music.load("img/start.wav")
        elif music == "hit":
            file = pygame.mixer.music.load("img/hit.wav")
        elif music == "fire":
            file = pygame.mixer.music.load("img/fire.wav")
        # 加载音效文件
    def play(self):
        # if not pygame.mixer.music.get_busy():
        pygame.mixer.music.play()

if __name__ == "__main__":
    MainGame().startGame()


import logging
from logging.handlers import RotatingFileHandler
import colorlog

# Create the logger
# pylint: disable=invalid-name
logging.basicConfig()
Log = logging.getLogger()
logging.getLogger('javaobj').setLevel(logging.INFO)
logging.getLogger('httpx').setLevel(logging.INFO)
logging.getLogger('kazoo').setLevel(logging.INFO)

# time formatter - date - time - UTC offset
# e.g. "08/16/1988 21:30:00 +1030"
# see time formatter documentation for more
date_format = "%Y-%m-%d %H:%M:%S %z"

def configure(level=logging.INFO, logfile=None):
  """ Configure logger which dumps log on terminal

  :param level: logging level: info, warning, verbose...
  :type level: logging level
  :param logfile: log file name, default to None
  :type logfile: string
  :return: None
  :rtype: None
  """

  # Remove all the existing StreamHandlers to avoid duplicate
  for handler in Log.handlers:
    if isinstance(handler, logging.StreamHandler):
      Log.handlers.remove(handler)

  Log.setLevel(level)

  # if logfile is specified, FileHandler is used
  if logfile is not None:
    log_format = "[%(asctime)s] [%(levelname)s]: %(message)s"
    formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
    file_handler = logging.FileHandler(logfile)
    file_handler.setFormatter(formatter)
    Log.addHandler(file_handler)
  # otherwise, use StreamHandler to output to stream (stdout, stderr...)
  else:
    log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s"
    formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    Log.addHandler(stream_handler)

def init_rotating_logger(level, logfile, max_files, max_bytes):
  """Initializes a rotating logger

  It also makes sure that any StreamHandler is removed, so as to avoid stdout/stderr
  constipation issues
  """
  logging.basicConfig()

  root_logger = logging.getLogger()
  log_format = "[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s"

  root_logger.setLevel(level)
  handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)
  handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))
  root_logger.addHandler(handler)

  for handler in root_logger.handlers:
    root_logger.debug("Associated handlers - %s", str(handler))
    if isinstance(handler, logging.StreamHandler):
      root_logger.debug("Removing StreamHandler: %s", str(handler))
      root_logger.handlers.remove(handler)

def set_logging_level(cl_args):
  """simply set verbose level based on command-line args

  :param cl_args: CLI arguments
  :type cl_args: dict
  :return: None
  :rtype: None
  """
  if 'verbose' in cl_args and cl_args['verbose']:
    configure(logging.DEBUG)
  else:
    configure(logging.INFO)


import argparse
import os
import time
from datetime import datetime

import torch
from torch import nn
from torch.utils.data import TensorDataset, DataLoader
from tqdm import tqdm
from tst import Transformer

from seq2seq_transduction.data import load_dataset
from seq2seq_transduction.utils import time_since, plot_loss_history

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def train(dataset_path, batch_size, num_epochs, learning_rate):
    normalized_carrier_train, normalized_params_train, normalized_modulated_train, _, _ = load_dataset(dataset_path, kind="train")
    train_dataset = TensorDataset(torch.from_numpy(normalized_carrier_train).float(),
                            torch.from_numpy(normalized_params_train).float(),
                            torch.from_numpy(normalized_modulated_train).float())
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=1,
                             pin_memory=True, drop_last=True)

    normalized_carrier_validate, normalized_params_validate, normalized_modulated_validate, _, _ = load_dataset(dataset_path, kind="validate")
    validate_dataset = TensorDataset(torch.from_numpy(normalized_carrier_validate).float(),
                            torch.from_numpy(normalized_params_validate).float(),
                            torch.from_numpy(normalized_modulated_validate).float())
    validate_loader = DataLoader(validate_dataset, batch_size=batch_size, shuffle=True, num_workers=1,
                             pin_memory=True, drop_last=True)

    p = {
        "d_model": 64,  # Latent dim
        "q": 8,         # Query size
        "v": 8,         # Value size
        "h": 8,         # Number of heads
        "N": 3,         # Number of encoder and decoder to stack
        "attention_size": 12,  # Attention window size
        "dropout": 0.3,        # Dropout rate
        "pe": None,            # Positional encoding
        "chunk_mode": None,
        "d_input": normalized_carrier_train.shape[-1] + normalized_params_train.shape[-1],
        "d_output": normalized_modulated_train.shape[-1],
        "batch_size": batch_size,
        "seq_len": normalized_carrier_train.shape[1]
    }

    model = Transformer(p["d_input"], p["d_model"], p["d_output"], p["q"], p["v"], p["h"], p["N"], p["attention_size"],
                        p["dropout"], p["chunk_mode"], p["pe"]).to(device)
    output_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, os.pardir, "output", "transformer")
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    loss_history = train_loop(model, train_loader, validate_loader, num_epochs, learning_rate)
    timestamp = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S")
    torch.save({"parameters": p, "state_dict": model.state_dict()}, os.path.join(output_dir, f"{timestamp}_model.pt"))
    plot_loss_history(loss_history, os.path.join(output_dir, "{}_loss.png".format(timestamp)), labels=["Train", "Validate"])

def train_loop(model, train_loader, val_loader, n_epochs, learning_rate):
    start = time.time()
    loss_history = []

    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-4)
    criterion = nn.MSELoss()

    for epoch in range(1, n_epochs + 1):
        train_loss = 0
        model.train()
        for carrier_sig, params, modulated_sig in tqdm(train_loader):
            param_seq_batch = params.unsqueeze(1).repeat(1, carrier_sig.size(1), 1)
            input_seq_batch = torch.cat((param_seq_batch, carrier_sig), dim=-1).to(device)
            target_seq_batch = modulated_sig.to(device)
            batch_loss = train_step(input_seq_batch, target_seq_batch, model, optimizer, criterion, validate=False)
            train_loss += batch_loss

        avg_train_loss = train_loss / len(train_loader)
        print('%s (%d %d%%) %.4f' % (time_since(start, epoch / n_epochs),
                                     epoch, epoch / n_epochs * 100, avg_train_loss))

        val_loss = 0
        model.eval()
        with torch.no_grad():
            for carrier_sig, params, modulated_sig in tqdm(val_loader):
                param_seq_batch = params.unsqueeze(1).repeat(1, carrier_sig.size(1), 1)
                input_seq_batch = torch.cat((param_seq_batch, carrier_sig), dim=-1).to(device)
                target_seq_batch = modulated_sig.to(device)
                batch_loss = train_step(input_seq_batch, target_seq_batch, model, optimizer, criterion, validate=True)
                val_loss += batch_loss
        avg_val_loss = val_loss / len(val_loader)

        loss_history.append([avg_train_loss, avg_val_loss])
    return loss_history

def train_step(input_tensor, target_tensor, model, optimizer, criterion, validate=False):
    output = model(input_tensor)
    loss = criterion(output, target_tensor)
    if not validate:
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return loss.item()

def main(args):
    batch_size = 32
    learning_rate = 5e-4
    num_epochs = 100
    train(args.dataset_path, batch_size, num_epochs, learning_rate)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("dataset_path", type=str)
    main(parser.parse_args())