import importlib
import json
import os
import os.path

from settings.settingtype import SettingType, LocalSetting

class Settings(object):
    def __init__(self):
        self._settings_module = None

    def __getattr__(self, name):
        if not self._settings_module:
            mod_name = SettingType().get_env()
            self._settings_module = importlib.import_module(mod_name)
            # 将本地配置更新到settings模块中
            LocalSetting().add_local_setting(self._settings_module)
        return getattr(self._settings_module, name)

settings = Settings()
'全局设置'

class PersistData(object):
    def __init__(self):
        self._data = {}

    def _get_data_file(self):
        data_dir = os.getenv("TCA_APP_DATA_DIR")  # 支持通过环境变量指定app数据目录，存放节点NODE_UUID唯一值
        if data_dir:
            data_dir = os.path.abspath(data_dir)
        else:
            data_dir = os.path.join(settings.BASE_DIR, '.appdata')
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        return os.path.join(data_dir, 'appdata.json')

    def _load_data(self):
        if not self._data:
            datafile = self._get_data_file()
            if os.path.exists(datafile):
                with open(self._get_data_file(), 'r') as f:
                    self._data = json.load(f)

    def __getitem__(self, key):
        self._load_data()
        return self._data[key]

    def __setitem__(self, key, value):
        self._data[key] = value
        datafile = self._get_data_file()
        with open(datafile, 'w') as f:
            json.dump(self._data, f)

    def get(self, key, default=None):
        self._load_data()
        return self._data.get(key, default)

persist_data = PersistData()
'应用的持久化数据存取'


import time
import unittest
import os

import redis
from selenium import webdriver

os.environ['RATCHAT_TESTING'] = 'False'
from ratchat import redis_db
assert type(redis_db) == redis.client.StrictRedis
redis_db.flushall()

host = "http://0.0.0.0:5000"

class TestChat(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Chrome()

    def tearDown(self):
        self.driver.close()

    def get_active_users(self):
        active_users_element = self.driver.find_element_by_id('users-window')
        return active_users_element.text.split('\n')

    def get_message_list(self, filter_by=None, driver=None):
        if driver is None:
            driver = self.driver
        chat_window = driver.find_element_by_id('chat-window')
        messages = chat_window.text.splitlines()
        if filter_by is not None:
            return [message for message in messages if filter_by in message]
        return messages

    def send_input(self, string, driver=None):
        if driver is None:
            driver = self.driver
        chat_input = driver.find_element_by_id('chat-input')
        chat_input.clear()
        chat_input.send_keys(string)
        chat_input.submit()

    def test_basic_chat_functionality(self):

        # Reggie loads page and sees a chat window

        self.driver.get(host)
        self.assertIn('ratchat', self.driver.title)

        # Reggie sends a message and sees that a username has been
        # assigned automatically

        original_messages = self.get_message_list()
        self.send_input('Hello room')
        new_messages = self.get_message_list()
        self.assertGreater(len(new_messages), len(original_messages))

        # Reggie sees that the automatically assigned username shows
        # up in a window with all the other active users

        time.sleep(1)
        name = self.get_message_list(filter_by='Hello room')[-1].split(':')[0]
        active_users = self.get_active_users()

        self.assertIn(name, active_users)

    def test_chat_commands(self):
        self.driver.get(host)

        # Reggie saw a message from the server that showed him how
        # to see a list of commands he could use

        messages = self.get_message_list(filter_by='server')
        self.assertIn('server: Type /help for a list of commands.', messages)

        # Reggie decides to change his alias

        self.send_input('/callme Reginald')
        self.send_input("What's my name?")
        messages = self.get_message_list(filter_by="What's my name?")
        name = messages[-1].split(':')[0]
        self.assertEqual(name, 'Reginald')

        # Reggie decides to create a password protected login

        self.send_input('/login testuser1 testpass')
        messages = self.get_message_list(filter_by='server')
        self.assertIn('server: Login Successful', messages)
        time.sleep(2)
        active_users = self.get_active_users()
        self.assertIn('testuser1', active_users)

        # A friend logs in as testuser2 and a stranger enters the chat

        driver2 = webdriver.Firefox()
        driver2.get(host)
        time.sleep(2)
        self.send_input('/login testuser2 testpass', driver=driver2)
        time.sleep(2)

        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--incognito')
        driver3 = webdriver.Chrome(chrome_options=chrome_options)
        driver3.get(host)

        time.sleep(2)

        # Reggie notices the friend in the active users window

        active_users = self.get_active_users()
        self.assertIn('testuser2', active_users)

        # Reggie sends a private message to his friend and sees it in his
        # chat_window

        self.send_input('This is public')
        self.send_input('/msg testuser2 This is private.')
        time.sleep(2)
        last_message = self.get_message_list()[-1]
        self.assertIn('This is private.', last_message)

        # Reggie's friend also sees the message

        last_message = self.get_message_list(driver=driver2)[-1]
        self.assertIn('This is private.', last_message)

        # The stranger did not receive the message

        last_message = self.get_message_list(driver=driver3)[-1]
        self.assertIn('This is public', last_message)
        driver3.close()

        # testuser2 logs off and is no longer visible in the users window

        driver2.quit() # Using quit instead of close because of issue with Selenium
        time.sleep(2)
        active_users = self.get_active_users()
        self.assertNotIn('testuser2', active_users)

        # Reggie can't send any more private messages to testuser2.
        # The server notifies him that testuser2 is not available

        self.send_input('/msg testuser2 You still here?')
        last_message = self.get_message_list()[-1]
        self.assertIn('testuser2 is not currently active', last_message)

    def test_registered_names(self):
        pass


import tensorflow as tf
from tf_agents.agents import DqnAgent
from tf_agents.agents.dqn.dqn_agent import element_wise_squared_loss
from tf_agents.networks.q_network import QNetwork
from tf_agents.replay_buffers.tf_uniform_replay_buffer import TFUniformReplayBuffer

class ReplayDqnAgent(DqnAgent):

    def __init__(self, time_step_spec, observation_spec, action_spec,
                 replay_memory_size, replay_memory_batch_size, data_batch_size,
                 q_network_layers=(64,), alpha=0.0001, gamma=1.0,
                 epsilon_greedy=0.1, boltzmann_temperature=None):

        q_network = QNetwork(observation_spec, action_spec, fc_layer_params=q_network_layers)
        optimizer = tf.compat.v1.train.AdamOptimizer(alpha)
        train_step_counter = tf.compat.v2.Variable(0)

        super().__init__(time_step_spec, action_spec, q_network, optimizer,
                         gamma=gamma,
                         epsilon_greedy=epsilon_greedy,
                         boltzmann_temperature=boltzmann_temperature,
                         train_step_counter=train_step_counter,
                         td_errors_loss_fn=element_wise_squared_loss)

        self.replay_memory_size = replay_memory_size
        self.replay_memory_batch_size = replay_memory_batch_size
        self.data_batch_size = data_batch_size

        self._initialize()

    def _initialize(self):
        super()._initialize()
        self._initialize_replay_memory()

    def _initialize_replay_memory(self):
        self.replay_buffer = TFUniformReplayBuffer(self.collect_data_spec, self.data_batch_size, self.replay_memory_size)
        dataset = self.replay_buffer.as_dataset(self.replay_memory_batch_size,
                                                num_steps=2, num_parallel_calls=3).prefetch(3)
        self.replay_memory_iterator = iter(dataset)

    def feedback(self, datapoint):
        replay_buffer = self.replay_buffer
        replay_buffer.add_batch(datapoint)


'''
rggSession - for the Random Game Generator project

A class representing a savable game state of pogs, lines, and maps.

By Doctus (kirikayuumura.noir@gmail.com)

    This file is part of RandomGameGenerator.

    RandomGameGenerator is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    RandomGameGenerator is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public License
    along with RandomGameGenerator.  If not, see <http://www.gnu.org/licenses/>.
'''
from sys import maxsize

from libraries.rggPog import Pog
from libraries.rggMap import Map
from libraries.rggEvent import pogUpdateEvent, pogDeleteEvent, mapCreatedEvent, mapRemovedEvent
from libraries.rggJson import loadObject
from libraries.rggSystem import clearLines, drawLine, findRandomAppend

class Session(object):

	def __init__(self):
		self.maps = {}
		self.pogs = {}

		self.lines = []
		self.linesDict = {}

		self.maphack = 0

	def _addMap(self, mappe):
		'''Adds a map that is already loaded and has an ID.'''
		assert(mappe.ID is not None)
		if mappe.drawOffset == [0, 0]:
			pos = self.maphack
			mappe.drawOffset = (pos, 0)
		self.maps[mappe.ID] = mappe
		self.maphack += mappe.pixelSize[0] + 25
		mapCreatedEvent(mappe)

	def _addPog(self, pog):
		'''Adds a pog to the session without informing rggEvent.'''
		assert(pog.ID is not None)
		self.pogs[pog.ID] = pog

	def _findUniqueMapID(self, src):
		'''Get a unique id for a map.'''
		id = src or findRandomAppend()
		while id in list(self.maps.keys()):
			id += findRandomAppend()
		return id

	def _findUniquePogID(self, src):
		'''Get a unique id for a pog.'''
		id = src or findRandomAppend()
		while id in list(self.pogs.keys()):
			id += findRandomAppend()
		return id

	def addPog(self, pog):
		'''Adds a pog to the session, informing rggEvent of the addition.'''
		assert(pog.ID is not None)
		#rggResource.srm.processFile(localuser(), pog._src)
		self.pogs[pog.ID] = pog
		pogUpdateEvent(pog)
		self.hideAllHiddenPogs()

	def removePog(self, pog):
		'''Deletes a pog.'''
		assert(pog.ID is not None)
		pogDeleteEvent(self.pogs[pog.ID])
		self.pogs[pog.ID].destroy()
		del self.pogs[pog.ID]

	def removeAllPogs(self):
		'''Deletes all pogs in the session.'''
		for pog in list(self.pogs.values()):
			pogDeleteEvent(pog)
			pog.destroy()
		self.pogs = {}

	def addMap(self, mappe):
		'''Creates a new map and assigns it a unique ID.'''
		if mappe.drawOffset == [0, 0]:
			pos = self.maphack
			mappe.drawOffset = (pos, 0)
		self.maphack += mappe.pixelSize[0] + 25

		ID = self._findUniqueMapID(mappe.mapname)
		mappe.ID = ID

		#rggResource.srm.processFile(localuser(), map.tileset)

		self.maps[ID] = mappe

		mapCreatedEvent(mappe)

	def addDumpedMap(self, dump, ID):
		'''Adds a map from a JSON dump with the specified ID.'''
		mappe = Map.load(dump)
		mappe.ID = ID
		self._addMap(mappe)

	def getMapExists(self, ID):
		'''Returns whether a map with the given ID exists in this session.'''
		return ID in list(self.maps.keys())

	def getMap(self, ID):
		'''Returns the map with a given ID.'''
		return self.maps[ID]

	def findTopMap(self, mapPosition):
		'''Returns the top map at a given position, or None.'''
		for mappe in list(self.maps.values()):
			size = mappe.pixelSize
			if mapPosition[0] >= mappe.drawOffset[0] and mapPosition[0] <= mappe.drawOffset[0] + size[0]:
				if mapPosition[1] >= mappe.drawOffset[1] and mapPosition[1] <= mappe.drawOffset[1] + size[1]:
					return mappe
		return None

	def closeMap(self, ID):
		'''Deletes a specified map and its tiles.'''
		self.maps[ID]._deleteTiles()
		del self.maps[ID]
		mapRemovedEvent(ID)

	def closeAllMaps(self):
		'''Deletes all maps and their tiles.'''
		self.maphack = 0
		for mappe in list(self.maps.values()):
			mappe._deleteTiles()
			mapRemovedEvent(mappe.ID)
		self.maps = {}

	def findTopPog(self, position):
		'''Returns the top pog at a given position, or None.'''
		layer = -maxsize
		locked = True
		top = None
		for pog in list(self.pogs.values()):
			if (layer >= pog.layer and not (locked and not pog._locked)) or (not locked and pog._locked):
				continue
			if pog.pointCollides(position):
				top = pog
				layer = top.layer
				locked = pog._locked
		return top

	def refreshPogs(self):
		'''Forcibly refreshes all pogs in the session.'''
		for pog in list(self.pogs.values()):
			pog.forceUpdate()

	def storeLines(self):
		'''Stores all lines of which the session is aware in a JSON-compatible format in self.lines'''
		self.lines = []

		for thickness, lines in list(self.linesDict.items()):
			for item in lines:
				self.lines.extend( [item[0], item[1], item[2], item[3], thickness, item[4], item[5], item[6]] )

	def restoreLines(self):
		'''Draws all lines of which the session is aware.'''
		for thickness, lines in list(self.linesDict.items()):
			for item in lines:
				drawLine(item[0], item[1], item[2], item[3], thickness, item[4], item[5], item[6])

	def _pointIntersectRect(self, point, rect):
		'''Check used by deleteLine.'''
		if point[0] < rect[0] or point[0] > rect[0] + rect[2]: return False
		if point[1] < rect[1] or point[1] > rect[1] + rect[3]: return False
		return True

	def deleteLine(self, x, y, w, h):
		'''Remove all lines within the x, y, w, h rect from the session's awareness. This does not delete the lines themselves.'''
		for thickness, lines in list(self.linesDict.items()):
			dellist = []
			for line in lines:
				if self._pointIntersectRect((line[0], line[1]), (x, y, w, h)) or self._pointIntersectRect((line[2], line[3]), (x, y, w, h)):
					dellist.append(line)
			for index in dellist:
				lines.remove(index)

	def addLine(self, line):
		'''Add a line to the session's awareness from a tuple of parameters. This does not create a graphical line.'''
		if not line[4] in self.linesDict:
			self.linesDict[line[4]] = []
		self.linesDict[line[4]].append([line[0], line[1], line[2], line[3], line[5], line[6], line[7]])

	def hideAllHiddenPogs(self):
		'''Checks through all pogs to make sure the ones that should be hidden, are. Workaround for a glwidget issue.'''
		for pog in list(self.pogs.values()):
			if pog.hidden:
				pog.hide()

	def clear(self):
		'''Clear all session data to prepare for loading a new one.'''
		for map in list(self.maps.values()):
			map._deleteTiles()
		for pog in list(self.pogs.values()):
			pog.destroy()
		clearLines()
		self.maps = {}
		self.pogs = {}
		self.lines = []
		self.linesDict = {}
		self.maphack = 0

	def hide(self):
		'''Hide all maps, pogs, and lines while retaining their data.'''
		for map in list(self.maps.values()):
			map.hide()
		for pog in list(self.pogs.values()):
			pog._realHide(True)
		clearLines()

	def show(self):
		'''Restores display of maps, pogs, and lines after a hide() call.'''
		for map in list(self.maps.values()):
			map.show()
		for pog in list(self.pogs.values()):
			pog._realHide(False)
		self.restoreLines()

	def dump(self):
		'''Serialize to an object valid for JSON dumping.'''

		self.storeLines()

		return dict(
			pogs=dict([(pog.ID, pog.dump()) for pog in list(self.pogs.values())]),
			maps=dict([(mappe.ID, mappe.dump()) for mappe in list(self.maps.values())]),
			lines=self.lines,
			maphack=self.maphack)

	@staticmethod
	def load(obj):
		"""Deserialize a new map from a dictionary."""
		sess = Session()

		pogs = loadObject('Session.pogs', obj.get('pogs'))
		for ID, pog in list(pogs.items()):
			loaded = Pog.load(pog)
			loaded.ID = ID
			sess._addPog(loaded)

		for pog in list(sess.pogs.values()):
			if pog.hidden:
				pog.hide()

		maps = loadObject('Session.maps', obj.get('maps'))
		for ID, mappe in list(maps.items()):
			loaded = Map.load(mappe)
			loaded.ID = ID
			sess._addMap(loaded)

		linez = obj.get('lines')
		brk = lambda x,y: [x[i:i+y] for i in range(0,len(x),y)]
		for line in brk(linez, 8):
			sess.addLine(line)

		sess.restoreLines()

		try:
			mh = obj.get('maphack')
			sess.maphack = int(mh)
		except:
			print("Loading old session - map placement may be wrong")

		return sess


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module that contains Maya DCC scene file parser implementation
"""

from __future__ import print_function, division, absolute_import

import sys
import time
import logging

from artella import api
from artella.core import utils, splash

from artella.externals.Qt import QtCore, QtWidgets

logger = logging.getLogger('artella')

class MayaDowndloaderWorkerSignals(QtCore.QObject):
    finished = QtCore.Signal()
    statusUpdated = QtCore.Signal(str, int, int, int, int, int)

class MayaDownloaderWorker(QtCore.QRunnable):
    def __init__(self, artella_drive_client, file_paths):
        super(MayaDownloaderWorker, self).__init__()

        self.signals = MayaDowndloaderWorkerSignals()

        self._artella_drive_client = artella_drive_client
        self._file_paths = file_paths

    def run(self):
        if not self._artella_drive_client or not self._file_paths:
            self.signals.finished.emit()
            return

        self._artella_drive_client.download(self._file_paths)

        # We force the waiting to a high value, otherwise Artella Drive Client will return that no download
        # is being processed
        time.sleep(3.5)

        while True:
            progress, fd, ft, bd, bt = self._artella_drive_client.get_progress()
            current = int(bd / 1024)
            total = int(bt / 1024)
            progress_status = '{}/{} KiB'.format(current, total)
            self.signals.statusUpdated.emit(progress_status, progress, current, total, fd, ft)
            if progress >= 100 or bd == bt:
                break

        self.signals.finished.emit()

class MayaSceneParserThreadPool(QtCore.QObject):

    poolStarted = QtCore.Signal(list)
    poolFinished = QtCore.Signal()
    workerFinished = QtCore.Signal()
    workerStatusUpdated = QtCore.Signal(str, int, int, int, int, int)

    def __init__(self, max_thread_count=1):
        super(MayaSceneParserThreadPool, self).__init__()

        self._pool = QtCore.QThreadPool()
        self._pool.setMaxThreadCount(max_thread_count)

    def _on_worker_finished(self):
        self.workerFinished.emit()
        self.poolFinished.emit()

    def start(self, artella_drive_client, file_paths):
        self._file_paths = file_paths
        self._processed = 0

        self.poolStarted.emit(file_paths)

        worker = MayaDownloaderWorker(artella_drive_client, file_paths)
        worker.signals.finished.connect(self._on_worker_finished)
        worker.signals.statusUpdated.connect(self.workerStatusUpdated.emit)
        self._pool.start(worker)
        time.sleep(0.1)

class MayaDownloader(QtCore.QObject, object):
    """
    Class that defines Maya scene parser functions
    """

    def __init__(self):
        super(MayaDownloader, self).__init__()

        self._finished = False
        self._total_files = 0
        self._downloaded_files = list()

        self._progress_splash = None

        self._thread_pool = MayaSceneParserThreadPool(max_thread_count=10)
        self._thread_pool.poolStarted.connect(self._on_start_parsing)
        # self._thread_pool.workerFinished.connect(self._on_worker_finished)
        self._thread_pool.poolFinished.connect(self._on_download_finished)
        self._thread_pool.workerStatusUpdated.connect(self._on_status_updated)

    @utils.timestamp
    def download(self, file_paths=None, show_dialogs=True):
        """
        Parses all the contents of the given file path looking for file paths

        :param str or None file_paths: Absolute local file paths of the DCC file we want to parse. If not given,
            current opened DCC scene file path will be used
        :return:
        """

        self._finished = False

        artella_drive_client = api.get_client()
        if not artella_drive_client or not artella_drive_client.check(update=True):
            self._finished = True
            return self._downloaded_files

        if show_dialogs:
            self._progress_splash = splash.DownloadSplashDialog(downloader=self)
            # self._thread_pool.workerStatusUpdated.connect(self._progress_splash.update_download)

        file_paths = utils.force_list(file_paths)
        if not file_paths:
            return

        self._thread_pool.start(artella_drive_client, file_paths)

        if self._progress_splash:
            self._progress_splash.download(file_paths)
        else:
            while not self._finished:
                time.sleep(0.1)

        return self._downloaded_files

    def _on_start_parsing(self, file_paths):
        """
        Internal callback function that is called just before parsing process
        """

        self._total_files = len(file_paths)
        if self._progress_splash:
            self._progress_splash.set_max_progress_value(100)
            self._progress_splash.set_min_progress_value(0)
            self._progress_splash.set_progress_value(0, 'Downloading files ...')
            self._progress_splash.set_infinite(True)
            self._progress_splash.repaint()

    def _on_status_updated(self, status, progress, download_size, total_size, downloaded_files, total_files):
        if self._progress_splash and progress:
            self._progress_splash.set_infinite(False)
            status = '{} ({} / {})'.format(status, downloaded_files, total_files)
            self._progress_splash.set_max_progress_value(total_size)
            self._progress_splash.set_progress_value(download_size, status)

    def _on_download_finished(self):
        if self._progress_splash:
            self._progress_splash.end()

        self._downloaded_files = list(set(self._downloaded_files))

        self._finished = True

    # def _on_worker_finished(self, file_path):
    #     self._downloaded_files.append(file_path)
    #     if self._progress_splash:
    #         next_progress = self._progress_splash.get_progress_value() + 1
    #         self._progress_splash.set_progress_value(
    #             next_progress, 'Downloading files ... ({} / {})'.format(next_progress, self._total_files))

if __name__ == '__main__':
    app = QtWidgets.QApplication.instance()
    if not app:
        app = QtWidgets.QApplication(sys.argv)

    import artella.loader
    plugin_paths = [r'D:\dev\artella\plugins']
    dcc_paths = [r'D:\dev\artella\dccs']
    artella.loader.init(dev=True, dcc_paths=dcc_paths, plugin_paths=plugin_paths)

    files_to_download = [
        r"D:\shorts\artella-files\Crucible\Production\Seq-001\Shot-001\Anim\Shot-001-anim.ma",
        r"D:\shorts\artella-files\Showcase 2020\Production\Shot_114\Assets\overlord_baker.ma"
    ]
    parser = MayaDownloader()
    parsed_files = parser.download(files_to_download, show_dialogs=True)
    for parsed_file in parsed_files:
        print(parsed_file)

    sys.exit(app.exec_())


print("To find: The smallest 'N' digit number with the sum of its digits equal to 'S'")
s = int(input("Enter a value for S: "))
d = int(input("Enter a value for N: "))
for j in range(10**(d-1), 10**(d)):
    num = j
    sm = 0
    while j>0:
        l = j % 10
        sm += l
        j //= 10
    if sm == s:
        print(num)
        break
else:
    print("There is no",d,"digit number with sum of its digits equal to",s)


# script to calculate rate factors for e.g. ME weights to ensure that there is only a shape change and not a normalization/yield change
import ROOT
import glob
import os
import csv
import optparse

parser = optparse.OptionParser()
parser.add_option("-i",dest="input",
    help = "input fraction file")
parser.add_option("-r",dest="ratefactorfile",
    help = "input ratefactor file")
parser.add_option("-y",dest="year",
    help = "year of data taking period")
(opts, args) = parser.parse_args()

directory = os.path.realpath(opts.input)
print("calculating ratefactors for ntuples in directory: {}".format(directory))

with open(opts.input, "r") as f:
    fraction_file = f.readlines()
sample_dict = {}
for line in fraction_file:
    entry = line.replace("\n","").replace("\"","").split(",")
    if not entry[0] in sample_dict:
        sample_dict[entry[0]] = {}
    sample_dict[entry[0]][entry[1]] = [entry[2], entry[3], entry[4], entry[5]]

with open(opts.ratefactorfile, "r") as f:
    rf_file = f.readlines()

if opts.year == "2017":
    translations_5fs = {
        "TTbb_Powheg_Openloops_DL": "TTTo2L2Nu_TuneCP5_PSweights_13TeV-powheg-pythia8",
        "TTbb_Powheg_Openloops_new_pmx": "TTToSemiLeptonic_TuneCP5_PSweights_13TeV-powheg-pythia8",
        "TTbb_4f_TTToHadronic_TuneCP5-Powheg-Openloops-Pythia8": "TTToHadronic_TuneCP5_PSweights_13TeV-powheg-pythia8_new_pmx",
        }
elif opts.year == "2018":
    translations_5fs = {
        "TTbb_4f_TTTo2l2nu_TuneCP5-Powheg-Openloops-Pythia8": "TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8",
        "TTbb_4f_TTToSemiLeptonic_TuneCP5-Powheg-Openloops-Pythia8": "TTToSemiLeptonic_TuneCP5_13TeV-powheg-pythia8",
        "TTbb_4f_TTToHadronic_TuneCP5-Powheg-Openloops-Pythia8": "TTToHadronic_TuneCP5_13TeV-powheg-pythia8",
        }
elif opts.year == "2016":
    translations_5fs = {
        "TTbb_4f_TTTo2l2nu_TuneCP5-Powheg-Openloops-Pythia8": "TTTo2L2Nu_TuneCP5_PSweights_13TeV-powheg-pythia8",
        "TTbb_4f_TTToSemiLeptonic_TuneCP5-Powheg-Openloops-Pythia8": "TTToSemiLeptonic_TuneCP5_PSweights_13TeV-powheg-pythia8",
        "TTbb_4f_TTToHadronic_TuneCP5-Powheg-Openloops-Pythia8": "TTToHadronic_TuneCP5_PSweights_13TeV-powheg-pythia8",
        }
elif opts.year == "comb":
    translations_5fs = {
        "TTbbDL": "TTDL",
        "TTbbFH": "TTFH",
        "TTbbSL": "TTSL",
        }
else:
    sys.exit("need to specify a year for sample matching")

branches = [
    "nominal",
    #"Weight_scale_variation_muR_1p0_muF_1p0",
    "Weight_scale_variation_muR_2p0_muF_1p0",
    #"Weight_scale_variation_muR_2p0_muF_2p0",
    "Weight_scale_variation_muR_0p5_muF_1p0",
    #"Weight_scale_variation_muR_0p5_muF_0p5",
    "Weight_scale_variation_muR_1p0_muF_2p0",
    "Weight_scale_variation_muR_1p0_muF_0p5",
    #"Weight_LHA_306000_nominal",
    #"Weight_LHA_306000_up",
    #"Weight_LHA_306000_down",
    #"Weight_LHA_320900_nominal",
    #"Weight_LHA_320900_up",
    #"Weight_LHA_320900_down"
]
#branches += ["Weight_pdf_variation_306{:03}".format(i) for i in range(0,103)]
#branches += ["Weight_pdf_variation_32{:04}".format(i) for i in range(900,1001)]
branches += [
    "GenWeight_fsr_Def_down",
    "GenWeight_fsr_Def_up",
    #"GenWeight_fsr_Def_nom",
    "GenWeight_isr_Def_down",
    "GenWeight_isr_Def_up",
    #"GenWeight_isr_Def_nom",
    ]

new_file = []
for line in rf_file:
    line = line.replace("\n","").replace("\"","")
    entries = line.split(",")
    if entries[0] == "sample":
        entries.append("fraction_ttB_4FS")
        entries.append("fraction_ttB_5FS")
        entries.append("fraction_ttB_4FS_nom")
        entries.append("fraction_ttB_5FS_nom")
        entries.append("fraction_ttC_4FS")
        entries.append("fraction_ttC_5FS")
        entries.append("fraction_ttC_4FS_nom")
        entries.append("fraction_ttC_5FS_nom")
        entries.append("fraction_ttLF_4FS")
        entries.append("fraction_ttLF_5FS")
        entries.append("fraction_ttLF_4FS_nom")
        entries.append("fraction_ttLF_5FS_nom")
        entries.append("ratio_nominal_vs_varied_4FS")
        entries.append("ratio_nominal_vs_varied_5FS")

    else:
        ttbb_frac = sample_dict[entries[0]][entries[1]][0]
        nom_frac = sample_dict[entries[0]]["nominal"][0]
        ttcc_frac = sample_dict[entries[0]][entries[1]][2]
        nom_cc_frac = sample_dict[entries[0]]["nominal"][2]
        ttlf_frac = sample_dict[entries[0]][entries[1]][3]
        nom_lf_frac = sample_dict[entries[0]]["nominal"][3]

        rf_fiducial = float(entries[2])
        rf_inclusive = float(entries[3])

        nonapplicable = False
        if entries[0].startswith("TTbb") and entries[1] in branches:
            samplename_5fs = translations_5fs[entries[0]]

            ttbb_frac_5fs = sample_dict[samplename_5fs][entries[1]][0]
            nom_frac_5fs = sample_dict[samplename_5fs]["nominal"][0]
            ttcc_frac_5fs = sample_dict[samplename_5fs][entries[1]][2]
            nom_cc_frac_5fs = sample_dict[samplename_5fs]["nominal"][2]
            ttlf_frac_5fs = sample_dict[samplename_5fs][entries[1]][3]
            nom_lf_frac_5fs = sample_dict[samplename_5fs]["nominal"][3]
        else:
            continue

        entries.append(str(ttbb_frac))
        entries.append(str(ttbb_frac_5fs))
        entries.append(str(nom_frac))
        entries.append(str(nom_frac_5fs))
        entries.append(str(ttcc_frac))
        entries.append(str(ttcc_frac_5fs))
        entries.append(str(nom_cc_frac))
        entries.append(str(nom_cc_frac_5fs))
        entries.append(str(ttlf_frac))
        entries.append(str(ttlf_frac_5fs))
        entries.append(str(nom_lf_frac))
        entries.append(str(nom_lf_frac_5fs))
        entries.append(str(float(nom_frac)/float(ttbb_frac)))
        entries.append(str(float(nom_frac_5fs)/float(ttbb_frac_5fs)))

    new_file.append(",".join(entries))

outf = opts.ratefactorfile.replace(".csv","ttbb.csv")
with open(outf, "w") as f:
    f.write("\n".join(new_file))
print("wrote new file to {}".format(outf))


USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"

HEADERS = {
    "user-agent": USER_AGENT
}

def lesson_headers(lesson):
    headers = {
        "user-agent": USER_AGENT,
        "referer": lesson,
    }
    return headers

def course_headers(course, csrf):
    headers = {
        "user-agent": USER_AGENT,
        "referer": f"https://stepik.org/edit-course-permissions/{course}",
        "x-csrftoken": csrf,
        "xrequestedwith": "XMLHttpRequest",
        "content-type": "application/json; charset=UTF-8",
    }
    return headers

def login_headers(csrf, email):
    headers = {
        "user-agent": USER_AGENT,
        "x-csrftoken": csrf,
        "x-requested-with": "XMLHttpRequest",
        "referer": f"https://stepik.org/login"
    }
    return headers


import warnings
import pickle as pkl
import sys
import numpy as np
import pandas as pd
import mrtool

def select_covariates(df, candidate_covs, include_covs,
                      resp_name, se_name, study_id_name,
                      beta_gprior=None):
    """
    Selects and covariates from candidate_covs and returns them as a list.

    Args:
        df (pd.DataFrame): the data.
        candidate_covs (list of strings): each of which is the column index
            of a column of df whose inclusion in the model will be determined
            by lasso.
        include_covs (list of strings): columns of a covariates automatically
            included in the model.
        resp_name (string): denotes column with the response.
        se_name (string): denotes column with the SE of the response.
        study_id_name (string): column that defines random effect groups
    Returns:
        list of strings denoting covariates selected by CovFinder.
    """
    if not any([v == 'intercept' for v in include_covs]):
        include_covs = include_covs + ['intercept']
    if not any([v == 'intercept' for v in df.columns.values]):
        df['intercept'] = np.ones(df.shape[0])

    # Standardize the covariates
    norm_df = df.copy()
    for cov in candidate_covs:
        norm_df[cov] = (df[cov] - df[cov].mean())/df[cov].std()

    all_covs = include_covs + candidate_covs

    mrd = mrtool.MRData(obs=norm_df[resp_name].to_numpy(),
                        obs_se=norm_df[se_name].to_numpy(),
                        covs={v: norm_df[v].to_numpy() for v in all_covs},
                        study_id=norm_df[study_id_name].to_numpy())

    # Note that it selects covariates without a spline even if the model will be fit
    # with a spline on an include_cov.
    cfinder = mrtool.CovFinder(data=mrd,
                               covs=candidate_covs,
                               pre_selected_covs=include_covs,
                               beta_gprior=beta_gprior,
                               normalized_covs=True,
                               num_samples=1000,
                               laplace_threshold=1e-5,
                               power_range=(-8, 8),
                               power_step_size=0.5)

    cfinder.select_covs()

    return cfinder.selected_covs

def fit_with_covs(df, covs,
                  resp_name, se_name, study_id_name,
                  data_id_name=None,
                  z_covs=['intercept'],
                  trim_prop=0.0, spline_cov=None, spline_degree=None,
                  spline_knots_type=None, spline_knots=None,
                  spline_monotonicity=None,
                  full_spline_basis=False,
                  gprior_dict=None,
                  uprior_dict=None,
                  inner_max_iter=2000, outer_max_iter=1000):
    """
    Fits a model using the specified covariates.

    Args:
        df (pd.DataFrame): the data.
        covs (list of strings): columns containing covariate values
        resp_name (string): column containing the response
        se_name (string): column containing the SE of the response
        study_id_name (string): column that defines random effect groups
        z_covs (list of strings): list of covariates with random slopes
        trim_prop (float between 0.0 and 1.0): proportion of data points to trim
        spline_cov (string or None): If a string, then the covariate to be
            fit with a spline. If None, then indicates no spline should be fit
        spline_degree (int > 0): degree of the spline. None if spline_cov is None
        spline_knots_type (string, either 'domain' or 'frequency'): whether knots
            should be evenly spaced values or quantiles
        spline_knots (np.ndarray): locations of knots either as quantiles or as
            proportions of the distance from min to the max observed value.
        gprior_dict (None or dictionary of np.arrays or lists of np.arrays):
            dictionary whose keys are the entries of covs and whose values are
            np.arrays specifying priors to be placed on parameters. Priors for
            spline_cov are specified using a numpy.array of shape (2, k) where k
            is the number of parameters estimated for spline_cov. Priors for covariates
            not in z_covs must be specified using a np.array of shape (2,) whose
            entries are the mean and sd of the gaussian prior, respectively.
            Priors for z_covs must be lists of length 2, whose first entry
            specifies the prior for the fixed effect, and whose second specifies
            the prior for the random effect.
        uprior_dict (None or dictionary of np.arrays or lists of np.arrays): dictionary
            whose keys are
        inner_max_iter (int): maximum number of iterations for the inner loop.
        outer_max_iter (int): maximum number of iterations for the outer loop.
    Returns:
        MRBRT object

    """
    z_covs = list(set(z_covs + ['intercept']))
    if not any(df.columns.values == 'intercept'):
        df['intercept'] = 1.0
    elif not all(df['intercept'] == 1.0):
        sys.exit('Columns of df labeled intercept must only have entries of 1.0')

    if full_spline_basis:
        if uprior_dict is None:
            uprior_dict = {}
        uprior_dict.update({'intercept': np.array([0.0, 0.0])})

    if data_id_name is None:
        data_id_name = 'DataID'
        df = df.assign(DataID=np.arange(df.shape[0]))
    mrd = mrtool.MRData(obs=df[resp_name].to_numpy(),
                        obs_se=df[se_name].to_numpy(),
                        covs={v: df[v].to_numpy() for v in covs},
                        study_id=df[study_id_name].to_numpy(),
                        data_id=df[data_id_name].to_numpy()
                       )

    cov_model_args = {v: {'alt_cov': v, 'use_re': (v in z_covs)} for v in covs if v != spline_cov}

    if gprior_dict is not None:
        for key, val in gprior_dict.items():
            if key in z_covs:
                cov_model_args[key].update({'prior_beta_gaussian': val[0],
                                            'prior_gamma_gaussian': val[1]
                    })
            else:
                cov_model_args[key].update({'prior_beta_gaussian': val})

    if uprior_dict is not None:
        for key, val in uprior_dict.items():
            cov_model_args[key].update({'prior_beta_uniform': val})

    cov_model_list = [mrtool.LinearCovModel(**x) for x in cov_model_args.values()]
    if spline_cov is not None:
        n_knots = spline_knots.shape[0] if spline_degree is not None else None
        spline_degree = int(spline_degree)
        prior_spline_maxder = None if gprior_dict is None else gprior_dict.get(spline_cov)
        spline_cov_model = mrtool.LinearCovModel(spline_cov, use_re=False,
                                                 use_spline=True,
                                                 spline_degree=spline_degree,
                                                 spline_knots_type=spline_knots_type,
                                                 spline_knots=spline_knots,
                                                 spline_r_linear=(spline_degree > 1),
                                                 spline_l_linear=(spline_degree > 1),
                                                 use_spline_intercept=full_spline_basis,
                                                 prior_spline_monotonicity=spline_monotonicity,
                                                 prior_spline_maxder_gaussian=prior_spline_maxder)
        cov_model_list.append(spline_cov_model)

    mr = mrtool.MRBRT(data=mrd,
                      cov_models=cov_model_list,
                      inlier_pct=1-trim_prop)
    mr.fit_model(inner_print_level=5,
                 inner_max_iter=inner_max_iter,
                 outer_max_iter=outer_max_iter)
    return mr

def summarize_parameters(mr,
                         spline_cov='log_gdp_pc_17_ppp',
                         beta_samples=None,
                         num_draws=10000):
    cov_names = [v + '_' + str(i) for v in mr.fe_soln.keys() for i in range(mr.fe_soln.get(v).shape[0])]
    for i in range(len(cov_names)):
        if cov_names[i][:-2] != spline_cov:
            cov_names[i] = cov_names[i][:-2]
        betas = np.concatenate(list(mr.fe_soln.values()))
    use_draws_method = beta_samples is not None
    if use_draws_method:
        beta_ses = beta_samples.std(axis=0)
        beta_var = beta_ses**2
    else:
        lme_specs = mrtool.core.other_sampling.extract_simple_lme_specs(mr)
        hessn = mrtool.core.other_sampling.extract_simple_lme_hessian(lme_specs)

        beta_var = np.diag(np.linalg.inv(hessn))
        beta_ses = np.sqrt(beta_var)
    summary_df = pd.DataFrame([(mr.cov_model_names[i], bta, bta_se)
                               for i, x_var_idx in enumerate(mr.x_vars_indices)
                               for (bta, bta_se) in zip(mr.beta_soln[x_var_idx], beta_ses[x_var_idx])],
                              columns=['covariate', 'beta', 'beta_se'])
    summary_df['beta_variance'] = summary_df['beta_se']**2
    summary_df['gamma'] = np.concatenate([
        mr.gamma_soln[mr.z_vars_indices[mr.get_cov_model_index(cov_name)]]
        if mr.cov_models[mr.get_cov_model_index(cov_name)].use_re
        else np.repeat(np.nan, mr.cov_models[mr.get_cov_model_index(cov_name)].num_x_vars)
        for cov_name in mr.cov_model_names
    ])
    return summary_df

def predict(pred_df, mr):
    """
    Predicts response values for the data frame pred_df using the model fit mr.
    """
    pred_mrdata = mrtool.MRData(covs={v: pred_df[v].to_numpy() for v in mr.cov_model_names})
    preds = mr.predict(data=pred_mrdata, predict_for_study=False)
    return preds

# def mr_aic(mr):
    # ## This is flawed the penalty doesn't take into account the random effects variance
    # # The LimeTr objective is the -log-likelihood. We want 2*k_beta - 2*log-likelihood.
    # return 2 * mr.lt.objective(mr.lt.soln) + 2 * mr.lt.k_beta

def mr_r_squared(mr, conditional_on_random_fx=True):
    fits = mr.predict(data=mr.data, predict_for_study=conditional_on_random_fx)
    denominator = np.var(mr.data.obs)
    numerator = np.var(fits)
    return numerator / denominator

def k_fold_cv_gaussian_prior(k, df, resp_name, se_name, covs,
                             data_id_name, study_id_name,
                             constrained_covs=None,
                             beta_gpriors=None,
                             combine_gpriors=False,
                             fold_id_name=None,
                             initial_upper_prior_sd=1.0,
                             inner_max_iter=2000, outer_max_iter=1000,
                             sd_tol=1e-6, num_sds_per_step=10,
                             dev=False):
    df['fold_id'] = np.random.randn(df.shape[0])
    df['fold_id'] = pd.qcut(df['fold_id'], k, labels=list(range(k)))
    return cv_gaussian_prior(df, resp_name, se_name, covs,
                             data_id_name, study_id_name,
                             constrained_covs=constrained_covs,
                             beta_gpriors=beta_gpriors,
                             combine_gpriors=combine_gpriors,
                             initial_upper_prior_sd=initial_upper_prior_sd,
                             fold_id_name='fold_id',
                             inner_max_iter=inner_max_iter, outer_max_iter=outer_max_iter,
                             sd_tol=sd_tol, num_sds_per_step=num_sds_per_step,
                             dev=dev)

def cv_gaussian_prior(df, resp_name, se_name, covs,
                      data_id_name, study_id_name,
                      constrained_covs=None,
                      beta_gpriors=None,
                      combine_gpriors=False,
                      fold_id_name=None,
                      initial_upper_prior_sd=1.0,
                      inner_max_iter=2000, outer_max_iter=1000,
                      sd_tol=1e-6, num_sds_per_step=10,
                      dev=False):

    if not any(df.columns.values == 'intercept'):
        df['intercept'] = 1.0
    elif not all(df['intercept'] == 1.0):
        sys.exit('Columns of df labeled intercept must only have entries of 1.0')

    if fold_id_name is None:
        fold_id_name = data_id_name

    stdized_df = df[[fold_id_name, data_id_name, study_id_name, resp_name, se_name] + covs].copy()

    if constrained_covs is None:
        constrained_covs = {}

    unstdized_covs = ['intercept'] + list(constrained_covs.keys())
    if beta_gpriors is not None:
        if combine_gpriors:
            covs_to_stdize = list(set(covs) - set(unstdized_covs))
            for v in beta_gpriors.keys():
                if v in covs_to_stdize:
                    beta_gpriors[v] = beta_gpriors[v] * stdized_df[v].std()
        else:
            unstdized_covs = unstdized_covs + list(beta_gpriors.keys())
            unstdized_covs = list(set(unstdized_covs))
            covs_to_stdize = list(set(covs) - set(unstdized_covs))
    else:
        covs_to_stdize = list(set(covs) - set(unstdized_covs))
        beta_gpriors = {}

    stdized_df[covs_to_stdize] = (stdized_df[covs_to_stdize] - stdized_df[covs_to_stdize].mean(axis=0))
    stdized_df[covs_to_stdize] = stdized_df[covs_to_stdize] / stdized_df[covs_to_stdize].std(axis=0)
    # Create a list of pandas series for the train-test splitting.
    # The ith entry of the list is a boolean Series with value True wherever a row it is in the test
    # set for the ith unique value of the fold-id column and False wherever it is in the training set
    mask_list = [stdized_df[fold_id_name] == fid
            for fid in stdized_df[fold_id_name].unique()]
    # Use the list of masks to create a list of tuples of MRData objects.
    # The ith tuple has first entry that is the training data for the ith fold
    # and second entry for the test set data for the ith fold.
    train_test_mrd_list = [tuple(mrtool.MRData(obs=stdized_df.loc[m == truth_val, resp_name].to_numpy(),
                                               obs_se=stdized_df.loc[m == truth_val, se_name].to_numpy(),
                                               covs={v: stdized_df.loc[m == truth_val, v].to_numpy() for v in covs},
                                               study_id=stdized_df.loc[m == truth_val, study_id_name].to_numpy(),
                                               data_id=stdized_df.loc[m == truth_val, data_id_name].to_numpy())
                           for truth_val in [False, True]) for m in mask_list]
    if not combine_gpriors:
        gprior_cov_models = [mrtool.LinearCovModel(key, use_re=False, prior_beta_gaussian=val)
                             for key, val in beta_gpriors.items()]

    # Initialize empty numpy arrays for the prior SDs & mses
    prior_sds = np.array([], dtype=np.float64)
    mses = np.array([], dtype=np.float64)
    # Initialize bounds for the set of prior SDs for a single iteration
    lower_prior_sd = 1e-4
    upper_prior_sd = initial_upper_prior_sd
    # Create a copy of the number of prior SDs per step variable.
    n_sds_per_step = num_sds_per_step
    if dev:
        import time
    while upper_prior_sd - lower_prior_sd > sd_tol:
        if dev:
            tm = time.time()

        # Generate new values of the prior SD, evenly spaced on a log-scale
        new_prior_sds = np.geomspace(lower_prior_sd, upper_prior_sd, n_sds_per_step)
        # After the first iteration, increment the number of SDs variable by 2. This avoids
        # repeating the same value of SD in subsequent iterations, since the upper and lower
        # bounds of SD are defined by values of SD in the previous iteration.
        if n_sds_per_step == num_sds_per_step:
            n_sds_per_step += 2
        new_prior_sds = np.setdiff1d(new_prior_sds, prior_sds)

        if combine_gpriors:
            # Update the priors on covariates with gaussian priors specified in beta_gpriors by
            # taking a precision-weighted mean of the specified prior and the regularization prior
            # for the mean, and the standard deviation that results from adding the precisions of
            # the prior specified in beta_gpriors and the regularization prior.
            # Note that the result of this is that large amounts of regularization pull the
            # parameter estimates of all covariates towards 0 regardless of whether or not they
            # have priors specified in beta_gpriors.
            gprior_cov_model_lists = [
                    [mrtool.LinearCovModel(key, use_re=False,
                                           prior_beta_gaussian=np.array(
                                               [val[0] / (1 + (val[1]**2 / sd**2)),
                                               val[1] * sd / np.sqrt(val[1]**2 + sd**2)
                                               ]))
                     for key, val in beta_gpriors.items()]
                     for sd in new_prior_sds]
        else:
            gprior_cov_model_lists = [gprior_cov_models for sd in new_prior_sds]

        const_cov_model_lists = [[mrtool.LinearCovModel(key, use_re=False,
                                                        prior_beta_uniform=np.full((2,), val))
                                  for key, val in constrained_covs.items()]
                                 for sd in new_prior_sds]

        # Create a list of lists of covariate models. The ith entry of the outer list is a list
        # of all the covariate models for the ith value of SD.
        cov_model_lists = [[mrtool.LinearCovModel('intercept', use_re=True)] + \
                           [mrtool.LinearCovModel(v, use_re=False, prior_beta_gaussian=np.array([0, sd]))
                                for v in covs if v not in ['intercept'] +\
                                                            list(constrained_covs.keys()) +\
                                                            list(beta_gpriors.keys())] +\
                            gprior_cov_model_lists[i] + const_cov_model_lists[i]
                            for i, sd in enumerate(new_prior_sds)]
        new_mses = np.array(
                [
                    np.array([get_mse(train_mrd, test_mrd, cmod_list)
                        for train_mrd, test_mrd in train_test_mrd_list
                        ]).mean()
                    for cmod_list in cov_model_lists
                    ]
                )

        prior_sds = np.hstack([prior_sds, new_prior_sds])
        mses = np.hstack([mses, new_mses])
        ordr = np.argsort(prior_sds)
        prior_sds = prior_sds[ordr]
        mses = mses[ordr]

        min_index = np.argmin(mses)
        lower_prior_sd = prior_sds[min_index - 2 if min_index > 1 else 0]
        upper_prior_sd = prior_sds[min_index + 2
                if min_index < prior_sds.shape[0] - 2
                else prior_sds.shape[0] - 1]

        if dev:
            print('iteration took ' + str(time.time() - tm))
            print('just ran for ' + str(new_mses.shape[0]) + ' values of lambda')
    return (prior_sds, mses)

def get_mse(train_mrd, test_mrd,
            cov_model_list,
            inner_max_iter=1000,
            outer_max_iter=2000):

    mr = mrtool.MRBRT(data=train_mrd, cov_models=cov_model_list)
    mr.fit_model(inner_print_level=5,
                inner_max_iter=inner_max_iter,
                outer_max_iter=outer_max_iter)
    preds = mr.predict(test_mrd, predict_for_study=False)
    mse = ((preds - test_mrd.obs)**2).mean()
    return mse

def fit_signal_model(df,
                     resp_name, se_name, spline_cov, study_id_name, data_id_name,
                     other_cov_names, other_cov_gpriors=None,
                     h=0.1, num_samples=20, deg=2, n_i_knots=2, knots_type='domain',
                     knot_bounds = np.array([[0.1, 0.6], [0.4, 0.9]]),
                     interval_sizes = np.array([[0.1, 0.7], [0.1, 0.7], [0.1, 0.7]]),
                     ):

    cov_dict = {spline_cov: df[spline_cov].to_numpy()}
    cov_dict = {w: df[w].to_numpy() for w in other_cov_names + [spline_cov]}

    covs_with_prior = list(other_cov_gpriors.keys())

    cov_model_list = []
    for v in other_cov_names:
        if v in covs_with_prior:
            cov_model_list.append(mrtool.LinearCovModel(v, use_re=False,
                                                        prior_beta_gaussian=other_cov_gpriors[v])
                                 )
        else:
            cov_model_list.append(mrtool.LinearCovModel(v, use_re=False))

    mrd = mrtool.MRData(obs=df[resp_name].to_numpy(),
                        obs_se=df[se_name].to_numpy(),
                        study_id=df[study_id_name].to_numpy(),
                        data_id=df[data_id_name].to_numpy(),
                        covs=cov_dict)

    ensemble_cov_model = mrtool.LinearCovModel(alt_cov=spline_cov,
                                               use_spline=True,
                                               spline_degree=deg,
                                               spline_knots_type=knots_type,
                                               spline_knots=np.linspace(0.0, 1.0, n_i_knots + 2),
                                               spline_r_linear=True,
                                               spline_l_linear=True,
                                               use_spline_intercept=False,
    #                                            prior_spline_monotonicity='increasing',
                                               use_re=False)
    knot_samples = mrtool.core.utils.sample_knots(num_intervals=n_i_knots + 1,
                                                  knot_bounds=knot_bounds,
                                                  interval_sizes=interval_sizes,
                                                  num_samples=num_samples)

    signal_mr = mrtool.MRBeRT(data=mrd,
                       ensemble_cov_model=ensemble_cov_model,
                       ensemble_knots=knot_samples,
                       cov_models=cov_model_list,
                       inlier_pct=1.0 - h)

    signal_mr.fit_model(inner_max_iter=1000, outer_max_iter=2000)

    return signal_mr

def create_signal(signal_mr, spline_cov, spline_cov_values,
                  data_id_name, data_ids):

    pred_covs = {spline_cov: spline_cov_values}
    cov_model_names = [x.name for x in signal_mr.cov_models]
    pred_covs.update({v: np.zeros(spline_cov_values.shape[0])
                      for v in cov_model_names if v != spline_cov})

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        pred_mrd = mrtool.MRData(obs=np.ones(data_ids.shape),
                                 obs_se=np.ones(data_ids.shape),
                                 study_id=np.arange(data_ids.shape[0]),
                                 data_id=data_ids,
                                 covs=pred_covs)

    new_spline_cov = signal_mr.predict(pred_mrd, predict_for_study=False)
    # w = np.hstack([mdl.w_soln[:, None] for mdl in signal_mr.sub_models]).dot(signal_mr.weights)

    # pred_df = pred_mrd.to_df().assign(new_spline_cov=new_spline_cov, data_id=pred_mrd.data_id, w=w)
    pred_df = pred_mrd.to_df().assign(new_spline_cov=new_spline_cov,
                                      data_id=pred_mrd.data_id)
    pred_df = pred_df[['data_id', 'new_spline_cov']]
    pred_df = pred_df.rename({'data_id': data_id_name,
                             }, axis=1)
    return pred_df

def get_ws(signal_mr, data_id_name):
    w = np.hstack([mdl.w_soln[:, None]
                   for mdl in signal_mr.sub_models
                  ]).dot(signal_mr.weights)
    pred_df = signal_mr.data.to_df().assign(**{data_id_name: signal_mr.data.data_id, 'w': w})
    pred_df = pred_df[[data_id_name, 'w']]
    pred_df = pred_df.rename({'data_id': data_id_name,
                             }, axis=1)
    return pred_df

def create_fit_df(mr, df, resp_name, study_id_name, other_id_col_names, data_id_name):

    fit_mrd = mrtool.MRData(obs=np.zeros((df.shape[0],)),
            obs_se=np.zeros((df.shape[0],)),
            covs={v: df[v].to_numpy() for v in mr.data.covs.keys()},
            study_id=df[study_id_name].to_numpy(),
            data_id=df[data_id_name].to_numpy(),
            )
    fit_df = fit_mrd.to_df().rename({'study_id': study_id_name}, axis=1)
    fit_df[data_id_name] = fit_mrd.data_id
    fit_df = fit_df.merge(df[[study_id_name, data_id_name, resp_name] + other_id_col_names],
            on=[study_id_name, data_id_name])

    fit_df['fitted_fe_only'] = mr.predict(data=fit_mrd, predict_for_study=False)
    fit_df['fitted_fe_and_re'] = mr.predict(data=fit_mrd, predict_for_study=True)

    fit_df = fit_df[[study_id_name, data_id_name] +\
            other_id_col_names +\
            [resp_name, 'fitted_fe_only', 'fitted_fe_and_re']]

    return fit_df

def r2(mr, fit_df, resp_name):
    rmses = fit_df[[resp_name, 'fitted_fe_and_re', 'fitted_fe_only']].copy()
    rmses['fitted_fe_and_re'] = rmses['fitted_fe_and_re'] - rmses[resp_name]
    rmses['fitted_fe_only'] = rmses['fitted_fe_only'] - rmses[resp_name]

    rmses = np.sqrt(rmses[['fitted_fe_and_re', 'fitted_fe_only']].var(axis=0))
    rmses.name = 'RMSE'

    r2s = (fit_df[[resp_name, 'fitted_fe_and_re', 'fitted_fe_only']].corr()**2)
    r2s = r2s.loc[resp_name, ['fitted_fe_and_re', 'fitted_fe_only']]
    r2s.name = 'R_squared'
    r2s = pd.DataFrame(r2s, columns=['R_squared'])
    r2s = r2s.join(rmses)
    r2s.loc['fitted_fe_only', 'Sample_size'] = fit_df.shape[0]

    return r2s

def create_predictions(mr, signal_mr, preds_df,
                       resp_name, se_name, selected_covs,
                       study_id_name, data_id_name,
                       beta_samples=None, n_samples=1000, seed=24601):

    spline_cov = signal_mr.ensemble_cov_model_name

    if 'new_spline_cov' not in preds_df.columns:
        preds_df['idx'] = np.arange(preds_df.shape[0])
        signal_preds_df = create_signal(signal_mr, spline_cov,
                                        preds_df[spline_cov].to_numpy(),
                                        'idx', preds_df['idx'].to_numpy())
        preds_df = preds_df.merge(signal_preds_df, on='idx')

    if any([v not in preds_df.columns for v in selected_covs]):
        print([v for v in selected_covs if v not in preds_df.columns])
        print(spline_cov)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        preds_mrd = mrtool.MRData(obs=np.zeros((preds_df.shape[0],)),
                                  obs_se=np.zeros((preds_df.shape[0],)),
                                  data_id=np.arange(preds_df.shape[0]),
                                  study_id=np.ones(preds_df.shape[0]),
                                  covs={v: preds_df[v].to_numpy()
                                        for v in selected_covs}
                                  )

    # Predictions on the log scale
    preds_df['predicted_' + resp_name] = mr.predict(preds_mrd,
                                                    predict_for_study=False,
                                                    sort_by_data_id=True)

    np.random.seed(seed)
    if beta_samples is None:
        beta_samples = mrtool.core.other_sampling.sample_simple_lme_beta(n_samples, mr)
    # Generate 1000 draws for UIs.
    preds_draws = mr.create_draws(data=preds_mrd, beta_samples=beta_samples,
                                  gamma_samples=np.full((beta_samples.shape[0], 1),
                                                        mr.gamma_soln))
    preds_df['log_icer_stdev'] = preds_draws.std(axis=1)
    # Calculate quantiles of the draws.
    ci_preds = np.quantile(preds_draws, [0.5, 0.025, 0.975, 0.05, 0.95], axis=1).T

    ci_suffixes = ['_median', '_lower',  '_upper', '_90_lower', '_90_upper']
    preds_df[['predicted_' + resp_name + v for v in ci_suffixes]] = ci_preds

    # Calculate the mean on the ICER scale by taking the average of the converted predictions.
    preds_df['predicted_' + resp_name.replace('log_', '')] = np.exp(preds_draws).mean(axis=1)
    # Convert quantiles from log scale
    log_columns = ['predicted_' + resp_name + v for v in ci_suffixes]
    lin_columns = [v.replace('log_', '') for v in log_columns]
    preds_df[lin_columns] = np.exp(preds_df[log_columns])

    preds_df = preds_df.drop('idx', axis=1)

    return preds_df


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 13 22:00:29 2020

@author: Onkar
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Mar 17 20:04:25 2020

@author: Onkar
"""

import string

# appostrophe short-wards taken from :
# https://drive.google.com/file/d/0B1yuv8YaUVlZZ1RzMFJmc1ZsQmM/view
appos = {
"aren't" : "are not","can't" : "can not","couldn't" : "could not",
"didn't" : "did not","doesn't" : "does not","don't" : "do not",
"hadn't" : "had not","hasn't" : "has not","haven't" : "have not",
"he'd" : "he would","he'll" : "he will","he's" : "he is","i'd" : "I would",
"i'll" : "I will","i'm" : "I am","isn't" : "is not","it's" : "it is",
"it'll":"it will","i've" : "I have","let's" : "let us","mightn't" : "might not",
"mustn't" : "must not","shan't" : "shall not","she'd" : "she would",
"she'll" : "she will","she's" : "she is","shouldn't" : "should not",
"that's" : "that is","there's" : "there is","they'd" : "they would",
"they'll" : "they will","they're" : "they are","they've" : "they have",
"we'd" : "we would","we're" : "we are","weren't" : "were not",
"we've" : "we have","what'll" : "what will","what're" : "what are",
"what's" : "what is","what've" : "what have","where's" : "where is",
"who'd" : "who would","who'll" : "who will","who're" : "who are",
"who's" : "who is","who've" : "who have","won't" : "will not",
"wouldn't" : "would not","you'd" : "you would","you'll" : "you will",
"you're" : "you are","you've" : "you have","wasn't": "was not",
"we'll":"we will","didn't": "did not"
}

# stopwards taken from https://www.ranks.nl/stopwords
stopwards = [
"", "a","about","above","after","again","against","all","am","an","and",
"any","are","aren't","as","at","be","because","been","before","being","below",
"between","both","but","by","can't","cannot","could","couldn't","did","didn't",
"do","does","doesn't","doing","don't","down","during","each","few","for","from",
"further","had","hadn't","has","hasn't","have","haven't","having","he","he'd",
"he'll","he's","her","here","here's","hers","herself","himself","his","how",
"how's","i","i'd","i'll","i'm","i've","if","in","into","is","isn't","it","it's",
"its","itself","let's","me","more","most","mustn't","my","myself","no","nor",
"not","of","off","on","once","only","or","other","ought","our","ours",
"ourselves","out","over","own","same","shan't","she","she'd","she'll","she's",
"should","shouldn't","so","some","such","than","that","that's","the","their",
"theirs","them","themselves","then","there","there's","these","they","they'd",
"they'll","they're","they've","this","those","through","to","too","under",
"until","up","very","was","wasn't","we","we'd","we'll","we're","we've","were",
"weren't","what","what's","when","when's","where","where's","which","while",
"who","who's","whom","why","why's","with","won't","would","wouldn't","you",
"you'd","you'll","you're","you've","your","yours","yourself","yourselves",
"b", 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'
]


import pickle
from datetime import date
from time import sleep

import openai
import tqdm
from data_model.document import Document
from src.utils.date_utils import get_date_today_str
from src.utils.openai_utils import get_openai_key

openai.api_key = get_openai_key()
OPENAI_MODEL_EMBEDDING = "text-embedding-ada-002"

def add_embedding_openai(doc: Document, nb_openai_used_tokens: int):
    doc_ai = doc.model_copy(deep=True)
    doc_ai.embedding_model = OPENAI_MODEL_EMBEDDING
    for idx_sub_doc in tqdm(range(0, len(doc_ai.sub_documents), 20)):
        results = get_embedding(
            [
                doc_.text
                for doc_ in doc_ai.sub_documents[idx_sub_doc : idx_sub_doc + 20]
            ],
            model=OPENAI_MODEL_EMBEDDING,
        )
        for idx, emb in enumerate(results["data"]):
            doc_ai.sub_documents[idx_sub_doc + idx].embedding = emb["embedding"]
        nb_openai_used_tokens += get_nb_tokens(results)
        sleep(20)
    save_embeddings(doc_ai)
    return doc_ai, nb_openai_used_tokens

def get_embedding(list_text, model=OPENAI_MODEL_EMBEDDING):
    list_text = [text.replace("\n", " ") for text in list_text]
    return openai.Embedding.create(input=list_text, model=model)

def get_embedding_values(embedding: dict):
    return embedding["data"]

def get_nb_tokens(embedding: dict):
    return embedding["usage"]["total_tokens"]

def save_embeddings(embeddings):
    str_date = get_date_today_str()
    with open(
        f"../data/processed/province-sud/BaieDesCitrons_barriere_01062023_{str_date}.pickle",
        "wb",
    ) as handle:
        pickle.dump(embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL)


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 14 12:04:37 2016

@author: hasan
"""
import numpy as np
import math as m
import random

def PLA(X, Y, w, my_g):

    iteration = 0
    n = 1 #learning rate
    misclassified = np.copy(X)
    misclassified_Y = np.copy(Y)

    def get_misclassified(w):
        points = []
        labels = []
        i = 0
        for x in X:
            g = my_g(w, x)
            if g!=Y[i]:
                points.append(x)
                labels.append(Y[i])
            i += 1
        return [points, labels]

    #PLA
    while len(misclassified)>0 and iteration<1000:
        random_index = random.randint(0, len(misclassified)-1)
        random_x = misclassified[random_index]
        random_y = misclassified_Y[random_index]
        g = my_g(w, random_x)
        if g != random_y:
            w[0] = w[0] + (n*(random_y)*random_x[0])
            w[1] = w[1] + (n*(random_y)*random_x[1])
            w[2] = w[2] + (n*(random_y)*random_x[2])
            [misclassified , misclassified_Y] = get_misclassified(w)
            iteration += 1
        else:
            misclassified = np.delete(misclassified, random_index, 0)
            misclassified_Y = np.delete(misclassified_Y, random_index, 0)

    return iteration

# Linear Regression
def LinearRegression(X, Y):

    X_m = np.matrix(X)
    Y_m = np.matrix(Y).transpose()
    Xdagger = (X_m.transpose()*X_m).getI()*X_m.transpose()
    w = Xdagger*Y_m
    return np.array(w)

# Linear Regression
def LinearRegressionRegularized(X, Y, lambdafactor):

    X_m = np.matrix(X)
    Y_m = np.matrix(Y).transpose()
    Xdagger = (X_m.transpose()*X_m + lambdafactor*np.identity(X_m.shape[1])).getI()*X_m.transpose()
    w = Xdagger*Y_m
    return np.array(w)

def calcGradient(x, y, w):
    x_m = np.matrix(x).transpose()
    w_m = np.matrix(w)
    return np.multiply(-1*y, x_m)/(1. + m.exp(y*(w_m*x_m)))

def LogisticRegression(X, Y, w):
    N = 100
    ETA = 0.01
    WTOL = 0.01
    done = False
    epoch = 0
    while not done:
        w_prev = w
        shuffled_indices = np.random.permutation(N);
        for i in range(N):
            index = shuffled_indices[i];
            g = calcGradient(X[index], Y[index], w)
            w = w - ETA * g.transpose()
        epoch += 1
        done = np.linalg.norm(w - w_prev) < WTOL;
#    return epoch
    return [np.array(w.ravel().tolist()[0]), epoch]

def cross_entropy_error(w, X, Y):
    X = np.matrix(X)
    w_m = np.matrix(w)
    i = 0
    Ein = 0.
    for x_m in X:
        Ein += m.log(1 + m.exp(-1*Y[i]*(w_m*x_m.transpose())))
        i += 1
    return Ein/i


import sys
#sys.stdin = open("input.txt")

n = int(input())
arr = list(map(int,input().split()))
max_money = int(input())

dy = [110]*(max_money+1)
#print(dy)
dy[0]=0 #0으로 해야 한다. 그래야 된다. 초기화 이게 중요하다. 오답 나오는 이유
#다이나믹의 초기화 중요

#이건필요없다.
# dy[1]=1
# dy[2]=1
# dy[5]=1
#print(arr)
# 다이 나믹 프로그래밍으로 풀었다. bottom up 방식
for i in range(1, n+1):
    for j in range(arr[i-1], max_money+1):
        #print(j) # i - 1 하는것에서 시간 엄청 끌었다. 정수 조심해라.
        #print("end")
       dy[j] = min(dy[j - arr[i-1]] + 1 , dy[j])
# print(dy)
print(dy[max_money])
# print(n,arr,max_money)

동전교환(동적계획법)_그리디가능.py


import json

from qmaton import Automaton, AutomatonSerializer, State

class DumbAutomaton(Automaton):
    STATE = State("state", "#000")

    def __init__(self, width, length):
        super().__init__(width, length, DumbAutomaton.STATE)
        self.states = [DumbAutomaton.STATE]
        self.rule = self.main_rule

    def main_rule(self, x, y):
        return DumbAutomaton.STATE

def test_toJSON():
    dab = DumbAutomaton(3, 4)
    out = json.loads(json.dumps(dab, cls=AutomatonSerializer))
    assert "states" in out
    assert out["states"] == [["state", "#000"]]
    assert "grid_size" in out
    assert out["grid_size"] == [3, 4]
    assert "grid" in out
    assert len(out["grid"]) == dab.length
    assert len(out["grid"][0]) == dab.width

def test_fromJSON():
    dab = DumbAutomaton(3, 4)
    out = json.loads(json.dumps(dab, cls=AutomatonSerializer), object_hook=AutomatonSerializer.decode)
    assert out["grid_size"] == dab.grid_size
    assert out["grid"] == dab.grid


from __future__ import absolute_import, division, unicode_literals
from future.builtins import (  # noqa
    ascii, bytes, chr, dict, filter, hex, input, int, list, map, next, object,
    oct, open, pow, range, round, str, super, zip
)

import django_filters
from django.db import models

from app.filters import UUIDFilter, empty_choice, icontains
from app.model_fields import UUIDField

from ...models import Post

class PostFilterSet(
    django_filters.FilterSet
):
    filter_overrides = {
        UUIDField: icontains,
        models.CharField: icontains,
        models.ForeignKey: {
            'filter_class': UUIDFilter,
            'extra': lambda f: {
                'lookup_type': 'exact',
            }
        }
    }

    state = django_filters.ChoiceFilter(
        choices=empty_choice + tuple(Post.state_choices)
    )

    approval = django_filters.ChoiceFilter(
        choices=empty_choice + tuple(Post.approval_choices)
    )

    class Meta:
        model = Post
        fields = ['id', 'user', 'group', 'name', 'state', 'approval']


'''

 ██████╗ ██╗      ██████╗ ██████╗  █████╗ ██╗
██╔════╝ ██║     ██╔═══██╗██╔══██╗██╔══██╗██║
██║  ███╗██║     ██║   ██║██████╔╝███████║██║
██║   ██║██║     ██║   ██║██╔══██╗██╔══██║██║
╚██████╔╝███████╗╚██████╔╝██████╔╝██║  ██║███████╗
 ╚═════╝ ╚══════╝ ╚═════╝ ╚═════╝ ╚═╝  ╚═╝╚══════╝

███████╗██╗   ██╗██████╗ ████████╗██╗████████╗██╗     ███████╗███████╗
██╔════╝██║   ██║██╔══██╗╚══██╔══╝██║╚══██╔══╝██║     ██╔════╝██╔════╝
███████╗██║   ██║██████╔╝   ██║   ██║   ██║   ██║     █████╗  ███████╗
╚════██║██║   ██║██╔══██╗   ██║   ██║   ██║   ██║     ██╔══╝  ╚════██║
███████║╚██████╔╝██████╔╝   ██║   ██║   ██║   ███████╗███████╗███████║
╚══════╝ ╚═════╝ ╚═════╝    ╚═╝   ╚═╝   ╚═╝   ╚══════╝╚══════╝╚══════╝

!!! Make Sure You Have Specified API Keys In app/configure.py Before Running !!!
!!! Make Sure You Have Installed All Dependencies In requirements.txt !!!
Read the LICENSE file for License
Read README.md for information about the project
Run app/app.py with Python 3.10.1 to Start Applet

'''

#* Imports Required Dependencies
# Make Sure You Have Installed These from requirements.txt

import pyaudio
import websockets
import asyncio
import base64
import json
from configure import assemblyai_key
import settings
from datetime import datetime
import sys
import os

# Gets System Time For Transcription File
timedate = datetime.now()
# Gets Home Directory Path
homeDirectoryPath = os.path.expanduser('~')

# Configures Stream
FRAMES_PER_BUFFER = 3200
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 16000
p = pyaudio.PyAudio()
stream = p.open(
    format=FORMAT,
    channels=CHANNELS,
    rate=RATE,
    input=True,
    frames_per_buffer=FRAMES_PER_BUFFER
)

# API URL For AssemblyAI API
URL = "wss://api.assemblyai.com/v2/realtime/ws?sample_rate=16000"

# Used Internally To Communicate With AssemblyAI
async def send_receive():
    print('     INFO: Connecting websocket to url: ' + URL)
    async with websockets.connect(
        URL,
        extra_headers=(("Authorization", assemblyai_key),),
        ping_interval=5,
        ping_timeout=20
    ) as _ws:
        await asyncio.sleep(0.1)
        print("     INFO: Receiving SessionBegins ...")
        session_begins = await _ws.recv()
        print("     INFO: Sending messages ...")

        async def send():
            while True:
                try:
                    # Sends Audio Data To API
                    data = stream.read(FRAMES_PER_BUFFER)
                    data = base64.b64encode(data).decode("utf-8")
                    json_data = json.dumps({"audio_data": str(data), })
                    print("     INFO: Sent Audio Data To AssemblyAI")
                    await _ws.send(json_data)
                except websockets.exceptions.ConnectionClosedError as e:
                    print(e)
                    assert e.code == 4008
                    break
                except Exception as e:
                    assert False, "     ERROR: Not a websocket 4008 error"
                # Checks If Application Is Still Running
                if settings.running == True: # If True
                    await asyncio.sleep(0.01) # Schedules Repeat Of Send Audio Data Loop
                else: # If Not
                    sys.exit() # Terminates Application

        async def receive():
            global homeDirectoryPath
            global timedate
            while True:
                try:
                    # Recieve Audio Data
                    result_str = await _ws.recv()
                    print("     INFO: Recieved Text From API- "+ json.loads(result_str)['text'])
                    mySubtitle = (json.loads(result_str)['text'])
                    # If Transcription Is Enabled
                    if json.loads(result_str)['message_type'] == 'FinalTranscript':
                        if settings.transcriptionEnabled == "Enabled":
                            # Adds To Transcription File
                            fileName = homeDirectoryPath + \
                                "/globalsubtitles_transcription_" + \
                                str(timedate)
                            file = open(fileName, "a")
                            file.write(mySubtitle + "\n")
                            file.close()
                    # Displays Subtitle Based On User Configuration
                    splitSubtitle = mySubtitle.split()
                    wordCount = len(splitSubtitle)
                    if wordCount > settings.wordcount:
                        removable = wordCount-settings.wordcount
                        del splitSubtitle[:removable]
                        joinedString = ' '.join(
                            [str(item) for item in splitSubtitle])
                        settings.subtitleVar = joinedString
                    else:
                        settings.subtitleVar = mySubtitle
                except websockets.exceptions.ConnectionClosedError as e:
                    print(e)
                    assert e.code == 4008
                    break
                except Exception as e:
                    assert False, "Not a websocket 4008 error"

        send_result, receive_result = await asyncio.gather(send(), receive())


# (경사하강법)
import tensorflow as tf

# data[[공부한시간, 실제성적]]
# x_data = 공부한시간
# y_data = 실제성적
data = [[2, 81], [4, 93], [6, 91], [8, 97]]
x_data = [x_row[0] for x_row in data]
y_data = [y_row[1] for y_row in data]
print("x_data : {}".format(x_data))
print("y_data : {}".format(y_data))

# 학습률 값
learning_rate = 0.1

# 기울기 a는 0~10사이 임의값으로
# y절편 b는 0~100사이 임의값으로
# 텐서플로우의 random_uniform()함수로 임의의수를 생성해줌
# random_uniform(뽑아낼 값의수, 최소값, 최대값, 데이터형식(float64), 실행시 같은값이 나올수있게 seed값 설정)
a = tf.Variable(tf.random_uniform([1], 0, 10, dtype=tf.float64, seed=0))
b = tf.Variable(tf.random_uniform([1], 0, 100, dtype=tf.float64, seed=0))

# 일차방정식. y(예측한성적) = a(기울기) * x(공부한시간) + b(y절편)
y = a * x_data + b

# 평균 제곱근 오차.
# 수식1: y(예측성적) - y_data(실제성적)
# 수식2: tf.square(제곱).  y(예측성적) - y_data(실제성적) ^2
# 수식3: tf.reduce_mean(평균값계산) tf.reduce_mean((y(예측성적) - y_data(실제성적) ^2))
# 수식4: tf.sqrt(제곱근) tf.sqrt(tf.reduce_mean((y(예측성적) - y_data(실제성적) ^2)))
rmse = tf.sqrt(tf.reduce_mean(tf.square(y - y_data)))

# GradientDescentOptimizer()함수는 텐서플로우 경사하강법 수식
gradient_decent = tf.train.GradientDescentOptimizer(learning_rate).minimize(rmse)

# 텐서플로우 session함수로 필요한리소스를 할당시키고 실행시킬 준비를 함
# session을 통해 구현될 함수를 '그래프'라고 부름
# session.run('그래프명')으로 함수실행
# tf.global_variables_initializer는 변수초기화 함수. 변수를 초기화시키고
# gradient_decent을 2001번 실행시키고 100번마다 결과를 출력시킴
with tf.Session() as sess:
    # 변수 초기화
    sess.run(tf.global_variables_initializer())
    # 2001번 실행(0 ~ 2000. 총2001번)
    for step in range(2001):
        sess.run(gradient_decent)
        # 100번마다 결과출력
        if step % 100 == 0:
            print(
                "Epoch: %.f, RMSE = %.f, 기울기a = %.4f, y절편 b = %.4f" % (step, sess.run(rmse), sess.run(a), sess.run(b)))


import os
import csv
from tqdm import tqdm

from predict import predict_single
from corp_img import crop

def label_trans(index: int):
    CLASSES = (
        'F5', 'P7', 'F2', 'W1', 'S4', 'T1', 'C14', 'B3', 'A7', 'A8', 'C2', 'P3', 'F8', 'C8', 'W2', 'S7', 'C13', 'T7',
        'L3',
        'Y1', 'M2', 'S5', 'V1', 'T2', 'S6', 'C10', 'S1', 'R2', 'D2', 'V2', 'C9', 'P2', 'H1', 'U2', 'H3', 'N1', 'T5',
        'A9',
        'D1', 'C6', 'C5', 'T8', 'P5', 'K2', 'P4', 'H2', 'A3', 'B1', 'E2', 'K3', 'C12', 'C15', 'L4', 'S2', 'R1', 'W3',
        'T9',
        'C11', 'M5', 'E4', 'R3', 'F7', 'U1', 'C3', 'K1', 'M1', 'A6', 'F3', 'E3', 'C1', 'B2', 'T6', 'P1', 'K5', 'K4',
        'A4',
        'L2', 'C16', 'S3', 'C4', 'A5', 'I1', 'A1', 'E1', 'P6', 'F6', 'C7', 'M4', 'F1', 'T10', 'T3', 'L1', 'Z1', 'A2',
        'T4',
        'M3', 'R4', 'T11')
    return CLASSES[index]

def main():
    test_data_root = '/workspace/pycharm_project/mmrotate/data/2023/test/images/'
    csv_root = '/workspace/pycharm_project/mmrotate/2023df/'
    csv_file = os.path.join(csv_root, 'example_swin_ft_0.46.csv')

    with open('out3.csv', 'w') as csv_f:
        writer = csv.writer(csv_f)
        writer.writerow(['ImageID', 'LabelName', 'Conf', 'X1', 'Y1', 'X2', 'Y2', 'X3', 'Y3', 'X4', 'Y4'])

    with open(csv_file, encoding='utf-8-sig') as f:
        for row in tqdm(csv.reader(f, skipinitialspace=True)):
            final_data = []
            # print(row)
            tif_name = row[0]
            cls = row[1]
            # conf = row[2]
            tif_path = os.path.join(test_data_root, tif_name)  # 图像路径
            pos_list = []
            for i in range(3, 11):
                pos_list.append(float(row[i]))
            outimg, _ = crop(tif_path, pos_list, cls)  # 剪裁图像
            new_cls, new_conf = predict_single(outimg)  # 图像分类预测,输出：类别，置信度
            # new_cls = label_trans(new_cls_index)
            final_data.append(tif_name)
            final_data.append(new_cls)
            final_data.append(new_conf)
            final_data = final_data + pos_list
            # print(final_data)
            with open('out3.csv', 'a', newline='') as csv_f:
                # with open('example.csv', 'w') as csv_file:
                writer = csv.writer(csv_f)
                writer.writerow(final_data)
    print("finish")

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
from const.ConstVariable import TCP_PROTOCOL_NUMBER, UDP_PROTOCOL_NUMBER, SINGLE_TIME_CAPTURE_PACKETS

# 一次进行数据包的抓取所获取的数据结构
class Packets:
    # here are some statistics result 这些一定要是静态字段从而可以便于进行区分
    # --------------------------------------------------------------------------------------
    packet_count = 0
    udp_packet_count = 0
    tcp_packet_count = 0
    urg_count = 0
    ack_count = 0
    psh_count = 0
    rst_count = 0
    syn_count = 0
    fin_count = 0
    # --------------------------------------------------------------------------------------
    # 用来存储conv matrix 的总和
    sum_conv_matrix = np.zeros((6, 6), dtype=np.float32)
    # 用来存储抓取了多少轮
    capture_round = 0
    tcp_capture_round = 0

    def __init__(self):
        # network layer field
        # --------------------------------------------------------------------------------------
        self.transport_protocol = []
        self.source_ip = []  # source ip
        self.destination_ip = []  # destination ip
        self.header_length = []  # header length
        self.total_length = []  # total length
        self.ttl = []  # ttl
        # --------------------------------------------------------------------------------------

        # transport layer field
        # --------------------------------------------------------------------------------------
        self.source_port = []
        self.destination_port = []
        # --------------------------------------------------------------------------------------

        # six transport layer option field
        # --------------------------------------------------------------------------------------
        self.urgent_bit = []
        self.acknowledgement_bit = []
        self.push_bit = []
        self.reset_bit = []
        self.synchronize_bit = []
        self.finish_bit = []
        # --------------------------------------------------------------------------------------

    @classmethod
    def get_statistics(cls):
        """
        get the statistics result
        :return: a dict
        """
        return {
            "packet_count": cls.packet_count,
            "udp_packet_count": cls.udp_packet_count,
            "tcp_packet_count": cls.tcp_packet_count,
            "urg_count": cls.urg_count,
            "ack_count": cls.ack_count,
            "psh_count": cls.psh_count,
            "rst_count": cls.rst_count,
            "syn_count": cls.syn_count,
            "fin_count": cls.fin_count
        }

    def clear(self):
        """
        clear all the packets 清空所有的数据包
        :return: None
        """
        self.transport_protocol.clear()
        self.source_ip.clear()  # source ip
        self.destination_ip.clear()  # destination ip
        self.header_length.clear()  # header length
        self.total_length.clear()  # total length
        self.ttl.clear()  # ttl

        self.source_port.clear()
        self.destination_port.clear()

        self.urgent_bit.clear()
        self.acknowledgement_bit.clear()
        self.push_bit.clear()
        self.reset_bit.clear()
        self.synchronize_bit.clear()
        self.finish_bit.clear()

        Packets.packet_count = 0
        Packets.udp_packet_count = 0
        Packets.tcp_packet_count = 0
        Packets.urg_count = 0
        Packets.ack_count = 0
        Packets.psh_count = 0
        Packets.rst_count = 0
        Packets.syn_count = 0
        Packets.fin_count = 0


"""
" author : kaneshige
" created : 19.12.2021 20:49:37
"""

import sys
import itertools
from collections import deque

def main():
    s = readline().rstrip()
    print(int(s[0])*int(s[2]))

if __name__ == "__main__":
    "----------Constants------------"
    INF = (1 << 62) - 1
    MOD = 10 ** 9 + 7
    # MOD = 998244353
    # sys.setrecursionlimit(10**6)
    "----------Input------------"
    readline = sys.stdin.readline
    "----------Run------------"
    main()


# Creez un programme qui affiche la racine carrée d'un entier positif.

import sys
import math
arguments = sys.argv

try:
    int_arguments = int(arguments[1])

except:
    print("Vous devez rentrer un nombre")
    exit()

if len(arguments) != 2:
    print("Erreur")
    exit()

else:
    print(math.sqrt(int(arguments[1])))


from __future__ import barry_as_FLUFL
import sys
import numpy as np
import matplotlib as mpl
from env import GreenhouseEnv, STEP, TIME_MAX
from ddpg.ddpg import DDPGagent
from ddpg.utils import *
from tqdm import tqdm
from correo import send_correo
import time
import os
# from torch.utils.tensorboard import SummaryWriter
# from typing_extensions import final
from get_report import create_report
from params import PARAMS_TRAIN,all_params, save_params
from get_report_constants import Constants
from graphics import save_Q, figure_reward, figure_state
from graphics import figure_rh_par, figure_prod, figure_actions
from graphics import figure_inputs, compute_indexes, create_path
from graphics import save_rewards,figure_cost_gain
from benchmark import season1_nn,violin_reward,violin_actions
from sim import sim

EPISODES = PARAMS_TRAIN['EPISODES']
STEPS = PARAMS_TRAIN['STEPS']
BATCH_SIZE = PARAMS_TRAIN['BATCH_SIZE']
SHOW = PARAMS_TRAIN['SHOW']
SERVER = PARAMS_TRAIN['SERVER']
if SERVER: SHOW = False #No puedes mostrar nada en el servidor
INDICE = PARAMS_TRAIN['INDICE'] #Indice con el que se va a simular
SAVE_FREQ = PARAMS_TRAIN['SAVE_FREQ']

from functools import partialmethod
tqdm.__init__ = partialmethod(tqdm.__init__, disable=SERVER)

def train_agent(agent, noise, path, episodes=EPISODES, save_freq=EPISODES):
    rewards = []
    avg_rewards = []
    penalties = []
    abs_rewards = []
    for episode in range(1, episodes + 1):
        with tqdm(total=STEPS, position=0) as pbar:
            pbar.set_description(f'Ep {episode + 1}/'+str(EPISODES))
            env = GreenhouseEnv()
            state = np.array(list(env.state.values()))
            noise.reset()
            episode_reward = 0
            abs_reward = 0
            episode_penalty = 0
            for step in range(STEPS):
                action                  = agent.get_action(state)
                action                  = noise.get_action(action)
                new_state, reward, done = env.step(action) # modify
                agent.memory.push(state, action, reward, new_state, done)
                if len(agent.memory) > BATCH_SIZE:
                    agent.update(BATCH_SIZE)
                episode_reward          += float(reward)
                Ganancia                = float(env.G(env.state['h'])) if step % 24 == 0 else 0
                abs_reward              += Ganancia
                Costo                   = -float(reward) if step % 24 == 0 else -(reward - Ganancia)
                episode_penalty         += float(Costo)
                pbar.set_postfix(reward='{:.2f}'.format(episode_reward/STEPS), NF='{:2f}'.format(float(env.dirGreenhouse.V('NF'))), H='{:2f}'.format(float(env.dirGreenhouse.V('H'))))
                pbar.update(1)
                state = new_state
                if done:
                    #sys.stdout.write("episode: {}, reward: {}, average _reward: {} \n".format(episode, np.round(episode_reward, decimals=2), np.mean(rewards[-10:])))
                    break

        if episode % save_freq == 0:
            agent.save(path+'/nets', name="_{}".format(episode))

        rewards.append(episode_reward)
        abs_rewards.append(abs_reward)
        penalties.append(episode_penalty)
        avg_rewards.append(np.mean(rewards[-10:]))
        #writer_reward.add_scalar("Reward", episode_reward, episode)
        #writer_abs.add_scalar("Absolute reward", abs_reward, episode)
        #writer_penalty.add_scalar("Penalty", episode_penalty, episode)
    agent.save(path+'/nets')
    return rewards, avg_rewards, penalties, abs_rewards

def main():
    env = GreenhouseEnv()
    agent = DDPGagent(env)
    noise = OUNoise(env.action_space)
    action_dim =  env.action_space.shape[0]
    state_dim = env.observation_space.shape[0]
    #writer_reward = SummaryWriter()
    #writer_abs = SummaryWriter()
    #writer_penalty = SummaryWriter()
    t1 = time.time()
    mpl.style.use('seaborn')
    if len(sys.argv) != 1:
    # Load trained model
        PATH = sys.argv[1:].pop()
        print('Se cargo el modelo')
        agent.load(PATH + '/nets')
    else:
        PATH = create_path()

    Constants(PATH)
    all_params['PARAMS_DDPG']['hidden_sizes'] = agent.hidden_sizes

    save_params(all_params,PATH+'/output')
    #agent.actor.eval()
    #agent.critic.eval()

    rewards, avg_rewards, penalties, abs_rewards = train_agent(agent, noise, PATH, save_freq=SAVE_FREQ)

    figure_reward(rewards, avg_rewards, penalties, abs_rewards,PATH)
    save_rewards(rewards, avg_rewards, penalties, abs_rewards,PATH)
    S_climate, S_data, S_prod, A, df_inputs,start,Qdic = sim(agent, ind=INDICE)
    save_Q(Qdic,PATH)
    start = df_inputs['Date'].iloc[0]
    final_indexes = compute_indexes(start,STEPS,env.frec) #Es necesario crear nuevos indices para las graficas, depende de STEP
    figure_cost_gain(Qdic,final_indexes,PATH)
    figure_state(S_climate,final_indexes,PATH)
    figure_rh_par(S_data,final_indexes,PATH)
    figure_prod(S_prod,final_indexes,PATH)
    figure_actions(A,final_indexes,PATH)
    figure_inputs(df_inputs,PATH)

    t2 = time.time()
    season1_nn(agent,PATH,'')
    violin_reward(PATH,'nn') ##puede ser nn ó expert
    violin_actions(PATH,'nn')

    if not(SHOW):
        create_report(PATH,t2-t1)
        send_correo(PATH + '/reports/Reporte.pdf')
        pass

if __name__=='__main__':
    main()


#defの理解のため
import numpy as np
test = np.empty(0)
test = np.arange(1,4)
print(test)
def collatz(a):
    while a != 1:
        if a % 2 == 0:
            a = a /2
        else:
            a = a*3 +1
        print([].append(a))
        return a
print(collatz(3))

def f(n):
    L = []
    ans = []
    if n == 0 or n == 1:
        ans = 1
        L.append(ans)
        return L
    else:
        ans = f(n-1)+f(n-2)
        L.append(ans)
        return L
def main():
    for i in range(11):
        print(f(i))
main()

def f(n):
    if n == 0:
        return [1]
    elif n == 1:
        return [1, 1]
    else:
        fn1 = f(n-1)
        return fn1 + [fn1[-1] + fn1[-2]]

def main():
    for i in range(11):
        print("f({0}) = {1}".format(i, f(i)))

main()