import gym
import numpy as np
import matplotlib.pyplot as plt

NUM_RUNS = 50

MAX_EPISODE_STEPS = 5000
NUM_EPISODES = 100
NUM_ACTIONS = 3

env = gym.make('MountainCar-v0').env
env._max_episode_steps = MAX_EPISODE_STEPS

POLYNOMIAL_FEATURES = True
POLYNOMIAL_DEGREE = 2

'''
https://github.com/openai/gym/wiki/MountainCar-v0
Summary for me:
    There are 3 different actions (push left, nothing, push right) which
    correspond to (0, 1, 2).
    An state is a pair (position,velocity).
    Position ranges from -1.2 to 0.6 and velocity from -0.07 to 0.07
'''

def convert_to_features(state, action, polynomial = POLYNOMIAL_FEATURES,
                        max_degree = POLYNOMIAL_DEGREE):

    if polynomial:
        '''
        Converts the state to a polynomial of degree max_degree.
        However, we cap the degree of action at 1, since we cannot expect
        higher powers of action to deliver meaningful extra features.
        (Action takes values in (0, 1, 2) )
        '''

        subfeatures = []
        for n in range(max_degree + 1):
            for k in range(n+1):
                subfeatures += [state[0]**(n-k) * state[1]**(k)]

        subfeatures = np.array(subfeatures)
        features = np.zeros(2*len(subfeatures))
        features[:len(subfeatures)] = subfeatures
        features[len(subfeatures):] = action * subfeatures

        return features

    else:
        ''' Directly returns the state and action as a feature '''

        return np.append(state, action)

state = env.reset()

NUM_FEATURES = len(convert_to_features(state, 0))

learning_rate = 1e-4
discount = 0.9

def action_value(state, action, weights):
    ''' Approximate the action value of an state'''
    features = convert_to_features(state, action)
    return features @ weights

def choose_action(state, weights, epsilon = 0.33):
    ''' A random action will be picked with probability epsilon '''
    if epsilon > np.random.uniform():
        action = np.random.randint(0, NUM_ACTIONS)
    else:
        action = np.argmax([action_value(state, action, weights)
                    for action in range(NUM_ACTIONS)])
    return action

def update_weights(previous_state, state, action, next_action,
                   weights, reward, done = False):
    gradient = convert_to_features(previous_state, action)
    if done:
        update = learning_rate * (reward
                        - action_value(previous_state, action, weights))

    else:
        update = learning_rate * (reward
                        + discount * action_value(state, next_action, weights)
                        - action_value(previous_state, action, weights))
    update *= gradient
    weights -= update # Gradient descent
    return weights

''' Calculate average reward per run and per episode '''
average_rewards = np.zeros((NUM_RUNS, NUM_EPISODES))

for j in range(NUM_RUNS):
    ''' Start of another run '''
    print('Run:', j)
    weights = np.random.rand(NUM_FEATURES)

    for k in range(NUM_EPISODES):
        ''' Start of another episode '''
        state = env.reset()
        action = choose_action(state, weights.copy())
        cumulative_reward = 0
        for step in range(MAX_EPISODE_STEPS):
            #env.render()
            previous_state = state
            state, reward, done, info = env.step(action)
            next_action = choose_action(state, weights.copy())
            weights = update_weights(previous_state, state, action,
                                       next_action, weights.copy(), reward, done)

            cumulative_reward += reward
            if done:
                break
            action = next_action
        average_reward = cumulative_reward
        average_rewards[j,k] = average_reward

average_rewards_per_run = average_rewards.sum(axis=0)/NUM_RUNS/MAX_EPISODE_STEPS

plt.plot(average_rewards_per_run)
plt.xlabel('Episode')
plt.ylabel('Average reward per episode and run')
if POLYNOMIAL_FEATURES:
    title = 'Using polynomial features\n'
else:
    title = 'Using states and actions directly as features\n'
title += 'Average of ' + str(NUM_RUNS) + ' runs with ' \
    + str(MAX_EPISODE_STEPS) + ' steps per episode'
plt.title(title)


import os
import shutil
import subprocess
import traceback
from typing import Any, List

from src.manager.launcher.launcher_interface import ILauncher, LauncherException

class LauncherRos(ILauncher):
    """
    Launcher for ROS/Gazebo

    It's configuration should follow this spec:

    {
      "type": "module",
      "module": "ros",
      "resource_folders": [
        "$EXERCISE_FOLDER/launch"
      ],
      "model_folders": [
        "$CUSTOM_ROBOTS/f1/models"
      ],
      "plugin_folders": [
      ],
      "parameters": [],
      "launch_file": "$EXERCISE_FOLDER/launch/simple_line_follower_ros_headless_default.launch"
    }
    """
    exercise_id: str
    type: str
    module: str
    resource_folders: List[str]
    model_folders: List[str]
    plugin_folders: List[str]
    parameters: List[str]
    launch_file: str

    ros_command_line: str = shutil.which('roslaunch')
    process: Any = None

    def run(self):
        try:
            # generate entry_point environment variable
            os.environ["EXERCISE_FOLDER"] = f"{os.environ.get('EXERCISES_STATIC_FOLDER')}/{self.exercise_id}"

            # expand variables in configuration paths
            resource_folders = [os.path.expandvars(
                path) for path in self.resource_folders]
            model_folders = [os.path.expandvars(
                path) for path in self.model_folders]
            plugin_folders = [os.path.expandvars(
                path) for path in self.plugin_folders]
            launch_file = os.path.expandvars(self.launch_file)

            env = dict(os.environ)
            env["GAZEBO_RESOURCE_PATH"] = f"{env.get('GAZEBO_RESOURCE_PATH', '')}:{':'.join(resource_folders)}"
            env["GAZEBO_MODEL_PATH"] = f"{env.get('GAZEBO_MODEL_PATH', '')}:{':'.join(model_folders)}"
            env["GAZEBO_PLUGIN_PATH"] = f"{env.get('GAZEBO_PLUGIN_PATH', '')}:{':'.join(plugin_folders)}"

            parameters = " ".join(self.parameters)
            command = f"{self.ros_command_line} {parameters} {launch_file}"
            self.process = subprocess.Popen(command, env=env, shell=True,
                                            # stdin=subprocess.PIPE,
                                            # stdout=subprocess.PIPE,
                                            # stderr=subprocess.STDOUT
                                            )
            # print(self.process.communicate())
        except Exception as ex:
            traceback.print_exc()
            raise ex

    def is_running(self):
        return True if self.process.poll() is None else False

    def terminate(self):
        if self.is_running():
            self.process.terminate()
        else:
            raise LauncherException("The process is not running")


import codecs
import json
import uuid
import re
from scrapy.utils.markup import remove_tags
from scrapy.utils.serialize import ScrapyJSONEncoder
from typing import Union

from scrapper.crowler.data.object import Variant, Step

class JsonWithEncodingPipeline(object):

    def __init__(self):
        self.file = \
            codecs.open(f'scraped_data_{uuid.uuid4()}.json',
                        'w',
                        encoding='utf-8')
        self.parsed_count = 0
        self.delete_spaces = re.compile(r"[\f\n\r\t\v]")

    def delete_tags(self, text: Union[str, None]) -> Union[str, None]:
        if text is not None:
            return remove_tags(text)
        else:
            return None

    def process_step(self, step: Step) -> Step:
        text = self.delete_spaces.sub("", step["text"])
        text = [chunk for chunk in text.split("<p>") if len(chunk) > 2]
        step["text"] = "<p>".join(text)
        return step

    def process_variant(self, variant: Variant) -> Variant:
        variant["variant_name"] = self.delete_tags(variant["variant_name"])
        variant["steps"] = [
            self.process_step(step) for step in variant["steps"]
        ]
        return variant

    def process_item(self, article, spider):
        if (self.parsed_count + 1) % 1000 == 0:
            self.file.close()
            self.file = \
                codecs.open(f'scraped_data_{uuid.uuid4()}.json',
                            'w',
                            encoding='utf-8')

        article["variants"] = [
            self.process_variant(variant) for variant in article["variants"]
        ]
        line = json.dumps(article, cls=ScrapyJSONEncoder) + "\n"
        self.file.write(line)
        self.parsed_count += 1
        return article

    def spider_closed(self, spider):
        self.file.close()


# -*- coding: utf-8 -*-
"""
Created on Wed Jun  5 00:48:35 2019

@author: Vamshi Krishna
"""

import matplotlib.pyplot as plt
emp_names=['vamshi','preethi','santhosh','deexita']
emp_salary=[80000,75000,60000,80000]
plt.pie(emp_salary,labels=emp_names,radius=2,autopct='%0.0f%%',shadow=True,explode=[0.2,0,0,0])
plt.show()


# Extract Adobe Analytics data strings from Adobe Assurance (Project Griffon) logs
# 24 Oct 2022 11:18
# Michael Powe
# reminder of what we are looking for
# (json_data['events'][3]['payload']['ACPExtensionEventData']['hitUrl'])

import json
import urllib.parse
from argparse import ArgumentParser
from pprint import pprint

def main():
    parser = ArgumentParser()
    parser.add_argument("-f", "--file", help="file from which to read")
    parser.add_argument("-o", "--output", help="file to which to write results")
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="print output to screen as well as to file",
    )
    args = parser.parse_args()

    # source data file (default if no command line input)
    griffon_data: str = (
        r"c:\Users\micha\Dropbox\src\python\json\data\AssuranceTraining.json"
    )

    # written output file (default if no command line input)
    url_data: str = r"c:\Users\micha\Dropbox\src\python\json\data\assurance-urls.txt"

    inputfile: str = args.file if args.file is not None else griffon_data
    outputfile: str = args.output if args.output is not None else url_data

    context_list: list[str]
    uri_list: list[str]
    uri_list, context_list = process_file(inputfile, pfarg=args.verbose)

    write_data(outputfile, uri_list, context_list)

def process_file(infile: str, pfarg: bool = None) -> tuple[list[str], list[str]]:
    """
    Processes the file of JSON generated by Adobe Assurance and collects the
    URI data string and the variable context data.

    :rtype: object
    :param arg: True to print to screen, default is False.
                Will be set True by the presence of the `-v` command line option
    :type arg: bool
    :param infile: Complete path and filename for the file to be processed
    :type infile: str
    """
    pfarg = True if pfarg is not None and pfarg is not False else False

    f = open(infile, "r")

    # list to collect matching URIs. It will then be written to file at the end
    uri_list: list[str] = []
    # list to collect context data
    context_list: list[str] = []

    json_data: dict = json.load(f)

    length: int = len(json_data["events"])

    for k in range(length):
        try:
            # this is what we use for validating data is correct
            if "hitUrl" in json_data["events"][k]["payload"]["ACPExtensionEventData"]:
                # remove the URI encoding and add to list for printing
                item: str = urllib.parse.unquote(
                    json_data["events"][k]["payload"]["ACPExtensionEventData"].get(
                        "hitUrl"
                    )
                )
                if pfarg:
                    print(item, "\n")
                uri_list.append(item)
            # this is what we use for validating context data variables (for SDR, &c)
            elif (
                "contextdata"
                in json_data["events"][k]["payload"]["ACPExtensionEventData"]
            ):
                item = (json_data["events"][k]["payload"]["ACPExtensionEventData"]).get(
                    "contextdata"
                )
                if pfarg:
                    print()
                    pprint(item)
                    print()
                context_list.append(item)
        # 'key' refers to dictionary key, not a keyboard key
        except KeyError:
            print(">>>>>>>>>> <<<<<<<<<<")
            print(">>>>>>>>>> key error, `ACPExtensionEventData` not found <<<<<<<<<<")
            print(
                ">>>>>>>>>> payload is ",
                json_data["events"][k]["payload"],
                "<<<<<<<<<<",
            )
            print(">>>>>>>>>> <<<<<<<<<<\n")
    f.close()
    return uri_list, context_list

def write_data(url_data: str, uri_list: list[str], context_list: list[str]) -> None:
    """
    Writes the accumulated URI and context data to a file. URI is written to
    the top of the file and context data follows it.

    :param uri_list: list in which data URI are captured from `hitUrl`
    :type uri_list: list
    :param context_list: list in which context data objects are captured
           from `contextdata`
    :type context_list: list
    :param url_data: the JSON file from which the data is to be extracted.
    :type url_data: str
    """
    w = open(url_data, "w")

    # output to file
    for i in uri_list:
        w.write(i + "\n")

    for i in context_list:
        w.write("\n")
        pprint(i, stream=w)

    w.close()

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

"""
Extensive database of location and timezone data for nearly every airport and landing strip in the world.
"""
from __future__ import annotations

import csv
from pathlib import Path
from typing import Dict, Literal, TypedDict

__project_name__ = __package__
# Release numbering follows the release date
__version__ = '20221121'
__min_python_version__ = (3, 8)
__author__ = 'Mike Borsetti <mike@borsetti.com>'
__copyright__ = 'Copyright 2020- Mike Borsetti'
__license__ = 'MIT'
__url__ = f'https://github.com/mborsetti/{__project_name__}'

Airport = TypedDict(
    'Airport',
    {
        'icao': str,
        'iata': str,
        'name': str,
        'city': str,
        'subd': str,
        'country': str,
        'elevation': float,
        'lat': float,
        'lon': float,
        'tz': str,
        'lid': str,
    },
)
CodeType = Literal['ICAO', 'IATA', 'LID']

def load(code_type: CodeType = 'ICAO') -> Dict[str, 'Airport']:
    """Loads airport data into a dict

    :param code_type: optional argument defining the key in the dictionary: 'ICAO' (default if omitted),
    'IATA' (for IATA Location Codes) or 'LID' (for U.S. FAA Location Identifiers).

    :return: a dict of dicts, each entry having the following keys:
        'icao': ICAO 4-letter Location Indicator or 4-alphanumeric FAA/TC LID
        'iata': IATA 3-letter Location Code or an empty string
        'name': Official name (latin script)
        'city': City
        'subd': Subdivision (e.g. state, province, region, etc.)
        'country': ISO 3166-1 alpha 2-code (plus 'XK' for Kosovo)
        'elevation': MSL elevation (the highest point of the landing area) in feet
        'lat': Latitude (decimal)
        'lon': Longitude (decimal)
        'tz': Timezone expressed as a tz database name (IANA-compliant) or empty string for country 'AQ' (Antarctica).
            Originally sourced from [TimeZoneDB](https://timezonedb.com)
        'lid': The FAA Location Identifier (for US country only; others is blank)
    """
    # with open(os.path.join(dir, 'airports.json'), encoding='utf8') as f:
    #     airports = json.load(f)
    # if code_type.lower() == 'icao':
    #     return airports
    # else:
    #     return {airport['iata']: airport for airport in dict(airports).values() if airport['iata']}
    #
    #
    key = code_type.lower()
    if key not in ('icao', 'iata', 'lid'):
        raise ValueError(f'code_type must be one of ICAO, IATA or LID; received {code_type}')
    this_dir = Path(__file__).parent
    airports: Dict[str, Airport] = {}
    with this_dir.joinpath('airports.csv').open(encoding='utf8') as f:
        reader = csv.DictReader(f, quoting=csv.QUOTE_NONNUMERIC)
        for row in reader:
            airports[row[key]] = row  # type: ignore[assignment]
    airports.pop('', None)
    return airports

# Python 3.9 code used to save the dict to CSV:
# with open('airports.csv', 'w', newline='') as f:
#     fieldnames = airports[list(airports.keys())[0]].keys()
#     writer = csv.DictWriter(f, fieldnames=fieldnames, quoting=csv.QUOTE_NONNUMERIC)
#     writer.writeheader()
#     for data in airports.values():
#         writer.writerow(data)


from __future__ import absolute_import, division
import scipy.ndimage as ndimage
from matplotlib import widgets
import glob
import datetime
import scipy.interpolate as interpolate
import numpy as np
import matplotlib.pyplot as plt

__all__ = ['Slit']

#==============================================================================
# Slit Class
#==============================================================================

class Slit:

    def __init__(self, image_animator):
        self.image_animator = image_animator
        self.axes = self.image_animator.axes
        self.points = []
        self.mpl_points = []
        self.mpl_curve = []
        self.anns = []
        self.res = 100
        self.curve_points = np.zeros([self.res,2])
        self.data = []
        self.data_run = []
        self.distance = []

    def add_point(self, x, y):
        self.points.append([x,y])
        self.mpl_points.append(self.axes.scatter(x,y))
        self.anns.append(self.axes.annotate('%i' % len(self.mpl_points), (x + 1, y + 1)))
        self.axes.figure.canvas.draw()

    def remove_point(self):
        if len(self.mpl_points) > 0:
            point = self.mpl_points.pop(-1)
            point.set_visible(False)
            annote = self.anns.pop(-1)
            annote.set_visible(False)
            self.points.pop(-1)
            self.axes.figure.canvas.draw()

    def remove_all(self, slits):
        if len(slits) > 0:
            for i in range(len(slits)):
                if len(slits[i].mpl_points) > 0:
                    for y in range(len(slits[i].mpl_points)):
                        point = slits[i].mpl_points.pop()
                        point.set_visible(False)
                        annote = slits[i].anns.pop()
                        annote.set_visible(False)
                        slits[i].points.pop()
                    if len(slits[i].mpl_curve) > 0:
                     for line in self.axes.lines:
                         self.axes.lines.pop()
            self.axes.figure.canvas.draw()

    def create_curve(self):
        flag = False # True - Beizer Curve else Interpolation
        if len(self.mpl_points) == 2:
            self.curve_points = self.linear_bezier(*self.points)
        elif len(self.mpl_points) == 3:
            if flag:
                self.curve_points = self.quad_bezier(self.points[0],self.points[-1],self.points[1])
            else:
                self.curve_points = self.interpol(*self.points)
        elif len(self.mpl_points) == 4:
            if flag:
                self.curve_points = self.cubic_bezier(self.points[0],self.points[2],self.points[-1],self.points[1])
            else:
                self.curve_points = self.interpol(*self.points)
        else:
            self.curve_points = self.interpol(*self.points)
        self.mpl_curve.append(self.axes.plot(self.curve_points[:,0], self.curve_points[:,1]))
        self.axes.figure.canvas.draw()

    def interpol(self,*args):
        x,y = zip(*args)
        if len(x) == 3:
            k = 2
        else:
            k = 3
        tck,u = interpolate.splprep([x,y], k = k)
        unew = np.linspace(0, self.res,self.res) / self.res
        curve = interpolate.splev(unew,tck)
        ans = np.zeros([self.res, 2])
        ans[:,0] = curve[0]
        ans[:,1] = curve[1]
        return ans

    def cubic_bezier(self, P0, P1, P2, P3):
        ans = np.zeros([self.res, 2])
        t = np.linspace(0, self.res,self.res) / self.res
        ans[:,0] = (1 - t)**3 * P0[0] + 3*(1 - t)**2 *t*P1[0] + 3*(1-t)*t**2*P2[0] + t**3*P3[0]
        ans[:,1] = (1 - t)**3 * P0[1] + 3*(1 - t)**2 *t*P1[1] + 3*(1-t)*t**2*P2[1] + t**3*P3[1]
        return ans

    def quad_bezier(self, P0, P1, P2):
        ans = np.zeros([self.res, 2])
        t = np.linspace(0, self.res,self.res) / self.res
        ans[:,0] = (1 - t)**2 * P0[0] + 2*(1 - t)*t*P1[0] + t**2*P2[0]
        ans[:,1] = (1 - t)**2 * P0[1] + 2*(1 - t)*t*P1[1] + t**2*P2[1]
        return ans

    def linear_bezier(self, P0, P1):
        ans = np.zeros([self.res, 2])
        t = np.linspace(0, self.res,self.res) / self.res
        ans[:,0] = (1 - t) * P0[0] + t*P1[0]
        ans[:,1] = (1 - t) * P0[1] + t*P1[1]
        return ans

    def get_slit_data(self, data, extent, order=1):
        if not hasattr(self, 'curve_points'):
            print('You have not yet generated a curve.')

        x_pixel = (self.curve_points[:,0] - extent[2] )/ ((extent[3] - extent[2]) / data.shape[2])
        y_pixel = (self.curve_points[:,1] - extent[0] )/ ((extent[1] - extent[0]) / data.shape[2])

        dist_x = (x_pixel[:-1] - x_pixel[1:]) ** 2
        dist_y = (y_pixel[:-1] - y_pixel[1:]) ** 2
        self.distance = np.sum(np.sqrt(dist_x + dist_y))

        if len(data.shape) == 2:
            slit = ndimage.interpolation.map_coordinates(data, [y_pixel,x_pixel], order=order)
        elif len(data.shape) == 3:
            slit = np.zeros([data.shape[0],self.res])
            for i in range(0,data.shape[0]):
                slit[i,:] = ndimage.interpolation.map_coordinates(data[i,:,:], [y_pixel,x_pixel], order=order)
        else:
            raise Exception
        return slit

    def get_run_diff(self, slit, sort='normal', j = 5):
        if sort == 'normal':
            self.data_run.append(slit[:-1] - slit[1:])
            return self.data_run[-1]
        elif sort == 'baseline':
            self.data_run.append(slit[:-1] - slit[0])
            return self.data_run[-1]
        elif sort == 'symmetric':
            self.data_run.append(slit[:-1] - slit[1:])
            return self.data_run[-1]
        elif sort == 'symmetric':
            self.data_run.append(slit[:-1] - slit[1:])
            return self.data_run[-1]
        elif sort == 'symmetric':
            self.data_run.append(slit[:-1] - slit[1:])
            return self.data_run[-1]


from __future__ import print_function, division
import we
import json
import numpy as np
import sys
if sys.version_info[0] < 3:
    import io
    open = io.open
"""
Hard-debias embedding
Man is to Computer Programmer as Woman is to Homemaker? Debiasing Word Embeddings
Tolga Bolukbasi, Kai-Wei Chang, James Zou, Venkatesh Saligrama, and Adam Kalai
2016
"""

def debias(E, gender_specific_words, definitional, equalize):
    gender_direction = we.doPCA(definitional, E).components_[0]
    #f = open("bolukbasi_debiased_gender_direction.txt", "w")
    #f.write(str(gender_direction))
    #f.close()

    specific_set = set(gender_specific_words)
    for i, w in enumerate(E.words):
        if w not in specific_set:
            E.vecs[i] = we.drop(E.vecs[i], gender_direction)
    E.normalize()
    candidates = {x for e1, e2 in equalize for x in [(e1.lower(), e2.lower()),
                                                     (e1.title(), e2.title()),
                                                     (e1.upper(), e2.upper())]}
    print(candidates)
    for (a, b) in candidates:
        if (a in E.index and b in E.index):
            y = we.drop((E.v(a) + E.v(b)) / 2, gender_direction)
            z = np.sqrt(1 - np.linalg.norm(y)**2)
            if (E.v(a) - E.v(b)).dot(gender_direction) < 0:
                z = -z
            E.vecs[E.index[a]] = z * gender_direction + y
            E.vecs[E.index[b]] = -z * gender_direction + y
    E.normalize()

if __name__ == "__main__":

    embedding_filename = "word2vec_no_reg.txt"
    definitional_filename = "definitional_pairs.json"
    gendered_words_filename = "gendered_words.json"
    equalize_filename = "equalized_pairs.json"
    debiased_filename = "bolukbasi2016_debiased_word2vec.txt"

    with open(definitional_filename, "r") as f:
        defs = json.load(f)
    print("definitional", defs)

    with open(equalize_filename, "r") as f:
        equalize_pairs = json.load(f)

    with open(gendered_words_filename, "r") as f:
        gender_specific_words = json.load(f)
    print("gender specific", len(gender_specific_words), gender_specific_words[:10])

    E = we.WordEmbedding(embedding_filename)

    print("Debiasing...")
    debias(E, gender_specific_words, defs, equalize_pairs)

    print("Saving to file...")
    if embedding_filename[-4:] == debiased_filename[-4:] == ".bin":
        E.save_w2v(debiased_filename)
    else:
        E.save(debiased_filename)

    print("\n\nDone!\n")


from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import average_precision_score
from sklearn.metrics import recall_score
# accuracy_scores: return the correctly classified samples. The set of labels predicted for a sample must exactly match the corresponding set of labels

# f1_score: F1 = 2 * (precision * recall) / (precision + recall), average could be micro, macro, weighted

# roc_auc_score: Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores. Note: this implementation is restricted to the binary classification task or multilabel classification task in label indicator format.

# average_precision_score: Compute average precision (AP) from prediction scores

# recall_score
#'micro': Calculate metrics globally by considering each element of the label indicator matrix as a label.
#'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.
#'weighted': Calculate metrics for each label, and find their average, weighted by support (the number of true instances for each label).

# accuracy, f1 score, and recall use y_true and y_pred
# roc_auc_score and average_precision_score use y_true and y_score

def evaluation_true_pred(y_validation, y_predicted):
    accuracy = accuracy_score(y_validation, y_predicted)
    f1 = f1_score(y_validation, y_predicted, average = 'weighted')
    recall = recall_score(y_validation, y_predicted, average = 'weighted')
    print("Accuray is {}".format(accuracy))
    print("F1 is {}".format(f1))
    print("Recall is {}".format(recall))

def evaluation_true_scores(y_validation, y_scores):
    roc = roc_auc_score(y_validation, y_scores)
    average_precision = average_precision_score(y_validation, y_scores)
    print("ROC is {}".format(roc))
    print("Average Precision is {}".format(average_precision))


# geospatial help functions
import geopandas as gpd
import matplotlib.pyplot as plt

def show_overlaps(data_temp, id_col):
    # checks and plots which areas overlap

    overlaps = []
    for index, row in data_temp.iterrows():
        data_temp1 = data_temp.loc[data_temp[id_col] != row[id_col]] # grab all rows, != this row
        temp_list = data_temp1[data_temp1.geometry.overlaps(row.geometry)][id_col].tolist() # grab rows with overlap
        # get index
        if temp_list != []:
            overlaps += temp_list

    data_overlaps = data_temp.copy()
    data_overlaps = data_overlaps.set_index(id_col)
    overlap_areas = data_overlaps.loc[overlaps]

    ax = data_temp.plot(figsize=size, linewidth=lw, edgecolor="k", color="whitesmoke")
    if len(overlap_areas) > 0:
        overlap_areas.plot(figsize=size, linewidth=lw, edgecolor="k", color="red", ax=ax)
    print("This many rows still overlap:", overlap_areas.shape[0])

def boundary_cut(df_list, cut_df):
    # cuts a list of dfs out of the input df
    for df in df_list:
        cut_df = gpd.overlay(cut_df, df, how="difference")
        return cut_df

def plot_bounds(gdf):
    # quick plot with standard parameters
    gdf.plot(figsize=(24, 20), color="whitesmoke", edgecolor="k", linewidth=0.5)


import numpy as np
import glob
import os
from blimpy import Waterfall
import blimpy as bl
import gc
import time
import matplotlib
import matplotlib.pyplot as plt

start=time.time()

def get_elapsed_time(start=0):
    end = time.time() - start
    time_label = 'seconds'
    if end > 3600:
        end = end/3600
        time_label = 'hours'
    elif end > 60:
        end = end/60
        time_label = 'minutes'
    return end, time_label

def plot_waterfall(wf, source_name=None, f_start=None, f_stop=None, **kwargs):
    # get plot data
    plot_f, plot_data = wf.grab_data(f_start=f_start, f_stop=f_stop)

    # determine extent of the plotting panel for imshow
    extent=(plot_f[0], plot_f[-1], (wf.timestamps[-1]-wf.timestamps[0])*24.*60.*60, 0.0)

    # plot and scale intensity (log vs. linear)
    kwargs['cmap'] = kwargs.get('cmap', 'viridis')
    plot_data = 10.0 * np.log10(plot_data)

    # get normalization parameters
    vmin = plot_data.min()
    vmax = plot_data.max()
    normalized_plot_data = (plot_data - vmin) / (vmax - vmin)

    # display the waterfall plot
    this_plot = plt.imshow(normalized_plot_data,aspect='auto',rasterized=True,interpolation='nearest',extent=extent,**kwargs)

    del plot_f, plot_data

    return this_plot

def one_wf_plot(fil_file_list,f_start=0,f_stop=0,drift_rate=0,source_name_list=[],save_dir=None,save_name=None,extension='png'):
    # set up the sub-plots
    n_plots = len(fil_file_list)
    fig,ax = plt.subplots(n_plots, sharex=True, sharey=True,figsize=(10, 2*n_plots))

    if not f_start or not f_stop:
        print('\nf_start and/or f_stop input error. Try again.\n')
        return None

    # define more plot parameters
    mid_f = np.abs(f_start+f_stop)/2.

    subplots = []

    # Fill in each subplot for the full plot
    for ii, filename in enumerate(fil_file_list):
        # identify panel
        subplot = plt.subplot(n_plots, 1, ii + 1)
        subplots.append(subplot)

        # read in data
        max_load = bl.calcload.calc_max_load(filename)
        wf = bl.Waterfall(filename, f_start=f_start, f_stop=f_stop, max_load=max_load)

        this_plot = plot_waterfall(wf, f_start=f_start, f_stop=f_stop)

        del wf
        gc.collect()

    plt.subplots_adjust(hspace=0,wspace=0)
    [axi.set_axis_off() for axi in ax.ravel()]
    fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)

    # save the figures
    if save_dir:
        if save_dir[-1] != "/":
            save_dir += "/"
        if save_name:
            path_png = save_dir+save_name
        else:
            path_png = save_dir + 'freq_' "{:0.6f}".format(mid_f) + '.' + extension
        plt.savefig(path_png, bbox_inches='tight',format=extension, pad_inches=0)

    return fig

blc32_f2_plots=sorted(glob.glob('/storage/home/nxt5197/work/PPO/TOI-216/node_by_node/blc32/Freqs_3904_to_4032/MJDate_59221/plots_TIC55652896_S_f2_snr10.0/*.png'))
os.chdir('/storage/home/nxt5197/scratch/PPO/TOI-216/')
fils=sorted(glob.glob('*h5'))
power_matrix = []
freq_ranges = []
counter_file = '/storage/home/nxt5197/work/589_Machine_Learning/plots/data/icounter.txt'
for i,png in enumerate(blc32_f2_plots):
    f_start = float(png.split('_freq_')[1].split('.png')[0])
    drift_rate = float(png.split('_freq_')[0].split('_dr_')[1])
    if abs(drift_rate) <= 0.10:
        half_f = 250
    elif abs(drift_rate) == 0.11:
        half_f = 272
    elif abs(drift_rate) == 0.13:
        half_f = 304
    elif abs(drift_rate) == 0.14:
        half_f = 337
    elif abs(drift_rate) == 0.32:
        half_f = 769
    elif abs(drift_rate) == 9.14:
        half_f = 21969
    elif abs(drift_rate) == 24.31:
        half_f = 58424
    elif abs(drift_rate) == 36.21:
        half_f = 87026
    else:
        print('You fucked up. You got the drift rate and frequency range wrong. Try again, buddy.')
        break
    f_mid = f_start+half_f*1e-6
    delta_f = 500
    f_stop = f_mid+delta_f/2*1e-6
    f_start = f_mid-delta_f/2*1e-6
    save_dir='/storage/home/nxt5197/work/589_Machine_Learning/plots/imgs/'
    save_name=png.split('/')[-1]
    image = one_wf_plot(fils,f_start=f_start,f_stop=f_stop,save_dir=save_dir,save_name=save_name,extension='png')
    end, time_label = get_elapsed_time(start)
    with open(counter_file, 'w') as f:
        f.write(f'{i+1} of {len(blc32_f2_plots)} for loops completed in {end:.2f} {time_label}.')
os.chdir('/storage/home/nxt5197/work/589_Machine_Learning/plots/data/')
np.save('power_matrix.npy', power_matrix)
np.save('freq_ranges.npy', freq_ranges)


import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

def hidden_init(layer):
    fan_in = layer.weight.data.size()[0]
    lim = 1. / np.sqrt(fan_in)
    return (-lim, lim)

class Actor(nn.Module):
    """Initialize parameters and build model.
        Args:
            state_size (int): Dimension of each state
            action_size (int): Dimension of each action
            max_action (float): highest action to take
            seed (int): Random seed
            h1_units (int): Number of nodes in first hidden layer
            h2_units (int): Number of nodes in second hidden layer

        Return:
            action output of network with tanh activation
    """

    def __init__(self, state_dim, action_dim, max_action, seed):
        super(Actor, self).__init__()
        self.seed = torch.manual_seed(seed)

        self.l1 = nn.Linear(state_dim, 400)
        self.l2 = nn.Linear(400, 300)
        self.l3 = nn.Linear(300, action_dim)

        self.max_action = max_action

    def forward(self, x):
        x = F.relu(self.l1(x))
        x = F.relu(self.l2(x))
        x = self.max_action * torch.tanh(self.l3(x))
        return x

class Critic(nn.Module):
    """Initialize parameters and build model.
        Args:
            state_size (int): Dimension of each state
            action_size (int): Dimension of each action
            max_action (float): highest action to take
            seed (int): Random seed
            h1_units (int): Number of nodes in first hidden layer
            h2_units (int): Number of nodes in second hidden layer

        Return:
            value output of network
    """

    def __init__(self, state_dim, action_dim, seed):
        super(Critic, self).__init__()
        self.seed = torch.manual_seed(seed)

        # Q1 architecture
        self.l1 = nn.Linear(state_dim + action_dim, 400)
        self.l2 = nn.Linear(400, 300)
        self.l3 = nn.Linear(300, 1)

        # Q2 architecture
        self.l4 = nn.Linear(state_dim + action_dim, 400)
        self.l5 = nn.Linear(400, 300)
        self.l6 = nn.Linear(300, 1)

    def forward(self, x, u):
        xu = torch.cat([x, u], 1)

        x1 = F.relu(self.l1(xu))

        x1 = F.relu(self.l2(x1))

        x1 = self.l3(x1)

        x2 = F.relu(self.l4(xu))

        x2 = F.relu(self.l5(x2))

        x2 = self.l6(x2)
        return x1, x2

    def Q1(self, x, u):
        xu = torch.cat([x, u], 1)

        x1 = F.relu(self.l1(xu))

        x1 = F.relu(self.l2(x1))

        x1 = self.l3(x1)
        return x1


import webapp2
import random

def rand_fortune():
    fortunes = ["To avoid criticism, do nothing, say nothing, be nothing", "Error 404: Fortune not found", "Optimist believe we live in the best of worlds and pessimists fear this is true.", "Of all our human resources, the most precious is the desire to improve"]
    index = random.randint(0,3)

    return fortunes[index]

class MainHandler(webapp2.RequestHandler):
    def get(self):
        header = "
Fortune Cookie
"

        fortune = rand_fortune()
        final_fortune_sent = "
"'Your fortune is: ' + fortune + "
"

        lucky_num = random.randint(1,100)
        num_sent = 'Your lucky number: ' + str(lucky_num)
        final_num_sent = "
" + num_sent + "
"

        another_cookie = "
<button>another cookie please</button>
"

        da_cookie = header + final_fortune_sent + final_num_sent + another_cookie
        self.response.write(da_cookie)

app = webapp2.WSGIApplication([
    ('/', MainHandler)
], debug=True)


import wave
import numpy as np
import scipy
from scipy.io.wavfile import read
from scipy.signal import hann
from scipy.fftpack import rfft
import matplotlib.pyplot as plt

def plotFreqSpec(filename,graphpath=None):
    # read audio samples
    framerate,data = read(filename)

    file = wave.open(filename, 'rb')
    nchannels, sampwidth, framerate, nframes = file.getparams()[0:4]

    data = data.astype(np.float, copy=False)

    if nchannels > 1:
        data2 = np.mean(a=data, axis=1)
        data = data2

    windowsize = 1024
    lackingFrames = nframes%windowsize
    data = np.append(data,[0]*lackingFrames)
    window = hann(windowsize)
    data = data/(2**(sampwidth*8))
    audio = data
    for i in range(0,nframes//windowsize):
        # apply a Hanning window
        audio[(i*windowsize):((i+1)*windowsize)] = audio[(i*windowsize):((i+1)*windowsize)]*window
    # fft
    mags = abs(rfft(audio))
    # convert to dB
    mags = 20*scipy.log10(mags)
    # normalise to 0 dB max
    mags -= max(mags)
    # plot
    plt.plot(mags)
    # label the axes
    plt.ylabel("Magnitude (dB)")
    plt.xlabel("Frequency Bin")
    # set the title
    plt.title("Spectrum")
    if graphpath is not None:
        plt.savefig("Spectrum.png")
    else:
        plt.show()

if __name__ == "__main__":
    plotFreqSpec("./static/music/temp.wav")


from django.shortcuts import render, redirect, reverse

# Create your views here.

from django.views.decorators.http import require_GET
from django.contrib.auth.decorators import login_required
from goods.models import Goods
from . import models

@require_GET
@login_required
def add(req, count, goods_id):
    goods = Goods.objects.get(pk=goods_id)
    user = req.user

    try:
        shopCart = models.ShopCart.objects.get(user=user, goods=goods)
        shopCart.count += int(count)
        shopCart.allTotal = shopCart.count * goods.price
        shopCart.save()
    except:
        shopCart = models.ShopCart(goods=goods, user=user)
        shopCart.count = int(count)
        shopCart.allTotal = shopCart.count * goods.price
        shopCart.save()

    return redirect(reverse('shopcart:list'))

#购物车的列表
@login_required
def list(req):
    shopcarts = models.ShopCart.objects.filter(user=req.user).order_by('-addTime')

    return render(req, 'shopcart/list.html', {'shopcarts': shopcarts})

#购物车的删除
def delete(req, shopcart_id):
    shopCart = models.ShopCart.objects.get(pk=shopcart_id)
    shopCart.delete()
    return redirect(reverse('shopcart:list'))


from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot

class TInteractObj(QObject):
    SigReceivedMessFromJS = pyqtSignal(dict)
    SigSendMessageToJS = pyqtSignal(str)
    def __init__(self, parent = None):
        super().__init__(parent)

    @pyqtSlot(str,result=str)
    def JSSendMessage(self, strParameter):
        print('JSSendMessage(%s) from Html' %strParameter)
        dict1 = {'param':strParameter,'result':''}
        self.SigReceivedMessFromJS.emit(dict1)
        return dict1['result']

    @pyqtSlot(result=str)
    def fun(self):
        print('TInteractObj.fun()')
        return 'hello1'


from django.shortcuts import render
from inicio.models import Paciente
from django.views.generic.list import ListView
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.contrib.auth.mixins import LoginRequiredMixin
from django.urls import reverse_lazy

def mi_vista(request):
    return render(request, 'inicio/index.html')

def acerca(request):
    return render(request, 'inicio/acerca.html')

class ListaPacientes(ListView):
    model = Paciente
    template_name = 'inicio/CBV/lista_pacientes.html'

class CrearPaciente(CreateView):
    model = Paciente
    template_name = 'inicio/CBV/crear_paciente.html'
    success_url = reverse_lazy('lista_pacientes')
    fields = ['nombre', 'apellido', 'dni', 'año_nacimiento']

class ModificarPaciente(LoginRequiredMixin, UpdateView):
    model = Paciente
    template_name = 'inicio/CBV/modificar_pacientes.html'
    success_url = reverse_lazy('lista_pacientes')
    fields = ['nombre', 'apellido', 'dni', 'año_nacimiento']

class EliminarPaciente(LoginRequiredMixin, DeleteView):
    model = Paciente
    template_name = 'inicio/CBV/eliminar_paciente.html'
    success_url = reverse_lazy('lista_pacientes')


from copy import deepcopy
from typing import Optional, List
from types import MethodType
from collections import defaultdict
from nltk.tokenize import sent_tokenize
from nltk.corpus import stopwords
import random
import spacy
import torch

from parlai.core.agents import create_agent, create_agent_from_shared
from parlai.core.params import ParlaiParser
from parlai.core.opt import Opt
from parlai.core.message import Message
from parlai.core.agents import Agent
from parlai.core.metrics import F1Metric
from parlai.agents.rag.args import setup_rag_args
from parlai.agents.bart.bart import BartAgent
from parlai.core.metrics import normalize_answer
from parlai.tasks.wizard_of_wikipedia.agents import (
    TOKEN_NOCHOSEN,
)

from parlai.tasks.wizard_of_wikipedia.agents import (
    TOKEN_KNOWLEDGE,
    TOKEN_END_KNOWLEDGE,
)

STOP_WORDS = stopwords.words('english')

def load_opt_from_file(opt_file):
    if not opt_file.endswith('.opt'):
        opt_file += '.opt'
    return Opt.load(opt_file)

def wow_get_batch_context(self, batch, orig_fun=None):
    """
    Set the beam context for n-gram context blocking specific for WoW data.

    For WoW, we don't want to consider the knowledge in the input for the context beam
    blocking. That's why we mask it out here.
    """
    ctxts = orig_fun(batch)
    knowledge_start_id = self.dict.txt2vec(TOKEN_KNOWLEDGE)
    knowledge_end_id = self.dict.txt2vec(TOKEN_END_KNOWLEDGE)

    def mask_ctxttensor_between_sublists(
        ctxts: torch.Tensor, sub1: List[int], sub2: List[int]
    ) -> torch.Tensor:
        """
        Generate a mask that masks out the context between sub1 and sub2.
        """
        mask = []
        for ctxt in ctxts:
            mask_idxs = []
            should_copy = False
            idx_pointer = 0
            id_to_match = sub1
            for j, token in enumerate(ctxt.cpu().numpy()):
                if token == id_to_match[idx_pointer]:
                    idx_pointer += 1
                    if idx_pointer == 1 and id_to_match == sub1:
                        mask_idxs.append([j])
                    elif idx_pointer >= len(id_to_match):
                        should_copy = id_to_match == sub1
                        idx_pointer = 0
                        id_to_match = sub2 if (id_to_match == sub1) else sub1
                        mask_idxs[-1].append(j)
                    else:
                        mask_idxs[-1].append(j)
                elif should_copy:
                    mask_idxs[-1].append(j)
                elif idx_pointer > 0:
                    idx_pointer = 0
                    del mask_idxs[-1]
            mask.append(
                [
                    0 if idx in [i for sl in mask_idxs for i in sl] else 1
                    for idx in range(len(ctxt))
                ]
            )
        return torch.LongTensor(mask).to(ctxts.device)

    ctxts *= mask_ctxttensor_between_sublists(
        ctxts, knowledge_start_id, knowledge_end_id
    )
    return ctxts

def find_supporting_sentence(question: str, answer: str, docs: List[str]) -> str:
    """
    Finds the supporting sentence for the answer in the docs.
    """
    # Remove the title of the documents.
    for i, doc in enumerate(docs):
        if ' | ' in doc:
            docs[i] = '. '.join(doc.split(' | ')[1:])
    concat_docs = '. '.join(docs)
    sentences = sent_tokenize(concat_docs)

    # Sort sentences according to recall with the answer and question.
    sorted_sentences = sorted(
        sentences,
        key=lambda sentence: (
            F1Metric._prec_recall_f1_score(
                normalize_answer(answer).split(), normalize_answer(sentence).split()
            )[0],
            F1Metric._prec_recall_f1_score(
                normalize_answer(question).split(), normalize_answer(sentence).split()
            )[0],
        ),
        reverse=True,
    )

    return sorted_sentences[0]

def extract_entities(
    sentence, pos=('PROPN', 'NOUN'), use_named_entities=True, use_noun_chunks=True
):
    global nlp
    if nlp is None:
        nlp = spacy.load("en_core_web_sm")
    doc = nlp(sentence)
    results = []
    if pos:
        for token in doc:
            if token.pos_ in pos:
                results.append(token)
    if use_named_entities:
        for ent in doc.ents:
            results.append(ent)
    if use_noun_chunks:
        for chunk in doc.noun_chunks:
            if chunk.text.lower() not in STOP_WORDS:
                results.append(chunk)
    results = list(set([r.text for r in results]))
    return results

def extract_knowledge(txt: str) -> List[str]:
    if not txt or not txt.split():
        return []
    entities = extract_entities(txt)
    return [e.lower() for e in (entities if entities else txt.split())]

def knowledge_from_dialogue_response(dialogue_response: str) -> str:
    """
    Get a knowledge response based on the dialogue response.

    We use a random entity from the dialogue response as knowledge. If there is no
    entity present, we use a random word. If there are no words present, we use
    TOKEN_NOCHOSEN.
    """
    knowledge_options = extract_knowledge(dialogue_response)
    if not knowledge_options:
        return TOKEN_NOCHOSEN
    return random.choice(knowledge_options)

class StackedKnowledgeDialogueAgent(Agent):
    """
    Stacked model that generates first the knowledge, and then the dialogue response.
    """

    @classmethod
    def add_cmdline_args(
        cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
    ) -> ParlaiParser:
        agent = parser.add_argument_group('StackedKnowledgeDialogueAgent options')

        additional_agent_parser = ParlaiParser(add_parlai_args=False)
        BartAgent.add_cmdline_args(additional_agent_parser)
        setup_rag_args(additional_agent_parser)

        for action in additional_agent_parser._actions:
            key = action.option_strings[-1]
            type = action.type

            for prefix in [
                'krm',  # knowledge response model
                'drm',  # dialogue response model
                'drmrw',  # dialogue response model rag-wiki
                'drmnk',  # dialogue response model no-knowledge
            ]:
                agent.add_argument(
                    f'--{prefix}-{key.strip("-")}',
                    type=type,
                    required=False,
                )

        agent.add_argument(
            '--knowledge-response-model-path',
            type=str,
            default=''
            'wow_knowledge_response_generation_sweep_1_Tue_Aug_17_1822/9aa/model',
            help='Model used to generate the knowledge response.',
        )
        agent.add_argument(
            '--dialogue-response-model-path',
            type=str,
            default='',
            help='Model used to generate the dialogue response.',
        )
        agent.add_argument(
            '--dialogue-response-no-knowledge-model-path',
            type=str,
            default='',
            help='Model used to generate the dialogue response without knowledge.',
        )
        agent.add_argument(
            '--dialogue-response-rag-wiki-model-path',
            type=str,
            default='',
            help='Model used to generate the dialogue response with Wiki knowledge.',
        )
        agent.add_argument(
            '--use-supporting-sentence-as-knowledge',
            type=bool,
            default=False,
            help='Instead of using the knowledge response directly to condition the dialogue'
            ' model, we search for the top supporting sentence and use this.',
        )
        agent.add_argument(
            '--beam-filter-for-knowledge-response',
            type=bool,
            default=False,
            help='Try to pick a beam that contains the knowledge response.',
        )
        agent.add_argument(
            '--beam-filter-questions',
            type=bool,
            default=False,
            help='Try to pick a beam that does not contain a question mark.',
        )
        agent.add_argument(
            '--beam-filter-self-references',
            type=bool,
            default=False,
            help='Try to pick a beam that does not contain self references like "I" and "me".',
        )
        agent.add_argument(
            '--beam-disregard-knowledge-for-context-blocking',
            type=bool,
            default=True,
            help='If True disregard the knowledge input for the context blocking.',
        )
        agent.add_argument(
            '--add-fixed-confidence',
            type=int,
            default=-1,
            help='Add a fixed confidence score of the knowledge response.',
        )
        agent.add_argument(
            '--add-confidence-as-str',
            type=bool,
            default=False,
            help='If we add a confidence score to the KRM, we add it as a str.',
        )
        return parser

    def __init__(self, opt, shared=None):
        self.id = 'StackedKnowledgeDialogueAgent'
        self._construct_opts(opt)

        self.knowledge_agent = None
        self.dialogue_agent = None
        self.dialogue_agent_no_knowledge = None
        self.dialogue_agent_rag_wiki = None

        if not shared:
            self._init_knowledge_model()
            self._init_dialogue_models()
        else:
            if 'knowledge_agent_share' in shared:
                self.knowledge_agent = create_agent_from_shared(
                    shared['knowledge_agent_share']
                )
            if 'dialogue_agent_share' in shared:
                self.dialogue_agent = create_agent_from_shared(
                    shared['dialogue_agent_share']
                )
            if 'dialogue_agent_no_knowledge_share' in shared:
                self.dialogue_agent = create_agent_from_shared(
                    shared['dialogue_agent_no_knowledge_share']
                )
            if 'dialogue_agent_rag_wiki_share' in shared:
                self.dialogue_agent = create_agent_from_shared(
                    shared['dialogue_agent_rag_wiki_share']
                )

        self.shared = shared
        super().__init__(opt, shared)

    @property
    def has_no_knowledge_dialogue_model(self):
        return (
            self.opts['init']['dialogue_response_no_knowledge_model_path']
            and self.opts['init']['dialogue_response_no_knowledge_model_path'] != 'None'
        )

    @property
    def has_rag_wiki_dialogue_model(self):
        return (
            self.opts['init']['dialogue_response_rag_wiki_model_path']
            and self.opts['init']['dialogue_response_rag_wiki_model_path'] != 'None'
        )

    def _agent_opt(
        self, filename, specific_override_args, general_override_args, **kwargs
    ):
        opt = load_opt_from_file(filename)
        opt['override'] = {}
        blocklist_general = ['model', 'model_file', 'init_model']
        standard_override_args = {
            'skip_generation': False,
            'inference': 'beam',
            'beam_block_ngram': 3,
            'beam_context_block_ngram': -1,
            'beam_size': 3,
        }
        general_override_args = {
            **general_override_args,
            **standard_override_args,
            **kwargs,
        }

        # Remove the prefix for the model for the specific override args.
        specific_override_args = {
            '_'.join(k.split('_')[1:]): v for k, v in specific_override_args.items()
        }

        # Specific for --indexer-type.
        if 'indexer_type' in specific_override_args:
            # TODO: do we also need to overwrite path_to_index?
            pass

        override_args = {**general_override_args, **specific_override_args}

        for k, v in override_args.items():
            if k not in blocklist_general and k in opt:
                opt['override'][k] = v

        return opt

    def _construct_opts(self, opt):
        self.opts = {}
        self.opts['init'] = opt
        override_opts = defaultdict(dict)
        for k, v in opt['override'].items():
            if k.startswith('krm_'):
                if v is not None:
                    override_opts['knowledge_agent'][k] = v
            elif k.startswith('drm_'):
                if v is not None:
                    override_opts['dialogue_agent'][k] = v
            elif k.startswith('drmnk_'):
                if v is not None:
                    override_opts['dialogue_agent_no_knowledge'][k] = v
            elif k.startswith('drmrw_'):
                if v is not None:
                    override_opts['dialogue_agent_rag_wiki'][k] = v
            else:
                override_opts['general'][k] = v
        self.opts['override'] = override_opts

        if opt['knowledge_response_model_path'] and opt[
            'knowledge_response_model_path'
        ] not in ['oracle']:
            self.opts['knowledge_agent'] = self._agent_opt(
                filename=opt['knowledge_response_model_path'],
                specific_override_args=override_opts['knowledge_agent'],
                general_override_args=override_opts['general'],
            )
        self.opts['dialogue_agent'] = self._agent_opt(
            filename=opt['dialogue_response_model_path'],
            specific_override_args=override_opts['dialogue_agent'],
            general_override_args=override_opts['general'],
        )
        if self.has_no_knowledge_dialogue_model:
            self.opts['dialogue_agent_no_knowledge'] = self._agent_opt(
                filename=opt['dialogue_response_no_knowledge_model_path'],
                specific_override_args=override_opts['dialogue_agent_no_knowledge'],
                general_override_args=override_opts['general'],
            )
        if self.has_rag_wiki_dialogue_model:
            self.opts['dialogue_agent_rag_wiki'] = self._agent_opt(
                filename=opt['dialogue_response_rag_wiki_model_path'],
                specific_override_args=override_opts['dialogue_agent_rag_wiki'],
                general_override_args=override_opts['general'],
            )

    def share(self):
        shared = super().share()
        shared['knowledge_agent_share'] = self.knowledge_agent.share()
        shared['dialogue_agent_share'] = self.dialogue_agent.share()
        if self.has_no_knowledge_dialogue_model:
            shared[
                'dialogue_agent_no_knowledge_share'
            ] = self.dialogue_agent_no_knowledge.share()
        if self.has_rag_wiki_dialogue_model:
            shared[
                'dialogue_agent_rag_wiki_share'
            ] = self.dialogue_agent_rag_wiki.share()
        return shared

    def _init_knowledge_model(self):
        # Initialize knowledge agent.
        if 'knowledge_agent' in self.opts:
            self.knowledge_agent = create_agent(
                self.opts['knowledge_agent'], requireModelExists=True
            )
            print('Options for Knowledge Response Agent')
            self.knowledge_agent.opt.log()
        elif self.opts['init']['knowledge_response_model_path'] == 'oracle':
            self.knowledge_agent = OracleKnowledgeAgent(self.opts['init'])

    def _init_dialogue_models(self):
        ## Init dialogue models.

        # Initialize dialogue agent that uses the predicted knowledge.
        self.dialogue_agent = create_agent(
            self.opts['dialogue_agent'], requireModelExists=True
        )
        # Monkey patch the get_batch_context to ignore the knowledge for
        # beam-context-blocking.
        if self.opts['init']['beam_disregard_knowledge_for_context_blocking']:
            orig_fun = self.dialogue_agent._get_batch_context
            self.dialogue_agent._get_batch_context = MethodType(
                lambda self, batch: wow_get_batch_context(
                    self, batch, orig_fun=orig_fun
                ),
                self.dialogue_agent,
            )

        print('Options for Dialogue Response Agent')
        self.dialogue_agent.opt.log()

        # Initialize dialogue agent that doesn't use knowledge.
        if self.has_no_knowledge_dialogue_model:
            self.dialogue_agent_no_knowledge = create_agent(
                self.opts['dialogue_agent_no_knowledge'], requireModelExists=True
            )

        # Initialize dialogue agent that uses RAG with Wiki.
        if self.has_rag_wiki_dialogue_model:
            self.dialogue_agent_rag_wiki = create_agent(
                self.opts['dialogue_agent_rag_wiki'], requireModelExists=True
            )

    def dialogue_reply(self, agent, observation):
        return self.batch_dialogue_reply(agent, [observation])[0]

    def batch_dialogue_reply(self, agent, observations):
        dialogue_observations = []
        # Observation for the dialogue model.
        for obs in observations:
            dialogue_observation = agent.observe(obs)
            agent.self_observe(dialogue_observation)
            dialogue_observations.append(dialogue_observation)

        return agent.batch_act(dialogue_observations)

    def generate_knowledge_observation(self, knowledges: List[str], observations):
        # Adjust the observation texts.
        knowledge_infused_observations = deepcopy(observations)
        for obs, knowledge in zip(knowledge_infused_observations, knowledges):
            if 'text' not in obs:
                continue
            text = obs.pop('text')

            if self.opts['init']['add_fixed_confidence'] >= 0:
                confidence = self.opts['init']['add_fixed_confidence']
                if self.opts['init']['add_confidence_as_str']:
                    confidence = {
                        0: 'low',
                        5: 'medium',
                        10: 'high',
                    }[confidence] + ' confidence'
                text += f'\n{TOKEN_KNOWLEDGE} {confidence}: {knowledge} {TOKEN_END_KNOWLEDGE}'
            else:
                text += f'\n{TOKEN_KNOWLEDGE} {knowledge} {TOKEN_END_KNOWLEDGE}'
            obs['text'] = text
        return knowledge_infused_observations

    def batch_act(self, observations):
        knowledge_agent_observations = [o['knowledge_agent'] for o in observations]
        raw_observations = [o['raw'] for o in observations]

        # Get the knowledge replies.
        if self.knowledge_agent is None:
            self._init_knowledge_model()
        batch_reply_knowledge = self.knowledge_agent.batch_act(
            knowledge_agent_observations
        )

        if (
            'top_docs' in batch_reply_knowledge[0]
            and self.opts['init']['use_supporting_sentence_as_knowledge']
        ):
            # The knowledge agent is a rag-style model. Instead of the actual knowledge
            # response, we will use the best matching sentence from the retrieved docs
            # as the knowledge conditioning.
            for i, reply_knowledge in enumerate(batch_reply_knowledge):
                reply_knowledge['support_sentence'] = find_supporting_sentence(
                    question=raw_observations[i]['text'],
                    answer=reply_knowledge['text'],
                    docs=reply_knowledge['top_docs'],
                )

        if self.dialogue_agent is None:
            self._init_dialogue_models()

        if self.dialogue_agent_no_knowledge:
            batch_reply_dialogue_no_knowledge = self.batch_dialogue_reply(
                self.dialogue_agent_no_knowledge, raw_observations
            )
        if self.dialogue_agent_rag_wiki:
            batch_reply_dialogue_rag_wiki = self.batch_dialogue_reply(
                self.dialogue_agent_rag_wiki, raw_observations
            )

        knowledge_infused_observations = self.generate_knowledge_observation(
            knowledges=[
                reply_knowledge.get('text', '')
                for reply_knowledge in batch_reply_knowledge
            ],
            observations=raw_observations,
        )
        batch_reply_dialogue = self.batch_dialogue_reply(
            self.dialogue_agent, knowledge_infused_observations
        )

        batch_reply_dialogue_knowledge_sentence = None
        if (
            self.opts['init']['use_supporting_sentence_as_knowledge']
            and 'support_sentence' in batch_reply_knowledge[0]
        ):
            knowledge_sentence_infused_observations = (
                self.generate_knowledge_observation(
                    knowledges=[
                        reply_knowledge.get('support_sentence', '')
                        for reply_knowledge in batch_reply_knowledge
                    ],
                    observations=raw_observations,
                )
            )
            batch_reply_dialogue_knowledge_sentence = self.batch_dialogue_reply(
                self.dialogue_agent, knowledge_sentence_infused_observations
            )

        for i in range(len(batch_reply_dialogue)):
            if batch_reply_knowledge and len(batch_reply_knowledge) > i:
                batch_reply_dialogue[i]['knowledge_response'] = batch_reply_knowledge[
                    i
                ].get('text', '')
                if 'support_sentence' in batch_reply_knowledge[i]:
                    batch_reply_dialogue[i]['support_sentence'] = batch_reply_knowledge[
                        i
                    ].get('support_sentence', '')
            if (
                self.dialogue_agent_no_knowledge
                and batch_reply_dialogue_no_knowledge
                and len(batch_reply_dialogue_no_knowledge) > i
            ):
                batch_reply_dialogue[i][
                    'text_no_knowledge'
                ] = batch_reply_dialogue_no_knowledge[i].get('text', '')

            if (
                self.dialogue_agent_rag_wiki
                and batch_reply_dialogue_rag_wiki
                and len(batch_reply_dialogue_rag_wiki) > i
            ):
                batch_reply_dialogue[i][
                    'text_rag_wiki'
                ] = batch_reply_dialogue_rag_wiki[i].get('text', '')

            if (
                batch_reply_dialogue_knowledge_sentence
                and len(batch_reply_dialogue_knowledge_sentence) > i
            ):
                batch_reply_dialogue[i][
                    'text_knowledge_sentence'
                ] = batch_reply_dialogue_knowledge_sentence[i].get('text', '')

        [
            self._filter_beams(
                reply=reply,
                filter_for_knowledge=self.opts['init'][
                    'beam_filter_for_knowledge_response'
                ],
                filter_questions=self.opts['init']['beam_filter_questions'],
                filter_self_references=self.opts['init']['beam_filter_self_references'],
            )
            for reply in batch_reply_dialogue
        ]

        return batch_reply_dialogue

    def _filter_beams(
        self,
        reply,
        filter_for_knowledge: bool = True,
        filter_questions: bool = False,
        filter_self_references: bool = False,
    ):
        knowledge = normalize_answer(reply['knowledge_response'])
        self_references = [
            'I live',
            'I love',
            ' me ',
            'my favorite',
            'My favorite',
            'do you know',
            'I have',
            'I like ',
            'My ',
        ]
        question_words = [
            'who',
            'when',
            'where',
            'what',
            'do you',
            'are you',
        ]

        def filter_fn(text: str) -> bool:
            normalized_text = normalize_answer(text)
            if filter_for_knowledge and knowledge not in normalized_text:
                return False
            if filter_questions and (
                '?' in text or any([qw in normalized_text for qw in question_words])
            ):
                return False
            if filter_self_references and any([ref in text for ref in self_references]):
                return False
            return True

        if not (
            'text' in reply
            and 'beam_texts' in reply
            and 'knowledge_response' in reply
            and len(reply['beam_texts']) > 1
        ):
            return

        beam_texts = [
            text
            for text, _ in sorted(reply['beam_texts'], key=lambda x: x[1], reverse=True)
        ]
        # print('\t' + '\n\t'.join(beam_texts[:10]))
        for text in [reply['text']] + beam_texts:
            if filter_fn(text):
                reply.force_set('text', text)
                return

    def self_observe(self, self_message: Message) -> None:
        # Hack: Feed back the final dialogue response to the knowledge model.
        # This is why we need to make sure that --mutators flatten.
        self.knowledge_agent.self_observe(self_message)

    def observe(self, observation):
        # Delete unused keys.
        for key in ['label_candidates', 'knowledge']:
            if key in observation:
                del observation[key]

        label_key = 'eval_labels' if 'eval_labels' in observation else 'labels'
        if 'nqopen' in self.opts['init']['task'].lower():
            knowledge_target = observation.get(label_key, '')
            if isinstance(knowledge_target, tuple):
                knowledge_target = '\t'.join(knowledge_target)
            observation['knowledge_target'] = knowledge_target
            observation['dialogue_response'] = ''
        else:
            observation['dialogue_response'] = observation.get(label_key, '')
            observation['knowledge_response'] = observation.get('checked_sentence', '')

        if self.knowledge_agent is None:
            self._init_knowledge_model()

        observations = {
            'raw': deepcopy(observation),
            'knowledge_agent': self.knowledge_agent.observe(observation),
        }
        self.observations = observations
        return observations

    def act(self):
        """
        Call batch_act with the singleton batch.
        """
        response = self.batch_act([self.observations])[0]
        self.self_observe(response)
        return response

class OracleKnowledgeAgent(Agent):
    def __init__(self, opt, shared=None):
        super().__init__(opt)
        self.id = 'OracleKnowledgeAgent'

    def get_knowledge(self, obs):
        labels_kword = 'labels' if 'train' in self.opt['datatype'] else 'eval_labels'
        if 'wizardofwikipedia' in self.opt['task'].lower().replace('_', ''):
            return obs.get('checked_sentence', '')
        if 'wizardofinternet' in self.opt['task'].lower().replace('_', ''):
            knowledge = obs.get('__selected-sentences__', '')
            if isinstance(knowledge, list):
                knowledge = '\n'.join(knowledge)
            return knowledge
        elif (
            'nqopen' in self.opt['task'].lower()
            or 'natural_questions' in self.opt['task'].lower()
        ):
            return obs.get(labels_kword, '')
        elif 'LightTeacherPlus' in self.opt['task']:
            if labels_kword not in obs or not obs[labels_kword]:
                return ''
            labels = obs[labels_kword]
            if not isinstance(labels, str):
                labels = labels[0]
            return knowledge_from_dialogue_response(labels)
        elif 'SummaryQA' in self.opt['task']:
            return obs.get(labels_kword, '')
        else:
            raise NotImplementedError(f'Task "{self.opt["task"]}" is not known.')

    def self_observe(self, obs):
        pass

    def batch_act(self, observations):
        return [self.act(obs) for obs in observations]

    def act(self, obs=None):
        if not obs:
            obs = self.observation
        if obs is None:
            return {'text': 'Nothing to repeat yet.'}
        reply = {}
        reply['id'] = self.getID()
        knowledge = self.get_knowledge(obs)
        if not isinstance(knowledge, str):
            knowledge = random.choice(knowledge)
        reply['text'] = knowledge
        return Message(reply)


import time
import nlp_pre_processing
nlp = nlp_pre_processing.NLPPreprocessor()

# Multiprocessing
from multiprocessing import Pool

start_time = time.time()

num_partitions = 20
num_cores = 15

print(f'Partition Number: {num_partitions} - Number of Cores: {num_cores}...')

def main_process_pipeline(df, func):
    df_split = np.array_split(df, num_partitions)
    pool = Pool(num_cores)
    df = pd.concat(pool.map(func, df_split))
    pool.close()
    pool.join()
    return df

def pre_process_wrapper(df):
    df['full_text'] = df['full_text'].apply(lambda text: nlp.pre_processing_pipeline(text))
    return df

print('Generate dump with pre-processed data...')
extraction_df = main_process_pipeline(df_sr_events_requests, pre_process_wrapper)

print(f'Pre-Processing in seconds: {(time.time() - start_time)}')
print(f'Pre-Processing in seconds: {(time.time() - start_time)}')
print('Multiprocessing data cleanup done...')


import requests
import googletrans
from pycatapi import Client

def load_random_cat():
    c = Client()
    cat_url = c.get_cat()
    return cat_url

def load_random_joke():
    url = "https://official-joke-api.appspot.com/random_joke"

    response = requests.get(url).json()
    joke = f'- {response["setup"]}\n- {response["punchline"]}\n:)'
    return translate_joke(joke)

def translate_joke(text):
    translator = googletrans.Translator()
    result = translator.translate(text, dest="ru")
    row = (f'{result.text}\nOrigin: {result.origin}').split('\n')
    return row

def base_config():
    return {
        "cat_url": load_random_cat(),
        "joke_text": load_random_joke()
    }


import os
import unittest
import ansible

from ansiblelint import Runner, RulesCollection
from pkg_resources import parse_version

class TestTaskIncludes(unittest.TestCase):

    def setUp(self):
        rulesdir = os.path.join('lib', 'ansiblelint', 'rules')
        self.rules = RulesCollection.create_from_directory(rulesdir)

    def test_block_included_tasks(self):
        filename = 'test/blockincludes.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 4)

    def test_block_included_tasks_with_rescue_and_always(self):
        filename = 'test/blockincludes2.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 4)

    def test_included_tasks(self):
        filename = 'test/taskincludes.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 4)

    @unittest.skipIf(parse_version(ansible.__version__) < parse_version('2.4'), "not supported with ansible < 2.4")
    def test_include_tasks_2_4_style(self):
        filename = 'test/taskincludes_2_4_style.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 4)

    @unittest.skipIf(parse_version(ansible.__version__) < parse_version('2.4'), "not supported with ansible < 2.4")
    def test_import_tasks_2_4_style(self):
        filename = 'test/taskimports.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 4)

    def test_include_tasks_with_block_include(self):
        filename = 'test/include-in-block.yml'
        runner = Runner(self.rules, filename, [], [], [])
        runner.run()
        self.assertEqual(len(runner.playbooks), 3)


import os, sys
import argparse

libpath = os.path.join(os.path.dirname(__file__), '../')
sys.path.append(libpath)
import src

from src.compiler.gen_relay_ir import gen_relay_ir
from src.compiler.visualize import visualize
from src.compiler.compiler import Compiler
from src.compiler.utils import dump_params

def frontendcompile(args):
    platform = args.platform
    model_path = args.model_path
    epoch = args.epoch
    output_path = args.output_path
    dump_params = args.dump_params
    img_shape = args.img_shape
    reconstruct_graph_name = args.reconstruct_graph_name
    reconstruct_params_name = args.reconstruct_params_name
    compiler_params_name = args.compiler_params_name
    compiler_graph_name = args.compiler_graph_name

    if not os.path.isdir(output_path):
        os.mkdir(output_path)

    model = gen_relay_ir(platform, model_path, img_shape, epoch)
    graph, params = visualize(model)

    tran_compiler = Compiler(graph, params)

    # Generate Moffett IR for reconstruct
    if reconstruct_graph_name and reconstruct_params_name:
        tran_compiler.save(reconstruct_graph_name, reconstruct_params_name, output_path)

    # Generate operation fused graph.
    if compiler_graph_name and compiler_params_name:
        tran_compiler.compile()
        # tran_compiler.CModel_transforms()
        tran_compiler.save(compiler_graph_name, compiler_params_name, output_path)
        if dump_params:
            dump_params(tran_compiler.params, compiler_params_name, output_path)
    print('Done!')

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser(usage="main.py platform model_path [optional]\ni.e.: pyhton3 main.py mxnet resnet50_v1d --epoch 159")
    arg_parser.add_argument("platform", type=str, help='model platform, i.e.: mxnet, tensorflow...')
    arg_parser.add_argument("model_path", type=str, help='path of model file, For mxnet model, prefix of model name\
                                                          (symbol will be loaded from prefix-symbol.json, parameters will be loaded from prefix-epoch.params.)\
                                                          For tensorflow model, *.pb')
    arg_parser.add_argument("--epoch", default = 0, type=int, help='Only used for mxnet model. epoch number of mxnet model we would like to load, default: 0.')
    arg_parser.add_argument("--output_path", default = "./moffett_ir", type=str, help='output directory of Moffett IR graphs and params, default: ./moffett_ir')
    arg_parser.add_argument("--dump_params", default=False, type=bool, help="Whether dump params to separate files.")
    arg_parser.add_argument("--img_shape", default=[224,224], nargs=2, type=int, help="The image shape of input in model, default: 224 224")
    arg_parser.add_argument("--reconstruct_graph_name", default = 'IR_for_reconstruct_graph', type=str, help='file name of Moffett IR for reconstruct graph, default: IR_for_reconstruct_graph')
    arg_parser.add_argument("--reconstruct_params_name", default = 'IR_for_reconstruct_params', type=str, help='file name of Moffett IR for reconstruct params, default: IR_for_reconstruct_params')
    arg_parser.add_argument("--compiler_graph_name", default='IR_fused_for_CModel_graph', type=str, help='file name of Moffett IR fused for CModel graph, default: IR_fused_for_CModel_graph')
    arg_parser.add_argument("--compiler_params_name", default = 'IR_fused_for_CModel_params', type=str, help='file name of Moffett IR fused for CModel params, default: IR_fused_for_CModel_params')
    args = arg_parser.parse_args()

    frontendcompile(args)


import functools
import torch.nn as nn

import os
import sys
import time
import random
import argparse
import torch

__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
           'resnet152']

def conv3x3(in_planes, out_planes, stride=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=False)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, norm_layer=None,
                 bn_eps=1e-5, bn_momentum=0.1, downsample=None, inplace=True):
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = norm_layer(planes, eps=bn_eps, momentum=bn_momentum)
        self.relu = nn.ReLU(inplace=inplace)
        self.relu_inplace = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = norm_layer(planes, eps=bn_eps, momentum=bn_momentum)
        self.downsample = downsample
        self.stride = stride
        self.inplace = inplace

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        if self.inplace:
            out += residual
        else:
            out = out + residual

        out = self.relu_inplace(out)

        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1,
                 norm_layer=None, bn_eps=1e-5, bn_momentum=0.1,
                 downsample=None, inplace=True):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = norm_layer(planes, eps=bn_eps, momentum=bn_momentum)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = norm_layer(planes, eps=bn_eps, momentum=bn_momentum)
        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
                               bias=False)
        self.bn3 = norm_layer(planes * self.expansion, eps=bn_eps,
                              momentum=bn_momentum)
        self.relu = nn.ReLU(inplace=inplace)
        self.relu_inplace = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride
        self.inplace = inplace

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        if self.inplace:
            out += residual
        else:
            out = out + residual
        out = self.relu_inplace(out)

        return out

class ResNet(nn.Module):

    def __init__(self, block, layers, in_channels=3, norm_layer=nn.BatchNorm2d, bn_eps=1e-5,
                 bn_momentum=0.1, deep_stem=False, stem_width=32, inplace=True):
        self.inplanes = stem_width * 2 if deep_stem else 64
        super(ResNet, self).__init__()
        if deep_stem:
            self.conv1 = nn.Sequential(
                nn.Conv2d(in_channels, stem_width, kernel_size=3, stride=2, padding=1,
                          bias=False),
                norm_layer(stem_width, eps=bn_eps, momentum=bn_momentum),
                nn.ReLU(inplace=inplace),
                nn.Conv2d(stem_width, stem_width, kernel_size=3, stride=1,
                          padding=1,
                          bias=False),
                norm_layer(stem_width, eps=bn_eps, momentum=bn_momentum),
                nn.ReLU(inplace=inplace),
                nn.Conv2d(stem_width, stem_width * 2, kernel_size=3, stride=1,
                          padding=1,
                          bias=False),
            )
        else:
            self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=7, stride=2, padding=3,
                                   bias=False)

        self.bn1 = norm_layer(stem_width * 2 if deep_stem else 64, eps=bn_eps,
                              momentum=bn_momentum)
        self.relu = nn.ReLU(inplace=inplace)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, norm_layer, 64, layers[0],
                                       inplace,
                                       bn_eps=bn_eps, bn_momentum=bn_momentum)
        self.layer2 = self._make_layer(block, norm_layer, 128, layers[1],
                                       inplace, stride=2,
                                       bn_eps=bn_eps, bn_momentum=bn_momentum)
        self.layer3 = self._make_layer(block, norm_layer, 256, layers[2],
                                       inplace, stride=2,
                                       bn_eps=bn_eps, bn_momentum=bn_momentum)
        self.layer4 = self._make_layer(block, norm_layer, 512, layers[3],
                                       inplace, stride=2,
                                       bn_eps=bn_eps, bn_momentum=bn_momentum)

    def _make_layer(self, block, norm_layer, planes, blocks, inplace=True,
                    stride=1, bn_eps=1e-5, bn_momentum=0.1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                norm_layer(planes * block.expansion, eps=bn_eps,
                           momentum=bn_momentum),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, norm_layer, bn_eps,
                            bn_momentum, downsample, inplace))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes,
                                norm_layer=norm_layer, bn_eps=bn_eps,
                                bn_momentum=bn_momentum, inplace=inplace))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        blocks = []
        x = self.layer1(x);
        blocks.append(x)
        x = self.layer2(x);
        blocks.append(x)
        x = self.layer3(x);
        blocks.append(x)
        x = self.layer4(x);
        blocks.append(x)

        return blocks

def load_model(model, model_file, is_restore=False):
    t_start = time.time()

    if model_file is None:
        return model

    # get the pretrained model dict
    if isinstance(model_file, str):
        state_dict = torch.load(model_file)
        if 'model' in state_dict.keys():
            state_dict = state_dict['model']
    else:
        state_dict = model_file
    t_ioend = time.time()

    # get the model dict
    model_dict = model.state_dict()

    # ini the corresponding layer
    i = 0
    j = 0
    # with open('./before.txt', 'wt') as f:
    #     print(model_dict, file=f)
    for k, v in state_dict.items():
        if k in model_dict.keys():
            if v.size() == model_dict[k].size():
                model_dict[k] = state_dict[k]
                i = i + 1
            j = j + 1
    print('total weight is',j)
    print('using weight is',i)
    # with open('./after.txt', 'wt') as f:
    #     print(model_dict, file=f)

    model.load_state_dict(model_dict, strict=False)
    ckpt_keys = set(state_dict.keys())
    own_keys = set(model.state_dict().keys())
    missing_keys = own_keys - ckpt_keys
    unexpected_keys = ckpt_keys - own_keys

    del state_dict
    t_end = time.time()
    print(
        "Load model, Time usage:\n\tIO: {}, initialize parameters: {}".format(
            t_ioend - t_start, t_end - t_ioend))

    return model

def resnet18(pretrained_model=None, **kwargs):
    model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)

    if pretrained_model is not None:
        model = load_model(model, pretrained_model)
    return model

def resnet34(pretrained_model=None, **kwargs):
    model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)

    if pretrained_model is not None:
        model = load_model(model, pretrained_model)
    return model

def resnet50(pretrained_model=None, **kwargs):
    model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)

    if pretrained_model is not None:
        model = load_model(model, pretrained_model)
    return model

def resnet101(pretrained_model=None, **kwargs):
    model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)

    if pretrained_model is not None:
        model = load_model(model, pretrained_model)
    return model

def resnet152(pretrained_model=None, **kwargs):
    model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)

    if pretrained_model is not None:
        model = load_model(model, pretrained_model)
    return model


import pathlib

#path to the desktop
desktop = pathlib.Path("/home/symon_kipkemei")

#create a new folder folder
new_path = pathlib.Path("/home/symon_kipkemei/screenshots")
new_path.mkdir(exist_ok=True)

# list items in desktop
for filepath in desktop.iterdir():
    #filter screenshots only
    if filepath.suffix == ".db":
        # create new path for each file
        new_file_path = new_path.joinpath(filepath.name)

        # move the screenshot there
        filepath.replace(new_file_path)