import sys
import time
import argparse

# Count the number of words occurence in the input file
def get_words_count(filename, sortby):
    with open(filename, encoding='utf-8') as f:
        words_count = {}

        # we read the file by line
        for line in f:
            words = line.split() # we split the line by whitespace

            for w in words:
                if w in words_count:
                    words_count[w] += 1
                else:
                    words_count[w] = 1

        if sortby == 'alphabet': # to sort by alphabetical order
            words_count = dict(sorted(words_count.items()))
        elif sortby == 'occurences': # to sort by number of occurences and then alpabetical
            words_count = dict(sorted(words_count.items(), key=lambda x: (-x[1], x[0])))

    return words_count

# This function allows us to easilly parse our arguments
def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("-s", "--sortby", "-sortby", default='occurences', nargs=1,
                        help="""To sort the results by alphabetical order (alphabet), by occurences (occurences), or no sorting (none)""")
    parser.add_argument('filename', metavar='filename', help='The file to analyse')
    return parser.parse_args()

# Main function
# parses line argument to get the input file passed as argument
def main():
    # we parse arguments
    args = parse_arguments()

    # we extrac the arguments
    filename = args.filename
    option = args.sortby[0].lower()

    # to sort by alphabetical order
    if option in ['alphabetical','alphabet', 'alpha', 'a']:
         sortby = 'alphabet'
    # to sort by number of occurences and then alpabetical
    elif option in ['occurences','occurence', 'occ', 'o']:
         sortby = 'occurences'
     # to specify that we don't want any sorting
    elif option in ['none', 'n']:
         sortby = 'none'
    else:
         print('unknown sort option: \'' + option +'\' use \'alphabet\', \'occurences\' or \'none\'')
         sys.exit(1)

    # we start a timer
    start_time = time.time()
    words_count =  get_words_count(filename, sortby)

    # we calculate the execution time before the print
    total_time = (time.time() - start_time)

    # we calculate and print the execution time
    print('\033[92mCOUNT DONE\033[0m in: {:.2f} s'.format(total_time))

    # we print every words
    # for w in words_count:
    #     print('{} {}'.format(w, words_count[w]))

    # we write the resulats in a file
    with open('output.txt', 'w') as f:
        for (w, c) in words_count.items():
            f.writelines('{} {}\n'.format(w, c))

    print('Results written into output.txt')

if __name__ == '__main__':
    main()


import os
import requests
from datetime import datetime
import json

GENDER = "male"
WEIGHT = 96.16
HEIGHT = 175.26
AGE = 25

NUTRI_APP_ID = os.getenv('NUTRI_APP_ID')
NUTRI_API_KEY = os.getenv('NUTRI_API_KEY')
nutri_exercise_endpoint = "https://trackapi.nutritionix.com/v2/natural/exercise"

""" POST NUTRI API TO GET EXERCISE CALORIES """
query_info = input("Tell me which exercises you did: ")

headers = {
    "x-app-id": NUTRI_APP_ID,
    "x-app-key": NUTRI_API_KEY,
}

params = {
 "query": query_info,
 "gender":GENDER,
 "weight_kg": WEIGHT,
 "height_cm": HEIGHT,
 "age": AGE
}

response = requests.post(url=nutri_exercise_endpoint, json=params, headers=headers) # post requests need the `json` keyword in the function and NOT `params` for you to have success
# response.raise_for_status()
exercise_data = response.json()
print("   ")
print("   ")
print(json.dumps(exercise_data, indent=4))
print("   ")
print("   ")

""" POST TO SHEETY API TO ADD EXERCISE TO YOUR GOOGLE SHEET """
sheety_post_endpoint = "https://api.sheety.co/82b93e3605e19854564fe47a3c8e3f6d/workoutTracking/workouts"

for exercise in exercise_data['exercises']:
    sheety_params = {
        "workout": {
            "date": datetime.now().strftime("%m/%d/%y"),
            "time": datetime.now().strftime("%I:%M:%S %p"),
            "exercise": exercise['name'].title(),
            "duration (mins)": f"{exercise['duration_min']}",
            "calories": exercise['nf_calories']
        }
    }

    sheety_headers = {
        "Authorization": f"Basic {os.getenv('SHEETY_AUTH')}"
    }

    sheety_response = requests.post(sheety_post_endpoint, json=sheety_params, headers=sheety_headers)
    print("   ")
    print("   ")
    print(sheety_response.json())


from flask import Blueprint, request

from app.validation import Validation

module = Blueprint('delivery', __name__)
validation = Validation()

@module.route('/', methods=['GET'])
def index():
    return 'ok'

@module.route('/couriers', methods=['POST'])
def couriers():
    request_couriers = request.json
    status, bad_id, good_courier = validation.check_courier(request_couriers)
    if not status:
        return {'validation_error': {'couriers': bad_id}}, 400
    return {'status': 'ok'}, 200


#!/usr/bin/env python

import cv2
import rospy
import platform
import numpy as np
from cv_bridge import CvBridge, CvBridgeError
from sensor_msgs.msg import Image
from std_msgs.msg import String

class camera_feed(object):

	def __init__(self):

		# ROS Node
		hostname =platform.node()
		node_name = hostname + "_camera_feed"
		rospy.init_node(node_name, anonymous=False, log_level=rospy.DEBUG)
		self.rate = rospy.Rate(30)

		# Videofeed
		self.vid = cv2.VideoCapture(0)
		self.bridge = CvBridge()

		# ROS Topic
		self.pub_image = rospy.Publisher("camera_feed", Image, queue_size=1)

	def loop(self):
		""" Main Camera Loop """

		ret, frame = self.vid.read()

		frame = cv2.resize(frame,(200,200))

		msg = self.bridge.cv2_to_imgmsg(frame, "bgr8")
		self.pub_image.publish(msg)

if __name__ == "__main__":
	feed = camera_feed()
	while not rospy.is_shutdown():
		feed.loop()
	cap.release()


from typing import List, Optional, Sequence, Union

import torch
import torch as to
from torch.distributions.bernoulli import Bernoulli
from torch.distributions.multivariate_normal import MultivariateNormal
from torch.distributions.normal import Normal
from torch.distributions.uniform import Uniform

import pyrado
from pyrado.utils.bijective_transformation import BijectiveTransformation, SqrtTransformation
from pyrado.utils.input_output import print_cbt

class DomainParam:
    """Class to store and manage (probably multiple) domain parameter a.k.a. physics parameter a.k.a. simulator parameter"""

    def __init__(
        self,
        name: Union[str, List[str]],
        clip_lo: Optional[Union[int, float]] = -pyrado.inf,
        clip_up: Optional[Union[int, float]] = pyrado.inf,
        roundint: bool = False,
    ):
        """
        Constructor, also see the constructor of DomainRandomizer.

        :param name: name of the parameter; can be a list of names if this domain parameters supports multiple
                     parameters, e.g., if the distribution respects correlations; this is usually not the case and the
                     only known subclass with this behavior is :py:class:`SelfPacedDomainParam`
        :param clip_lo: lower value for clipping
        :param clip_up: upper value for clipping
        :param roundint: flags if the parameters should be rounded and converted to an integer
        """
        if not isinstance(name, (str, list)):
            raise pyrado.TypeErr(given=name, expected_type=(str, list))

        self.name = name
        self.mean = None  # to be set by subclass
        self.clip_lo = clip_lo
        self.clip_up = clip_up
        self.roundint = roundint
        self.distr = None  # no randomization by default

    def __eq__(self, other):
        """Check if two `DomainParam` are equal by comparing all attributes defined in `get_field_names()`."""
        if not isinstance(other, DomainParam):
            raise pyrado.TypeErr(given=other, expected_type=DomainParam)

        for fn in self.get_field_names():
            if getattr(self, fn) != getattr(other, fn):
                return False
        return True

    def get_field_names(self) -> List[str]:
        """Get union of all hyper-parameters of all domain parameter distributions."""
        return ["name", "clip_lo", "clip_up", "roundint"]

    def adapt(self, domain_distr_param: str, domain_distr_param_value: Union[float, int, to.Tensor]):
        """
        Update this domain parameter.

        .. note::
            This function should by called by the subclasses' `adapt()` function.

        :param domain_distr_param: distribution parameter to update, e.g. mean or std
        :param domain_distr_param_value: new value of the distribution parameter
        """
        if domain_distr_param not in self.get_field_names():
            raise pyrado.KeyErr(
                msg=f"The domain parameter {self.name} does not have a domain distribution parameter "
                f"called {domain_distr_param}!"
            )
        setattr(self, domain_distr_param, domain_distr_param_value)

    def sample(self, num_samples: int = 1) -> Union[List[to.Tensor], List[List[to.Tensor]]]:
        """
        Generate new domain parameter values.

        :param num_samples: number of samples (sets of new parameter values)
        :return: list of tensors containing the new parameter values or list of list of tensors containing the new
                 values for each parameter separately; the outer list is for different parameters, the inner list is for
                 the number of samples
        """
        if not isinstance(num_samples, int):
            raise pyrado.TypeErr(given=num_samples, expected_type=int)
        if num_samples <= 0:
            raise pyrado.ValueErr(given=num_samples, g_constraint="0")

        if self.distr is None:
            raise RuntimeError("Trying to sample a domain parameter without a specified distribution!")

        # Draw num_samples samples (rsample is not implemented for Bernoulli)
        sample_tensor = self.distr.sample(sample_shape=to.Size([num_samples]))

        # Clip the values
        sample_tensor = to.clamp(sample_tensor, self.clip_lo, self.clip_up)

        # Round values to integers if desired
        if self.roundint:
            sample_tensor = to.round(sample_tensor).to(to.int32)

        # Convert the large tensor into a list of small tensors
        return list(sample_tensor)

class UniformDomainParam(DomainParam):
    """Domain parameter sampled from a normal distribution"""

    def __init__(self, mean: Union[int, float, to.Tensor], halfspan: Union[int, float, to.Tensor], **kwargs):
        """
        Constructor

        :param mean: nominal parameter value
        :param halfspan: half interval
        :param kwargs: forwarded to `DomainParam` constructor
        """
        super().__init__(**kwargs)

        self.mean = mean
        self.halfspan = halfspan
        self.distr = Uniform(self.mean - self.halfspan, self.mean + self.halfspan, validate_args=True)

    def get_field_names(self) -> List[str]:
        return ["name", "mean", "halfspan", "clip_lo", "clip_up", "roundint"]

    def adapt(self, domain_distr_param: str, domain_distr_param_value: Union[float, int]):
        # Set the attributes
        super().adapt(domain_distr_param, domain_distr_param_value)

        # Re-create the distribution, otherwise the changes will have no effect
        try:
            self.distr = Uniform(self.mean - self.halfspan, self.mean + self.halfspan, validate_args=True)
        except ValueError as err:
            print_cbt(
                f"Inputs that lead to the ValueError from PyTorch Distributions:"
                f"\ndomain_distr_param = {domain_distr_param}\n"
                f"low = {self.mean - self.halfspan}\nhigh = {self.mean + self.halfspan}"
            )
            raise err

class NormalDomainParam(DomainParam):
    """Domain parameter sampled from a normal distribution"""

    def __init__(self, mean: Union[int, float, to.Tensor], std: Union[int, float, to.Tensor], **kwargs):
        """
        Constructor

        :param mean: nominal parameter value
        :param std: standard deviation
        :param kwargs: forwarded to `DomainParam` constructor
        """
        super().__init__(**kwargs)

        self.mean = mean
        self.std = std
        self.distr = Normal(self.mean, self.std, validate_args=True)

    def get_field_names(self) -> List[str]:
        return ["name", "mean", "std", "clip_lo", "clip_up", "roundint"]

    def adapt(self, domain_distr_param: str, domain_distr_param_value: Union[float, int]):
        # Set the attributes
        super().adapt(domain_distr_param, domain_distr_param_value)

        # Re-create the distribution, otherwise the changes will have no effect
        try:
            self.distr = Normal(self.mean, self.std, validate_args=True)
        except ValueError as err:
            print_cbt(
                f"Inputs that lead to the ValueError from PyTorch Distributions:"
                f"\ndomain_distr_param = {domain_distr_param}\nloc = {self.mean}\nscale = {self.std}"
            )
            raise err

class MultivariateNormalDomainParam(DomainParam):
    """Domain parameter sampled from a normal distribution"""

    def __init__(self, mean: Union[int, float, to.Tensor], cov: Union[list, to.Tensor], **kwargs):
        """
        Constructor

        :param mean: nominal parameter value
        :param cov: covariance
        :param kwargs: forwarded to `DomainParam` constructor
        """
        super().__init__(**kwargs)

        self.mean = to.as_tensor(mean, dtype=to.get_default_dtype()).view(-1)
        self.cov = to.as_tensor(cov, dtype=to.get_default_dtype())
        if not self.cov.ndim == 2:
            raise pyrado.ShapeErr(msg="The covariance needs to be given as a matrix!")
        self.distr = MultivariateNormal(self.mean, self.cov, validate_args=True)

    def get_field_names(self) -> List[str]:
        return ["name", "mean", "cov", "clip_lo", "clip_up", "roundint"]

    def adapt(self, domain_distr_param: str, domain_distr_param_value: to.Tensor):
        if domain_distr_param == "cov" and domain_distr_param_value < 0:
            raise pyrado.ValueErr(given_name="cov", ge_constraint="0")

        # Set the attributes
        super().adapt(domain_distr_param, domain_distr_param_value)

        # Re-create the distribution, otherwise the changes will have no effect
        try:
            self.distr = MultivariateNormal(self.mean, self.cov, validate_args=True)
        except ValueError as err:
            print_cbt(
                f"Inputs that lead to the ValueError from PyTorch Distributions:"
                f"\ndomain_distr_param = {domain_distr_param}\nloc = {self.mean}\ncov = {self.cov}"
            )
            raise err

class BernoulliDomainParam(DomainParam):
    """Domain parameter sampled from a Bernoulli distribution"""

    def __init__(self, val_0: Union[int, float], val_1: Union[int, float], prob_1: float, **kwargs):
        """
        Constructor

        :param val_0: value of event 0
        :param val_1: value of event 1
        :param prob_1: probability of event 1, equals 1 - probability of event 0
        :param kwargs: forwarded to `DomainParam` constructor
        """
        super().__init__(**kwargs)

        self.val_0 = val_0
        self.val_1 = val_1
        self.prob_1 = prob_1
        self.mean = val_0 * (1 - prob_1) + val_1 * prob_1
        self.distr = Bernoulli(self.prob_1, validate_args=True)

    def get_field_names(self) -> List[str]:
        return ["name", "mean", "val_0", "val_1", "prob_1", "clip_lo", "clip_up", "roundint"]

    def adapt(self, domain_distr_param: str, domain_distr_param_value: Union[float, int]):
        # Set the attributes
        super().adapt(domain_distr_param, domain_distr_param_value)

        # Re-create the distribution, otherwise the changes will have no effect
        try:
            self.distr = Bernoulli(self.prob_1, validate_args=True)
        except ValueError as err:
            print_cbt(
                f"Inputs that lead to the ValueError from PyTorch Distributions:"
                f"\ndomain_distr_param = {domain_distr_param}\nprobs = {self.prob_1}"
            )
            raise err

    def sample(self, num_samples: int = 1) -> List[to.Tensor]:
        """
        Generate new domain parameter values.

        :param num_samples: number of samples (sets of new parameter values)
        :return: list of tensors containing the new parameter values
        """
        if not isinstance(num_samples, int):
            raise pyrado.TypeErr(given=num_samples, expected_type=int)
        if num_samples <= 0:
            raise pyrado.ValueErr(given=num_samples, g_constraint="0")

        if self.distr is None:
            raise RuntimeError("Trying to sample a domain parameter without a specified distribution!")

        # Draw num_samples samples (rsample is not implemented for Bernoulli)
        sample_tensor = self.distr.sample(sample_shape=to.Size([num_samples]))

        # Sample_tensor contains either 0 or 1
        sample_tensor = (to.ones_like(sample_tensor) - sample_tensor) * self.val_0 + sample_tensor * self.val_1

        # Clip the values
        sample_tensor = to.clamp(sample_tensor, self.clip_lo, self.clip_up)

        # Round values to integers if desired
        if self.roundint:
            sample_tensor = to.round(sample_tensor).to(to.int32)

        # Convert the large tensor into a list of small tensors
        return list(sample_tensor)

class SelfPacedDomainParam(DomainParam):
    def __init__(
        self,
        name: List[str],
        target_mean: to.Tensor,
        target_cov_flat: to.Tensor,
        init_mean: to.Tensor,
        init_cov_flat: to.Tensor,
        clip_lo: float,
        clip_up: float,
    ):
        """
        Constructor

        :param name: names of the parameters
        :param target_mean: target means of the contextual distribution
        :param target_cov_flat: target covariances of the contextual distribution
        :param init_mean: initial mean of the contextual distribution
        :param init_cov_flat: initial covariances of the contextual distribution
        :param clip_lo: lowest values to allow for sampling
        """

        if not isinstance(name, list):
            raise pyrado.TypeErr(given_name="name", expected_type=list, given=type(name))
        if not (target_mean.shape == init_mean.shape):
            raise pyrado.ShapeErr(msg="Target and init mean should have same shape!")
        if not (target_cov_flat.shape == init_cov_flat.shape):
            raise pyrado.ShapeErr(msg="Target and init standard deviations should have same shape!")

        super().__init__(name=name, clip_lo=clip_lo, clip_up=clip_up)

        self.target_mean = target_mean.double()
        self.target_cov_chol = target_cov_flat.double().sqrt().diag()
        self.context_mean = init_mean.double()
        self.context_cov_chol = init_cov_flat.double().sqrt().diag()

        self.init_mean = self.context_mean.detach().clone()
        self.init_cov_chol = self.context_cov_chol.detach().clone()

        self.dim = target_mean.shape[0]

        self._target_distr = MultivariateNormal(self.target_mean, self.target_cov, validate_args=True)
        self._context_distr = MultivariateNormal(self.context_mean, self.context_cov, validate_args=True)

    def get_field_names(self) -> Sequence[str]:
        """Get union of all hyper-parameters of all domain parameter distributions."""
        return ["context_mean", "context_cov_chol"]

    @staticmethod
    def make_broadening(
        name: List[str],
        mean: List[float],
        init_cov_portion: float = 0.001,
        target_cov_portion: float = 0.1,
        clip_lo: float = -pyrado.inf,
        clip_up: float = pyrado.inf,
    ) -> "SelfPacedDomainParam":
        """
        Creates a self-paced domain parameter having the same initial and target mean, but a larger variance on the
        target distribution.

        :param name: names of the parameters
        :param mean: means of the contextual distribution
        :param init_cov_portion: percentage of the mean the initial variance should be set to
        :param target_cov_portion: percentage of the mean the target variance should be set to
        :param clip_lo: lowest values to allow for sampling
        :param clip_up: highest values to allow for sampling
        """
        if not (len(mean) == len(name)):
            raise pyrado.ValueErr(msg="mean must have same length as names")
        mean = to.tensor(mean)
        return SelfPacedDomainParam(
            name=name,
            target_mean=mean,
            target_cov_flat=target_cov_portion * mean.abs(),
            init_mean=mean,
            init_cov_flat=init_cov_portion * mean.abs(),
            clip_lo=clip_lo,
            clip_up=clip_up,
        )

    @property
    def target_distr(self) -> MultivariateNormal:
        """Get the target distribution."""
        return self._target_distr

    @property
    def context_distr(self) -> MultivariateNormal:
        """Get the current contextual distribution."""
        return self._context_distr

    @property
    def target_cov(self) -> to.Tensor:
        """Get the target covariance matrix."""
        return self.target_cov_chol @ self.target_cov_chol.T

    @property
    def context_cov(self) -> to.Tensor:
        """Get the current covariance matrix."""
        return self.context_cov_chol @ self.context_cov_chol.T

    def adapt(self, domain_distr_param: str, domain_distr_param_value: to.Tensor):
        """
        Update this domain parameter.

        :param domain_distr_param: distribution parameter to update, e.g. mean or cov
        :param domain_distr_param_value: new value of the distribution parameter
        """

        # Set the attributes
        super().adapt(domain_distr_param, domain_distr_param_value)

        # Re-create the distributions, otherwise the changes will have no effect
        if domain_distr_param in ["context_mean", "context_cov_chol"]:
            try:
                self._context_distr = MultivariateNormal(self.context_mean, self.context_cov, validate_args=True)
            except ValueError as err:
                print_cbt(
                    f"Inputs that lead to the ValueError from PyTorch Distributions:\n"
                    f"init_distribution; domain_distr_param = {domain_distr_param}\nloc = {self.context_mean}\ncov = {self.context_cov}"
                )
                raise err

    def sample(self, num_samples: int = 1) -> List[to.tensor]:
        """Samples `num_samples` samples of the current context distribution."""

        if not (num_samples > 0):
            raise pyrado.ValueErr(given_name="num_samples", g_constraint=0)

        samples = self._context_distr.sample((num_samples,))
        valid = torch.logical_and(self.clip_lo <= samples, samples <= self.clip_up)
        while not valid.all():
            resamples = self._context_distr.sample((num_samples,))
            mask = torch.logical_not(valid)
            samples[mask] = resamples[mask]
            valid = torch.logical_and(self.clip_lo <= samples, samples <= self.clip_up)
        return list([list(sample) for sample in samples.T])


import os
import yaml
import re
import numpy
from uuv_cost_function import CostFunction
from .utils import init_logger, parse_param_input, SIMULATION_LOGGER

class OptConfiguration(object):
    CONFIG = None

    def __init__(self, input_data):
        if isinstance(input_data, str):
            assert os.path.isfile(input_data)

            self.opt_config_filename = input_data
            with open(self.opt_config_filename, 'r') as opt_config_file:
                self._opt_config = yaml.load(opt_config_file)
        elif isinstance(input_data, dict):
            self._opt_config = input_data

        assert 'cost_fcn' in self._opt_config, 'Cost function configuration available'

        if 'parameters' in self._opt_config:
            self.parameters = self._opt_config['parameters']
        else:
            self.parameters = None

        if 'max_num_processes' not in self._opt_config:
            self.max_num_processes = 2
        else:
            self.max_num_processes = self._opt_config['max_num_processes']

        assert self.max_num_processes > 0, \
            'Maximum number of simulation processes must be greater than zero'

        SIMULATION_LOGGER.info('Max. number of processes=%d' % self.max_num_processes)

        if 'log_filename' not in self._opt_config:
            self._log_filename = None
        else:
            self._log_filename = self._opt_config['log_filename']

        init_logger(self._log_filename)

        task = self._opt_config['task']

        SIMULATION_LOGGER.info('Task(s) input=' + str(self._opt_config['task']))
        if isinstance(task, list):
            SIMULATION_LOGGER.info('Multiple tasks found:')
            for t in task:
                SIMULATION_LOGGER.info('\t - %s' % t)

            self.tasks = task
        else:
            if os.path.isfile(task):
                SIMULATION_LOGGER.info('Retrieving filename for task function=' + task)
                self.tasks = [task]
            elif os.path.isdir(task):
                self.tasks = list()
                for f in os.listdir(task):
                    if '.yml' in f or '.yaml' in f:
                        self.tasks.append(os.path.join(task, f))

        atoi = lambda a: int(a) if a.isdigit() else a
        natural_keys = lambda text: [atoi(c) for c in re.split('(\d+)', text)]

        self.tasks.sort(key=natural_keys)
        SIMULATION_LOGGER.info('Task files=')
        for task in self.tasks:
            SIMULATION_LOGGER.info('\t - ' + task)

        self.results_dir = self._opt_config['output_dir']
        SIMULATION_LOGGER.info(
            'Retrieving output directory for partial results=' + self.results_dir)

        self.record_all = False

        if 'store_all_results' in self._opt_config:
            self.record_all = self._opt_config['store_all_results']

        SIMULATION_LOGGER.info('Record all partial results? ' + str(self.record_all))

        if 'store_kpis_only' in self._opt_config:
            self.store_kpis_only = self._opt_config['store_kpis_only']
        else:
            self.store_kpis_only = True

        self.evaluation_time_offset = 0

        if 'evaluation_time_offset' in self._opt_config:
            self.evaluation_time_offset = self._opt_config['evaluation_time_offset']

        assert self.evaluation_time_offset >= 0

        self.constraints = None
        self.cost_fcn = None

        self.params = None

        self.tasks_eval_fcn = 'numpy.mean(%s)'

        if 'task_eval_fcn' in self._opt_config:
            self.tasks_eval_fcn = self._opt_config['task_eval_fcn']

        if 'cost_fcn' in self._opt_config:
            SIMULATION_LOGGER.info('Initializing cost function')
            self.cost_fcn = CostFunction()
            if isinstance(self._opt_config['cost_fcn'], dict):
                cf = self._opt_config['cost_fcn']
                SIMULATION_LOGGER.info('Cost function imported from list')
            elif isinstance(self._opt_config['cost_fcn'], str):
                assert os.path.isfile(self._opt_config['cost_fcn']), 'Cost function file is invalid'
                assert '.yml' in self._opt_config['cost_fcn'] or '.yaml' in self._opt_config['cost_fcn']

                with open(self._opt_config['cost_fcn']) as cf_file:
                    cf = yaml.load(cf_file)

                SIMULATION_LOGGER.info(cf)

                SIMULATION_LOGGER.info('Cost function loaded from file, filename=' + self._opt_config['cost_fcn'])
            else:
                SIMULATION_LOGGER.error('Invalid input cost function')
                SIMULATION_LOGGER.error(self._opt_config['cost_fcn'])
                raise Exception('Invalid input cost function')

            self.cost_fcn.from_dict(cf)

        if 'cost_fcn_norm' in self._opt_config:
            self.cost_fcn.set_norm(self._opt_config['cost_fcn_norm'])

        SIMULATION_LOGGER.info('Cost function norm=' + str(self.cost_fcn.norm))

        if 'constraints' in self._opt_config:
            self.constraints = self._opt_config['constraints']
            if isinstance(self.constraints, list):
                self.cost_fcn.add_constraints(self._opt_config['constraints'])
                SIMULATION_LOGGER.info('Constraints imported from list')
            elif isinstance(self._opt_config['constraints'], str):
                assert os.path.isfile(self._opt_config['constraints']), 'Constraint file is invalid'
                assert '.yml' in self._opt_config['constraints'] or '.yaml' in self._opt_config['constraints']

                with open(self._opt_config['constraints']) as c_file:
                    constraints = yaml.load(c_file)

                self.cost_fcn.add_constraints(constraints)
                SIMULATION_LOGGER.info('Constraints loaded from file, filename=' + self._opt_config['constraints'])
            else:
                SIMULATION_LOGGER.error('Invalid input constraints list')
                raise Exception('Invalid input constraints list')

    @staticmethod
    def get_instance(input_data=None):
        if input_data is None:
            assert OptConfiguration.CONFIG is not None
            return OptConfiguration.CONFIG
        else:
            OptConfiguration.CONFIG = OptConfiguration(input_data)
            return OptConfiguration.CONFIG

    def get_constraint_tags(self):
        if self.cost_fcn is None:
            return None
        return self.cost_fcn.get_constraint_tags()

    def parse_input(self, args):
        assert 'input_map' in self._opt_config, 'Input parameter map is not available'
        self.params = parse_param_input(args, self._opt_config['input_map'])

    def print_params(self):
        if self.params is None:
            SIMULATION_LOGGER.info('No parameters have been loaded')
        else:
            SIMULATION_LOGGER.info('Simulation parameters=')
            for tag in self.params:
                SIMULATION_LOGGER.info('\t%s=%s' % (tag, str(self.params[tag])))

    def compute_cost_fcn(self, kpis):
        if self.cost_fcn is None:
            return None
        self.cost_fcn.set_kpis(kpis)
        return self.cost_fcn.compute()

    def compute_constraints(self, kpis):
        if self.cost_fcn is None:
            return None
        self.cost_fcn.set_kpis(kpis)
        return self.cost_fcn.compute_constraints()

    def evaluate_tasks(self, task_costs):
        assert isinstance(task_costs, list), 'Task costs must be given as a list'
        return eval(self.tasks_eval_fcn % task_costs)


import time
import traceback

from django.db import transaction
from django.http import HttpResponse

from regluit.core.models import Campaign

def test_read(request):

    try:

        row_id = 1

        print("Attempting to read row")

        # A read the waits for the exclusive lock for the row
        campaign = Campaign.objects.raw("SELECT * FROM core_campaign WHERE id=%d FOR UPDATE" % row_id)[0]

        print("Successfully read row data %d" % campaign.target)

    except:
        traceback.print_exc()

    return HttpResponse("Success")

def test_write(request):

    try:

        row_id = 1
        campaign = Campaign.objects.get(id=row_id)

        print("Attempting to write row via ordinary ORM")

        #
        # Modify the data.  This will block if any shared lock (Either FOR UPDATE or LOCK IN SHARED MODE is held
        #
        campaign.target = campaign.target + 10
        campaign.save()

        print("Successfully write new row data %d" % campaign.target)

    except:
        traceback.print_exc()

    return HttpResponse("Success")

@transaction.commit_on_success
def test_lock(request):

    try:
        row_id = 1

        print("Attempting to acquire row lock")

        campaign = Campaign.objects.raw("SELECT * FROM core_campaign WHERE id=%d FOR UPDATE" % row_id)[0]

        print("Row lock acquired, modifying data")

        # Modify the data
        campaign.target = campaign.target + 10
        campaign.save()

        #
        # test by sleeping here for 10 seconds.
        #
        # The FOR UPDATE request will lock the row exclusively.  All write/delete operations require a compatible lock
        # and will block until this transaction is complete.  Some reads will block, but some will not.  If we want to
        # block the read until this transaction is complete, the read should also acquire an exlusive OR a shared lock.
        #
        # As soon as the function completes, the transaction will be committed and the lock released.
        # You can modify the commit_on_success decorator to get different transaction behaviors
        #
        print("Thread sleeping for 10 seconds")
        time.sleep(10)
        print("Thread sleep complete")

    except:
        traceback.print_exc()

    return HttpResponse("Success")


from utils import detector_utils as detector_utils
import cv2
import tensorflow as tf
import numpy as np
import datetime
import argparse
from keras.models import load_model
import os
import json
import random
from collections import Counter
from google.cloud import texttospeech
from playsound import playsound
import time
import winsound

# Instantiates a client
client = texttospeech.TextToSpeechClient()

# Set the text input to be synthesized

# Build the voice request, select the language code ("en-US") and the ssml
# voice gender ("neutral")
voice = texttospeech.types.VoiceSelectionParams(
    language_code='en-US',
    ssml_gender=texttospeech.enums.SsmlVoiceGender.NEUTRAL)

# Select the type of audio file you want returned
audio_config = texttospeech.types.AudioConfig(
    audio_encoding=texttospeech.enums.AudioEncoding.MP3)

detection_graph, sess = detector_utils.load_inference_graph()
labels = json.load(open("../signclassification/labels.json"))
model = load_model("../signclassification/model.h5.olu.old")

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-sth',
        '--scorethreshold',
        dest='score_thresh',
        type=float,
        default=0.2,
        help='Score threshold for displaying bounding boxes')
    parser.add_argument(
        '-fps',
        '--fps',
        dest='fps',
        type=int,
        default=1,
        help='Show FPS on detection/display visualization')
    parser.add_argument(
        '-src',
        '--source',
        dest='video_source',
        default=0,
        help='Device index of the camera.')
    parser.add_argument(
        '-wd',
        '--width',
        dest='width',
        type=int,
        default=320,
        help='Width of the frames in the video stream.')
    parser.add_argument(
        '-ht',
        '--height',
        dest='height',
        type=int,
        default=180,
        help='Height of the frames in the video stream.')
    parser.add_argument(
        '-ds',
        '--display',
        dest='display',
        type=int,
        default=1,
        help='Display the detected images using OpenCV. This reduces FPS')
    parser.add_argument(
        '-num-w',
        '--num-workers',
        dest='num_workers',
        type=int,
        default=4,
        help='Number of workers.')
    parser.add_argument(
        '-q-size',
        '--queue-size',
        dest='queue_size',
        type=int,
        default=5,
        help='Size of the queue.')
    args = parser.parse_args()

    cap = cv2.VideoCapture(args.video_source)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, args.width)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, args.height)

    start_time = datetime.datetime.now()
    num_frames = 0
    im_width, im_height = (cap.get(3), cap.get(4))
    # max number of hands we want to detect/track
    num_hands_detect = 2

    cv2.namedWindow('Single-Threaded Detection', cv2.WINDOW_NORMAL)

    while True:
        # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
        ret, image_np = cap.read()
        image_np = cv2.flip(image_np, 1)
        try:
            image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
        except:
            print("Error converting to RGB")

        # Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
        # while scores contains the confidence for each of these boxes.
        # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)

        boxes, scores = detector_utils.detect_objects(image_np,
                                                      detection_graph, sess)

        # draw bounding boxes on frame
        all_images = detector_utils.draw_box_on_image(num_hands_detect, args.score_thresh,
                                                      scores, boxes, im_width, im_height,
                                                      image_np)

        majority_list = []
        if len(all_images) != 0:

            # print(np.shape(np.array(all_images)))
            # dell = cv2.imread("../signclassification/data/asl_alphabet_train/asl_alphabet_train/A/A3_custom_9207.jpg")
            # dell = cv2.resize(dell, (64, 64))
            # cv2.imshow("title2", dell)
            # prediction = model.predict(np.array([dell]))
            # prediction_indices = tf.keras.backend.eval(tf.argmax(prediction, axis=1))
            # print(f"This is the fake prediction: {prediction_indices}: {[labels[str(idx)] for idx in prediction_indices]}")

            prediction = model.predict(np.array(all_images))
            cv2.imshow("title", all_images[0])
            prediction_indices = tf.keras.backend.eval(tf.argmax(prediction, axis=1))
            print(f"This is the honest prediction: {prediction_indices}: {[labels[str(idx)] for idx in prediction_indices]}")
            majority_list.extend([labels[str(idx)] for idx in prediction_indices])

        # # Calculate Frames per second (FPS)

        num_frames += 1
        elapsed_time = (datetime.datetime.now() - start_time).total_seconds()
        fps = num_frames / elapsed_time

        # if num_frames % 2 == 0:
        #     for img in all_images:
        #         print("IMAGE HERE!!!")
        #         cv2.imwrite(f"../signclassification/data2/A/A{num_frames//10}_custom_{random.randint(0,10000)}.jpg", img)

        if num_frames % 10 == 0 and len(majority_list) > 0:
            cnt = Counter(majority_list)
            majority_list = []
            majority = cnt.most_common(1)[0]
            print(f"IMAGE HERE!!!: {majority}")
            synthesis_input = texttospeech.types.SynthesisInput(text=majority[0])
            response = client.synthesize_speech(synthesis_input, voice, audio_config)
            appendix = random.randint(0, 10000)
            # winsound.PlaySound(response.audio_content)
            # time.sleep(1000)
            with open(f'output-{appendix}.mp3', 'wb') as out:
                # Write the response to the output file.
                out.write(response.audio_content)
                print('Audio content written to file "output.mp3"')
                out.close()
            playsound(f'output-{appendix}.mp3', block=True)

        if (args.display > 0):
            # Display FPS on frame
            if (args.fps > 0):
                detector_utils.draw_fps_on_image("FPS : " + str(int(num_frames/10)),
                                                 image_np)

            cv2.imshow('Single-Threaded Detection',
                       cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR))

            if cv2.waitKey(27) & 0xFF == ord('q'):
                cv2.destroyAllWindows()
                break
        else:
            print("frames processed: ", num_frames, "elapsed time: ",
                  elapsed_time, "fps: ", str(int(fps)))


"""contains the packet class and functions to parse packets"""
import socket
import struct
from colorama import Fore
from pyket import network

def unpack(fmt, data):
    """unpacks data using the given format"""
    return struct.unpack(fmt, data[: struct.calcsize(fmt)])

def print_data(data: str) -> None:
    """prints the data of a packet"""
    try:
        print(f"Data: \n{Fore.LIGHTRED_EX}" + data.decode("utf-8") + f"{Fore.RESET}")
    except UnicodeDecodeError:
        pass

def print_header(packet):
    """prints the header of a packet"""
    print(f"{Fore.GREEN}-{Fore.RESET}" * 60)
    print(f"{Fore.GREEN}Destination MAC: {Fore.RESET}" + network.eth_addr(packet[:6]))
    print(
        f"{Fore.LIGHTRED_EX}Source MAC: {Fore.RESET}" + network.eth_addr(packet[6:12])
    )

def print_body(version, ihl, ttl, protocol, s_addr, d_addr):
    """prints the body of a packet"""
    print(
        f"{Fore.LIGHTRED_EX}Version: {Fore.RESET}{version}\n"
        f"{Fore.LIGHTRED_EX}IP Header Length: {Fore.RESET}{ihl}\n"
        f"{Fore.LIGHTRED_EX}TTL: {Fore.RESET}{ttl}\n"
        f"{Fore.LIGHTRED_EX}Protocol: {Fore.RESET}{protocol}\n"
        f"{Fore.LIGHTRED_EX}Source Address: {Fore.RESET}{s_addr}\n"
        f"{Fore.LIGHTRED_EX}Destination Address: {Fore.RESET}{d_addr}\n"
    )

def udp(packet, eth_length, protocol, iph_length):
    """parses udp packets"""
    if protocol == 17:
        u = iph_length + eth_length
        udph_length = 8
        udp_header = packet[u : u + 8]

        # Unpack the udp header
        udph = unpack("!HHHH", udp_header)

        source_port = udph[0]
        dest_port = udph[1]
        length = udph[2]
        checksum = udph[3]

        # Get data
        h_size = eth_length + iph_length + udph_length
        data = packet[h_size:]

        print_header(packet)

        print("\nPACKET TYPE: UDP\n")

        # Defining these in every function is necessary, but ugly
        ip_header = packet[eth_length : 20 + eth_length]
        iph = unpack("!BBHHHBBH4s4s", ip_header)
        version_ihl = iph[0]
        version = version_ihl >> 4
        ihl = version_ihl & 0xF
        ttl = iph[5]
        s_addr = socket.inet_ntoa(iph[8])
        d_addr = socket.inet_ntoa(iph[9])

        print_body(version, ihl, ttl, protocol, s_addr, d_addr)

        print(
            f"{Fore.LIGHTRED_EX}Source Port: {Fore.RESET}{source_port}\n"
            f"{Fore.LIGHTRED_EX}Destination Port: {Fore.RESET}{dest_port}\n"
            f"{Fore.LIGHTRED_EX}Length: {Fore.RESET}{length}\n"
            f"{Fore.LIGHTRED_EX}Checksum: {Fore.RESET}{checksum}\n"
        )

        print_data(data)
    else:
        pass

def tcp(packet, eth_length, protocol, iph_length):
    """parses TCP packets"""
    if protocol == 6:
        t = iph_length + eth_length
        tcp_header = packet[t : t + 20]

        # Unpack the TCP header
        tcph = unpack("!HHLLBBHHH", tcp_header)

        source_port = tcph[0]
        dest_port = tcph[1]

        # Get sequence & acknowledgement number
        sequence = tcph[2]
        acknowledgement = tcph[3]

        # Bit shifting to get flags
        doff_reserved = tcph[4]
        tcph_length = doff_reserved >> 4

        # Get data
        h_size = eth_length + iph_length + tcph_length * 4
        data = packet[h_size:]

        print_header(packet)

        print("\nPACKET TYPE: TCP\n")

        # Version
        ip_header = packet[eth_length : 20 + eth_length]
        iph = unpack("!BBHHHBBH4s4s", ip_header)
        version_ihl = iph[0]
        version = version_ihl >> 4
        ihl = version_ihl & 0xF
        ttl = iph[5]
        s_addr = socket.inet_ntoa(iph[8])
        d_addr = socket.inet_ntoa(iph[9])

        print_body(version, ihl, ttl, protocol, s_addr, d_addr)

        print(
            f"{Fore.LIGHTRED_EX}Source Port: {Fore.RESET}{source_port}\n"
            f"{Fore.LIGHTRED_EX}Destination Port: {Fore.RESET}{dest_port}\n"
            f"{Fore.LIGHTRED_EX}Sequence Number: {Fore.RESET}{sequence}\n"
            f"{Fore.LIGHTRED_EX}Acknowledgement: {Fore.RESET}{acknowledgement}\n"
            f"{Fore.LIGHTRED_EX}TCP Header Length: {Fore.RESET}{tcph_length}\n"
        )

        print_data(data)
    else:
        pass

def ipv4(packet, eth_length, eth_protocol, filter_protocol):
    """parses IPv4 packets"""
    if eth_protocol == 8:
        # Parse IP header
        ip_header = packet[eth_length : 20 + eth_length]

        # Unpack IP header
        iph = unpack("!BBHHHBBH4s4s", ip_header)

        # Version
        version_ihl = iph[0]
        ihl = version_ihl & 0xF

        # IP header length
        iph_length = ihl * 4

        # Protocol
        protocol = iph[6]

        if filter_protocol == "all":
            udp(packet, eth_length, protocol, iph_length)
            tcp(packet, eth_length, protocol, iph_length)
        elif filter_protocol == "udp":
            udp(packet, eth_length, protocol, iph_length)
        elif filter_protocol == "tcp":
            tcp(packet, eth_length, protocol, iph_length)

def capture(interface: str, filter_protocol) -> None:
    """captures packets"""
    s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.ntohs(3))
    s.bind((interface, 0))
    while True:
        packet = s.recvfrom(65565)
        packet = packet[0]

        eth_length = 14
        eth_header = packet[:eth_length]
        eth = unpack("!6s6sH", eth_header)
        eth_protocol = socket.ntohs(eth[2])

        ipv4(packet, eth_length, eth_protocol, filter_protocol=filter_protocol)


from setuptools import setup, find_packages

pkgs = find_packages()
readme = open('README.md', encoding='utf-8').read()

setup(
    name='nlprov',
    version='1.1.0',
    description='A Python package for common Natural Language Processing tasks',
    long_description=readme,
    long_description_content_type='text/markdown',
    author='Johnson & Johnson HTC Advanced Analytics',
    url='https://github.com/johnsonandjohnson/nlprov',
    packages=pkgs,
    install_requires=[
        'pandas>=1.0.0',
        'spacy>=3.4',
        'nltk>=3.4.3',
        'langid>=1.1.6',
        'scikit-learn>=0.21.3'
    ],
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'License :: OSI Approved :: MIT License',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: Healthcare Industry',
        'Intended Audience :: Information Technology',
        'Programming Language :: Python :: 3.7',
        'Topic :: Scientific/Engineering'
    ],
    tests_require=['pytest'],
    python_requires='>=3.7'
)

# https://github.com/tomchen/example_pypi_package/blob/d50c61f1317c0cc7d1fc8927d666da7355732c61/README.md


# Save the program before compiling it

# In this program using the pandas library we will read the "," seprated text files
# We will extract valuesof the longitude and latitude from  the text file seprately .....
# USing the for loop we will import the volcanoes in the file

import folium
import pandas

data = pandas.read_csv("Volcanoes.txt")

# Getting the list of the Columns
# W eprefer to work on the list because working on the list is faster

lat = list(data["LAT"])
lon = list(data["LON"])

map = folium.Map(location = [38.58,-99.09], zoom_start=6, tiles="Stamen Terrain")

# Adding the Children to the Map
# Creating the feature group & in the feature group we can add the marker and the polygon
fg = folium.FeatureGroup(name="My Map")
for lt, ln in zip(lat, lon):   # Tese are the list elements mention in the for loop
    map.add_child(folium.Marker(location = [lt, ln], popup = "Hi I am a Marker", icon = folium.Icon(color='green')))

map.save("Volcanoes.html")


import warnings
import numpy as np
import unittest

import grid2op
from grid2op.Parameters import Parameters
from lightsim2grid import LightSimBackend

from superposition_theorem.core.compute_power_flows import get_virtual_line_flow, get_sub_node1_idsflow, compute_flows_superposition_theorem_from_actions, get_delta_theta_sub_2nodes

class TestNodeMergingSup(unittest.TestCase):

    def setUp(self) -> None:
        env_name = "l2rpn_case14_sandbox"
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore")

            params = Parameters()
            params.NO_OVERFLOW_DISCONNECTION = True
            params.LIMIT_INFEASIBLE_CURTAILMENT_STORAGE_ACTION = True
            params.ENV_DC = True
            params.MAX_LINE_STATUS_CHANGED = 99999
            params.MAX_SUB_CHANGED = 99999

            self.env = grid2op.make(env_name,backend=LightSimBackend(), param=params, test=True)

            self.env.set_max_iter(20)
        self.chronic_id = 0
        self.max_iter = 10
        self.tol = 3e-5

    def tearDown(self) -> None:
        self.env.close()
        return super().tearDown()

    def test_2_combined_actions_node_merging_by_hand(self):
        """Testing the combination of two nodes merging actions at different substations. Computing by hand the superposition of their unitary action state
        to reach the combined action state"""

        self.env.set_id(self.chronic_id)
        self.env.reset()

        unitary_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 1, 1, 1, 1, 1))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (1, 1, 1, 1, 1))]}},  # sub4
                               ]

        # need ooposite list of actions to start with an observation where those lines are disconnected
        opposite_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 2, 2, 1, 2, 2))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (2, 1, 2, 1, 2))]}},  # sub4
                               ]

        combined_opposite_action = self.env.action_space(opposite_action_list[0])+self.env.action_space(opposite_action_list[1])
        obs_start, *_  = self.env.step(combined_opposite_action)

        id_sub1 = 5  # 3#2#3
        id_sub2 = 4  # 7#4#7

        unitary_actions = [self.env.action_space(unitary_act) for unitary_act in unitary_action_list]

        # computing obs_target load flow with the effect of combined action: this is the ground truth to compare to
        combined_action = unitary_actions[0] + unitary_actions[1]
        obs_target = obs_start.simulate(combined_action, time_step=0)[0]

        # computing intermediate states in which we applied one unitary action

        obs1 = obs_start.simulate(unitary_actions[0], time_step=0)[0]
        obs2 = obs_start.simulate(unitary_actions[1], time_step=0)[0]

        # computing delta theetas at a substation between nodes split

        delta_theta_sub1_obs2 = get_delta_theta_sub_2nodes(obs2,id_sub1)
        delta_theta_sub1_obs_start = get_delta_theta_sub_2nodes(obs_start, id_sub1)

        delta_theta_sub2_obs1 = get_delta_theta_sub_2nodes(obs1,id_sub2)
        delta_theta_sub2_obs_start = get_delta_theta_sub_2nodes(obs_start, id_sub2)

        assert(delta_theta_sub1_obs2!=0)
        assert(delta_theta_sub1_obs_start!=0)
        assert(delta_theta_sub2_obs1!=0)
        assert(delta_theta_sub2_obs_start!=0)

        # solving the linear system by "hand" and computing the superposition
        a = np.array([[1, 1 - delta_theta_sub1_obs2 / delta_theta_sub1_obs_start],
                      [1 - delta_theta_sub2_obs1 / delta_theta_sub2_obs_start, 1]])

        b = np.ones(2)
        betas = np.linalg.solve(a, b)

        p_target_computed = betas[0] * obs1.p_or + betas[1] * obs2.p_or + (1 - betas.sum()) * obs_start.p_or

        assert np.max(np.abs(obs_target.p_or - p_target_computed)) <= self.tol

    def test_2_combined_actions_node_merging_sup_theorem(self):
        """Testing the compute_flows_superposition_theorem_from_actions function
        in the case of a combination of two nodes merging actions at different substations"""

        self.env.set_id(self.chronic_id)
        self.env.reset()

        unitary_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 1, 1, 1, 1, 1))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (1, 1, 1, 1, 1))]}},  # sub4
                               ]

        # need ooposite list of actions to start with an observation where those lines are disconnected
        opposite_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 2, 2, 1, 2, 2))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (2, 1, 2, 1, 2))]}},  # sub4
                               ]

        combined_opposite_action = self.env.action_space(opposite_action_list[0])+self.env.action_space(opposite_action_list[1])
        obs_start, *_  = self.env.step(combined_opposite_action)
        id_sub1 = 5  # 3#2#3
        id_sub2 = 4  # 7#4#7
        idls_lines = []
        idls_subs = [id_sub1, id_sub2]

        unitary_actions = [self.env.action_space(unitary_act) for unitary_act in unitary_action_list]

        # computing intermediate states in which we applied one unitary action
        combined_action = self.env.action_space({})
        for unit_act in unitary_actions:
            combined_action+=unit_act
        obs_target = obs_start.simulate(combined_action, time_step=0)[0]

        # running superposition theorem function
        check_obs_target = False
        p_target_computed = compute_flows_superposition_theorem_from_actions(idls_lines, idls_subs, obs_start, unitary_actions, check_obs_target)

        assert np.max(np.abs(obs_target.p_or - p_target_computed)) <= self.tol

    def test_4_combined_actions_node_merging_sup_theorem(self):
        """Testing the compute_flows_superposition_theorem_from_actions function
             in the case of a combination of 4 node merging actions at 4 different substations"""
        self.env.set_id(self.chronic_id)
        self.env.reset()

        unitary_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 1, 1, 1, 1, 1))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (1, 1, 1, 1, 1))]}},  # sub4
                               {'set_bus': {'substations_id': [(3, (1, 1, 1, 1, 1, 1))]}},  # sub3
                               {'set_bus': {'substations_id': [(8, (1, 1, 1, 1, 1))]}},  # sub8
                               ]

        id_sub1 = 5  # 3#2#3
        id_sub2 = 4  # 7#4#7
        id_sub3 = 3
        id_sub4 = 8

        idls_lines = []
        idls_subs = [id_sub1, id_sub2,id_sub3, id_sub4]

        # need ooposite list of actions to start with an observation where those lines are disconnected
        opposite_action_list = [{'set_bus': {'substations_id': [(5, (1, 1, 2, 2, 1, 2, 2))]}},  # sub5
                               {'set_bus': {'substations_id': [(4, (2, 1, 2, 1, 2))]}},  # sub4
                               {'set_bus': {'substations_id': [(3, (2, 1, 1, 2, 1, 2))]}},  # sub3
                               {'set_bus': {'substations_id': [(8, (2, 1, 1, 2, 2))]}},  # sub8
                               ]

        combined_opposite_action=self.env.action_space({})
        for unit_act in opposite_action_list:
            combined_opposite_action+=self.env.action_space(unit_act)
        obs_start, *_  = self.env.step(combined_opposite_action)

        unitary_actions = [self.env.action_space(unitary_act) for unitary_act in unitary_action_list]

        # computing intermediate states in which we applied one unitary action
        combined_action = self.env.action_space({})
        for unit_act in unitary_actions:
            combined_action+=unit_act
        obs_target = obs_start.simulate(combined_action, time_step=0)[0]

        # running superposition theorem function
        check_obs_target = False
        p_target_computed = compute_flows_superposition_theorem_from_actions(idls_lines, idls_subs, obs_start, unitary_actions, check_obs_target)

        assert np.max(np.abs(obs_target.p_or - p_target_computed)) <= self.tol


# -*- coding: utf-8 -*-
#hadoop fs -put qq.txt /newscreeper/qq-20170413.txt
import time
from pyspark.sql import Row
from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("uniquness news").config("master", "local[*]").getOrCreate()
sc = spark.sparkContext
date_str = time.strftime("%Y%m%d", time.localtime())
file_path = "hdfs://localhost:9000/newscreeper/qq-" + date_str + ".txt"
lines = sc.textFile(file_path)
parts = lines.map(lambda l: l.split("\t"))

news = parts.map(lambda p: Row(time_str = p[0], url = p[1], title = p[2], content = p[3]))

schemaNews = spark.createDataFrame(news)
schemaNews.createOrReplaceTempView("news")

uniq_urls = spark.sql("SELECT distinct url from news").rdd.map(lambda p: p.url).collect()
print("===================uniq_urls: %d" % len(uniq_urls))
uniq_news_list = news.filter(lambda n: uniq_urls.index(n.url) > -1)

def filter_handler(n):
  if n.url in uniq_urls:
    uniq_urls.remove(n.url)
    return True
  else:
    return False

uniq_news_list = news.filter(filter_handler).map(lambda p: "\t".join([p.time_str, p.url, p.title, p.content])).collect()
print("===================uniq_news_list: %d" % len(uniq_news_list))
file = open("out_file/unqi.txt","wb")
try:
  for one_news in uniq_news_list:
    file.write((one_news + "\n").encode('utf-8'))
finally:
  file.close()


#crie um programa q leia ano de nascimento de 7 pessoas
#no final, mostre qnts pessoas ainda não atingiram a maioridade e
#qnts são maiores de idade (21 anos)

from datetime import date

maiores = 0
menores = 0
print('SER ADULTO É UMA DROGA')
print('Será que você já é adulto?')
for c in range(1, 8):
    ano = int(input('Digite seu ano de nascimento: '))
    idade = date.today().year - ano
    if idade <= 21:
        maiores += 1
    else:
        menores += 1
print('Nessa lista, {} pessoas são maiores de idade e {} são menores de idade.'.format(maiores, menores))


import copy
from typing import Callable, Dict, List

import requests
import tenacity
from airflow.hooks.http_hook import HttpHook
from airflow.models import Variable
from dateutil.parser import parse
from ods.kktix_ticket_orders.udfs import kktix_loader, kktix_transformer

SCHEDULE_INTERVAL_SECONDS: int = 3600
HTTP_HOOK = HttpHook(http_conn_id="kktix_api", method="GET")
RETRY_ARGS = dict(
    wait=tenacity.wait_none(),
    stop=tenacity.stop_after_attempt(3),
    retry=tenacity.retry_if_exception_type(requests.exceptions.ConnectionError),
)

def main(**context):
    """
    ETL pipeline should consists of extract, transform and load
    """
    schedule_interval = context["dag"].schedule_interval
    # If we change the schedule_interval, we need to update the logic in condition_filter_callback
    assert schedule_interval == "50 * * * *"  # nosec
    ts_datetime_obj = parse(context["ts"])
    year = ts_datetime_obj.year
    timestamp = ts_datetime_obj.timestamp()
    event_raw_data_array = _extract(year=year, timestamp=timestamp,)
    transformed_event_raw_data_array = kktix_transformer.transform(
        copy.deepcopy(event_raw_data_array)
    )
    kktix_loader.load(transformed_event_raw_data_array)
    print(f"Loaded {len(transformed_event_raw_data_array)} rows to BigQuery!")

    # pass these unhashed data through xcom to next airflow task
    return kktix_transformer._extract_sensitive_unhashed_raw_data(
        copy.deepcopy(event_raw_data_array)
    )

def _extract(year: int, timestamp: float) -> List[Dict]:
    """
    get data from KKTIX's API
    1. condition_filter_callb: use this callbacl to filter out unwanted event!
    2. right now schedule_interval_seconds is a hardcoded value!
    """
    event_raw_data_array: List[Dict] = []
    condition_filter_callback = (
        lambda event: str(year) in event["name"]
        and "registration" in event["name"].lower()
    )
    event_metadatas = get_event_metadatas(condition_filter_callback)
    for event_metadata in event_metadatas:
        event_id = event_metadata["id"]
        for attendee_info in get_attendee_infos(event_id, timestamp):
            event_raw_data_array.append(
                {
                    "id": event_id,
                    "name": event_metadata["name"],
                    "attendee_info": attendee_info,
                }
            )
    return event_raw_data_array

def get_attendee_infos(event_id: int, timestamp: float) -> List:
    """
    it's a public wrapper for people to get attendee infos!
    """
    attendance_book_id = _get_attendance_book_id(event_id)
    attendee_ids = _get_attendee_ids(event_id, attendance_book_id)
    attendee_infos = _get_attendee_infos(event_id, attendee_ids, timestamp)
    return attendee_infos

def get_event_metadatas(condition_filter: Callable) -> List[Dict]:
    """
    Fetch all the ongoing events
    """
    event_list_resp = HTTP_HOOK.run_with_advanced_retry(
        endpoint=f"{Variable.get('kktix_events_endpoint')}?only_not_ended_event=true",
        _retry_args=RETRY_ARGS,
    ).json()
    event_metadatas: List[dict] = []
    for event in event_list_resp["data"]:
        if condition_filter(event):
            event_metadatas.append(event)
    return event_metadatas

def _get_attendance_book_id(event_id: int) -> int:
    """
    Fetch attendance books
    """
    attendance_books_resp = HTTP_HOOK.run_with_advanced_retry(
        endpoint=f"{Variable.get('kktix_events_endpoint')}/{event_id}/attendance_books?only_not_ended_event=true",
        _retry_args=RETRY_ARGS,
    ).json()
    return attendance_books_resp[0]["id"]

def _get_attendee_ids(event_id: int, attendance_book_id: int) -> List[int]:
    """
    get all attendee ids!
    """
    attendee_ids = []
    attendees_resp = HTTP_HOOK.run_with_advanced_retry(
        endpoint=f"{Variable.get('kktix_events_endpoint')}/{event_id}/attendance_books/{attendance_book_id}",
        _retry_args=RETRY_ARGS,
    ).json()
    for signin_status_tuple in attendees_resp["signin_status"]:
        attendee_ids.append(signin_status_tuple[0])
    return attendee_ids

def _get_attendee_infos(
    event_id: int, attendee_ids: List[int], timestamp: float
) -> List:
    """
    get attendee infos, e.g. email, phonenumber, name and etc
    """
    attendee_infos = []
    for attendee_id in attendee_ids:
        attendee_info = HTTP_HOOK.run_with_advanced_retry(
            endpoint=f"{Variable.get('kktix_events_endpoint')}/{event_id}/attendees/{attendee_id}",
            _retry_args=RETRY_ARGS,
        ).json()
        if not attendee_info["is_paid"]:
            continue
        if (
            timestamp
            < attendee_info["updated_at"]
            < timestamp + SCHEDULE_INTERVAL_SECONDS
        ):
            attendee_infos.append(attendee_info)
    return attendee_infos


from sql_metadata import token, Parser
import copy
from connection import DB

class PreProcessor:
    """
     A class the converts input SQL query string into a decomposed query, and query component mapping dictionary.
    """

    def __init__(self, query, db) -> None:
        """
        Constructor to initialize the relevant components of the Preprocessor

        Parameter query: input SQL query string

        Parameter db: db connection instance
        """
        # self.query_split = sqlparse.split(query)[0] # assuming only 1 query at a time
        # self.query_split = sqlparse.format(query, reindent=True, keyword_case="upper").splitlines()
        # self.query = self.clean(self.query_split)
        print("Inside init")
        self.query = query
        self.db = db
        self.index_column_dict = {}
        self.parser = Parser(self.query)
        self.tokens = self.parser.tokens
        self.tables = self.parser.tables
        self.get_metadata()
        self.parser_columns, self.columns = self.get_columns()
        self.column_to_table_mapping = self.get_column_to_table_mapping()
        self.not_transformations = {
            "<": ">=",
            ">": "<=",
            "=": "<>",
            "<>": "=",
            "!=": "=",
            "<=": ">",
            ">=": "<"
        }
        self.curr_query_num = {
            "query_1": {
                0: 1
            }
        }
        self.tables_aliases = self.parser.tables_aliases
        self.aggregate_list = ["count", "sum", "avg", "approx_count_distinct", "max", "min", "stdev", "stdevp", "var", "varp", "string_agg"]
        self.decomposed_query, self.query_components = self.extract_keywords(self.tokens, {"query_1": {"subqueries": {}}}, {"query_1": {"subqueries": {}}})

    def clean(self, query_split):
        """
        Clean input split up SQL query

        Parameter query_split: split up SQL query
        """
        reconstructed_query = ""
        for clause in query_split:
            reconstructed_query += ' {}'.format(clause.strip())
        return reconstructed_query

    def print_token_debug_info(self, t:token.SQLToken):
        """
        Print debug information for token

        Parameter t: current token being processed
        """
        print("Value: " + str(t.value))
        print(t.is_keyword)
        print("Different types: ")
        print("Is keyword: " + str(t.is_keyword))
        print("Is punctuation: " + str(t.is_punctuation))
        print("Is dot: " + str(t.is_dot))
        print("Is wildcard: " + str(t.is_wildcard))
        print("Is integer: " + str(t.is_integer))
        print("Is float: " + str(t.is_float))
        print("Is comment: " + str(t.is_comment))
        print("Is as keyword: " + str(t.is_as_keyword))
        print("Is left parenthesis: " + str(t.is_left_parenthesis))
        print("Is right parenthesis: " + str(t.is_right_parenthesis))
        print("Parenthesis level: " + str(t.parenthesis_level))

    def get_metadata(self):
        """
        Get all requried metadata. This includes relation names, column names, and indexes.
        """
        result = self.db.execute("select table_name from information_schema.tables where table_schema='public';")
        self.all_column_names = {}
        self.all_table_names = [res[0] for res in result]
        print(self.all_table_names)
        for table in self.all_table_names:
            result = self.db.execute("select * from information_schema.columns where table_name='{}' order by ordinal_position".format(table))
            for val in result:
                self.all_column_names[val[3]] = table
        print(self.all_column_names)
        index_columns = self.db.execute("SELECT tablename,indexname,indexdef FROM pg_indexes WHERE schemaname = 'public' ORDER BY tablename, indexname")
        print("----------------------Index Meta Data-------------------------------------")
        print(index_columns)
        for index in index_columns:
            table = index[0]
            index_name = index[1]
            create_index = index[2].split("public.",1)[1]
            mk1 = create_index.find('(') + 1
            mk2 = create_index.find(')', mk1)
            column_name = create_index[ mk1 : mk2 ]
            self.index_column_dict[index_name] = [table,column_name]

    def get_relations(self):
        """
        Get all relation names from SQL input query
        """
        return self.parser.tables

    def get_columns(self):
        """
        Get all column names from SQL input query
        """
        return self.parser.columns, [column[column.find(".") + 1:] if column.find(".") != -1 else column for column in self.parser.columns]

    def get_column_to_table_mapping(self):
        """
        Map column names to tables to enrich SQL clauses
        """
        query_columns = {column: self.all_column_names[column] if column not in ['*', 'UNITS'] else column for column in self.columns}
        return query_columns

    def prepend_table_name_to_column(self, curr_query, t: token.SQLToken, curr_level):
        """
        Prepend table name to column

        Parameter curr_query: current decomposed query

        Parameter t: current token being processed

        Parameter curr_level: current level of query
        """
        # table name already present
        if "." in t.value.lower():
            return t
        # otherwise
        if t.value.lower() in self.columns:
            column_name = t.value.lower()
            t.value = self.column_to_table_mapping[t.value.lower()]
            old_val = t.value
            print("Old value")
            t = self.replace_alias_with_expression(curr_query, t)
            if (t.value == old_val):
                if (curr_level != 0):
                    t.value += "_" + str(curr_level)
            t.value += "." + column_name
        return t

    def print_query_debug_info(self):
        """
        Print debug information for query
        """
        for t in self.tokens:
            self.print_token_debug_info(t)

    def handle_as_keyword(self, curr_query, curr_component: list, tokens, t: token.SQLToken, i, last_keyword_token, last_comma_index):
        """
        Handle the "as" keyword when encountered in the inpur query

        Parameter curr_query: current decomposed query

        Parameter curr_component: current query component dictionary

        Parameter tokens: extracted SQL query tokens

        Parameter t: current token being processed

        Parameter i: current index while parsing the SQL query tokens

        Parameter last_comma_index: last comma index

        Parameter last_keyword_token: last keyword token
        """
        if t.value not in curr_query:
            curr_query[t.value.lower()] = {}
        aggregate = curr_query[last_keyword_token.value.lower()][-1]
        i += 1
        alias: token.SQLToken = tokens[i]
        curr_component.append(alias.value.lower())
        curr_query[t.value.lower()][alias.value.lower()] = aggregate
        last_comma_index = i + 1
        return curr_query, i, last_comma_index, curr_component

    # NOT IN USE
    def handle_table_name(self, curr_query, tokens, t:token.SQLToken, i, last_keyword_token:token.SQLToken, last_comma_index):
        if (t in self.tables):
            table = t
            curr_query[last_keyword_token.value.lower()].append(table.value.lower())
            # alias cases
            i += 1
            t = tokens[i]

            # case 1: explicit 'as' keyword
            if t.is_as_keyword():
                curr_query, i, last_comma_index = self.handle_as_keyword(curr_query, tokens, t, i, last_keyword_token, last_comma_index)

    def replace_alias_with_expression(self, curr_query, t: token.SQLToken):
        """
        Replace aliases appearing in a query with the corresponding full form. Helps in annotation.

        Parameter curr_query: current query component

        Parameter t: current token being processed
        """
        if "as" not in curr_query:
            return t
        if t.value.lower() in curr_query["as"]:
            t.value = curr_query["as"][t.value.lower()]
        return t

    def extract_where_having(self, curr_query, query_components, clause_tokens, clause_list: list, last_keyword_token: token.SQLToken, query_alias, curr_level):
        """
        Extract clauses from where/having tokens

        Parameter curr_query: current query component

        Parameter query_components: current query components

        Parameter clause_tokens: extracted where/having clause tokens

        Parameter last_keyword_token: last keyword token

        Parameter query_alias: alias for current query

        Parameter curr_level: current level of query
        """
        i = 0
        last_index = i
        curr_clause_not = False
        contains_subquery = False
        subquery_start_index = None
        subquery_alias = None
        # query_segment = (" ".join([t.value for t in clause_tokens]))[:-1]
        raw_clause_lst = []
        raw_tok_lst = []
        raw_clause = ""
        is_between = False
        while i < len(clause_tokens):
            t: token.SQLToken = clause_tokens[i]
            raw_tok_lst.append(t.value)
            t: token.SQLToken = self.prepend_table_name_to_column(curr_query, t, curr_level)
            t = self.replace_alias_with_expression(curr_query, t)
            # print("Value: " + str(t.value))
            if (t.value.lower() == "date"):
                i += 1
                continue
            if (t.value.lower() == "between"):
                val_count = 0
                vals = []
                between_index = i
                i += 1
                while i < len(clause_tokens) and val_count < 2:
                    t = clause_tokens[i]
                    raw_tok_lst.append(t.value)
                    print("RAW TOK LST: {}".format(raw_tok_lst))
                    if (t.value.lower() == "date" or t.value.lower() == "and"):
                        i += 1
                        continue
                    vals.append(t.value.lower())
                    val_count += 1
                    i += 1
                print("VALS: {}".format(vals))
                tok_lst = [tok.value for tok in clause_tokens[last_index: between_index]]
                col_name = " ".join(tok_lst)
                tok_lst = ["(", col_name, ">=", vals[0], ")", "and", "(", col_name, "<=", vals[1], ")"]
                print("TOK LST: {}".format(tok_lst))
                is_between = True
            if (t.parenthesis_level != last_keyword_token.parenthesis_level) and (t.is_keyword and t.value.lower() not in ["and", "or", "not"]):
                contains_subquery = True
                subquery_start_index = i
                # self.curr_subquery_num += 1
                # curr_query, query_components, i, subquery_alias = self.extract_subquery(curr_query, query_components, clause_tokens, i, last_keyword_token, t)
                if query_alias not in self.curr_query_num:
                    self.curr_query_num[query_alias] = {}
                if (curr_level + 1) not in self.curr_query_num[query_alias]:
                    self.curr_query_num[query_alias][curr_level + 1] = 1
                else:
                    self.curr_query_num[query_alias][curr_level + 1] += 1
                # self.curr_subquery_num += 1
                curr_query, query_components, i, subquery_alias = self.extract_subquery(curr_query, query_components, clause_tokens, i, last_keyword_token, t, self.curr_query_num[query_alias][curr_level + 1], curr_level + 1)
                i -= 1
                print("Subquery alias: " + subquery_alias)
            if t.is_keyword and (t.value.lower() == "and" or t.value.lower() == "or"):
                if not contains_subquery:
                    tok_lst = [tok.value for tok in clause_tokens[last_index: i]]
                    raw_clause += " ".join(raw_tok_lst)
                    raw_tok_lst = []
                else:
                    # print("reached this case")
                    tok_lst = [tok.value for tok in clause_tokens[last_index: subquery_start_index - 1]] + [subquery_alias]
                    raw_clause += " ".join(raw_tok_lst[:-2]) + subquery_alias
                    raw_tok_lst = []
                    contains_subquery = False
                    subquery_start_index = None
                    subquery_alias = None

                # print(tok_lst)
                if (curr_clause_not):
                    # print("Found NOT clause")
                    tok_lst[1] = self.not_transformations[tok_lst[1].lower()]
                    curr_clause_not = False
                # print(tok_lst)
                raw_clause_lst.append(raw_clause)
                raw_clause = ""
                clause_list.append(" ".join(tok_lst))
                last_index = i + 1
            elif (t.is_keyword and t.value.lower() == "not"):
                # raw_clause += t.value + " "
                curr_clause_not = True
                last_index = i + 1
            i += 1
        if not contains_subquery and not is_between:
            tok_lst = [tok.value for tok in clause_tokens[last_index: i]]
            # print("Raw token list")
            # print(raw_tok_lst)
            raw_clause = " ".join(raw_tok_lst)
        elif not is_between:
            tok_lst = [tok.value for tok in clause_tokens[last_index: subquery_start_index - 1]] + [subquery_alias]
            raw_clause = " ".join(raw_tok_lst[:-2]) + subquery_alias
            contains_subquery = False
            subquery_start_index = None
        # print(tok_lst)
        else:
            raw_clause = " ".join(raw_tok_lst)
        if (curr_clause_not):
            # print("Found NOT clause")
            tok_lst[1] = self.not_transformations[tok_lst[1].lower()]
            curr_clause_not = False
        # print(tok_lst)
        raw_clause_lst.append(raw_clause)
        clause_list.append(" ".join(tok_lst))
        curr_query[last_keyword_token.value.lower()] = clause_list
        # print("final raw tokens")
        # print(raw_clause_lst)
        return curr_query, raw_clause_lst

    def extract_subquery(self, curr_query, query_components, tokens, i, last_keyword_token: token.SQLToken, t: token.SQLToken, subquery_number, subquery_level):
        """
        Extract keywords from subquery (recursively)

        Parameter curr_query: current query component

        Parameter query_components: current query components

        Parameter tokens: extracted SQL query tokens

        Parameter i: current index while parsing the SQL query tokens

        Parameter last_keyword_token: last keyword token

        Parameter t: current token being processed

        Parameter subquery_number: number of new subquery under current query being processed

        Parameter subquery_level: subquery level for subquery to be extracted
        """
        subquery_tokens = []
        # a ')' with same parenthesis level as parent query marks end of subquery
        while t.value != ")" or t.parenthesis_level != last_keyword_token.parenthesis_level:
            t = tokens[i]
            subquery_tokens.append(t)
            i += 1
        # print("Subquery clause tokens: ")
        # for tok in subquery_tokens:
        #     print("Value: " + tok.value)
        # print("End of subquery tokens")
        subquery_alias = ("sub" * subquery_level) + "query_" + str(subquery_number)
        subquery_decomposed_query = {subquery_alias: {"subqueries": {}}}
        subquery_components = {subquery_alias: {"subqueries": {}}}
        subquery_tokens = subquery_tokens[:-1]
        # print(curr_query)
        # print(query_components)
        res_1, res_2 = self.extract_keywords(subquery_tokens, subquery_decomposed_query, subquery_components, t.parenthesis_level + 1, subquery_level, subquery_alias)
        curr_query["subqueries"][subquery_alias] = res_1[subquery_alias].copy()
        query_components["subqueries"][subquery_alias] = res_2[subquery_alias].copy()
        return curr_query, query_components, i, subquery_alias

    def collapse_from_last_comma(self, curr_query, last_comma_index, last_keyword_token: token.SQLToken, tokens, i, curr_level):
        """
        Collapse tokens from the last comma/token that can be perceived as a comma

        Parameter curr_query: current decomposed query

        Parameter last_comma_index: last comma index

        Parameter last_keyword_token: last keyword token

        Parameter tokens: extracted SQL query tokens

        Parameter i: current index while parsing the SQL query tokens

        Parameter curr_level: current level of query/subquery (where 0 represents highest level query)
        """
        if last_comma_index is not None and last_comma_index != i:
            token_lst = [self.replace_alias_with_expression(curr_query, self.prepend_table_name_to_column(curr_query, t, curr_level)).value.lower() for t in tokens[last_comma_index + 1: i]]
            curr_query[last_keyword_token.value.lower()].append("".join(token_lst))
        return curr_query, last_comma_index

    def extract_keywords(self, tokens, decomposed_query, query_components, parenthesis_level=0, curr_level=0, query_alias="query_1"):
        """
        Extract keywords from extracted SQL query tokens

        Parameter tokens: extracted SQL query tokens

        Parameter decomposed_query: preliminary decomposed query

        Parameter query_components: preliminary query components

        Parameter parenthesis_level: parenthesis level of current query/subquery

        Parameter curr_level: current level of query/subquery (where 0 represents highest level query)

        Parameter query_alias: alias for current query
        """
        last_keyword_token = token.SQLToken()
        last_keyword_token.parenthesis_level = parenthesis_level
        last_comma_index = None
        i = 0
        semicolon_present = False
        curr_component = []
        curr_query = decomposed_query[query_alias]
        curr_query_components = query_components[query_alias]
        # print(decomposed_query)
        # print(query_components)
        # print(curr_query)
        # print(curr_query_components)
        set_op = False
        while i < len(tokens):
            t: token.SQLToken = tokens[i]
            curr_component.append(t.value)
            t = self.prepend_table_name_to_column(curr_query, t, curr_level)
            t = self.replace_alias_with_expression(curr_query, t)
            # print("Value: " + str(t.value))
            # self.print_token_debug_info(t)

            if (t.value.lower() == ";"):
                semicolon_present = True
                curr_query, last_comma_index = self.collapse_from_last_comma(curr_query, last_comma_index, last_keyword_token, tokens, i, curr_level)
                curr_query_components[last_keyword_token.value.lower()] = " ".join(curr_component)
                last_comma_index = i
                print("Done processing the query!")
                i += 1
                continue

            if (t.value == ","):
                curr_query, last_comma_index = self.collapse_from_last_comma(curr_query, last_comma_index, last_keyword_token, tokens, i, curr_level)
                last_comma_index = i

            if (t.is_keyword):
                curr_query, last_comma_index = self.collapse_from_last_comma(curr_query, last_comma_index, last_keyword_token, tokens, i, curr_level)

                # print(t.parenthesis_level)
                # print(last_keyword_token.parenthesis_level)
                if (t.value.lower() == "first" or t.value.lower() == "last" or t.value.lower() == "distinct"):
                    curr_query_components[last_keyword_token.value.lower()] = " ".join(curr_component[:-1])
                    i += 1
                    continue

                # new keyword in same level query
                if (t.parenthesis_level == last_keyword_token.parenthesis_level):

                    if (t.value.lower() == "asc" or t.value.lower() == 'desc'):
                        # print("Reached desc/asc")
                        curr_query["order by"][-1] += " " + t.value.lower()
                        if (last_keyword_token is not None):
                            curr_query_components[last_keyword_token.value.lower()] = " ".join(curr_component[:-1])
                        # print(curr_query)
                        i += 1
                        continue

                    if (t.value.lower() == "as"):
                        curr_query, i, last_comma_index, curr_component = self.handle_as_keyword(curr_query, curr_component, tokens, t, i, last_keyword_token, last_comma_index)
                        i += 1
                        continue

                    curr_query[t.value.lower()] = []
                    curr_query_components[t.value.lower()] = []

                    if (last_keyword_token is not None):
                        curr_query_components[last_keyword_token.value.lower()] = " ".join(curr_component[:-1])

                    curr_component = curr_component[-1:]
                    last_keyword_token = t
                    last_comma_index = i

                    if (t.value.lower() == "where" or t.value.lower() == "having"):
                        clause = t.value.lower()
                        where_tokens = []
                        i += 1
                        t = tokens[i]
                        # a new keyword besides "and", "or", "not" or "date" marks the end of a group of where clauses
                        while ((not t.is_keyword) or (t.parenthesis_level > last_keyword_token.parenthesis_level) or (t.value.lower() in ["and", "or", "not", "date", "between"])):
                            where_tokens.append(t)
                            i += 1
                            if i < len(tokens):
                                t = tokens[i]
                            else:
                                break
                        # i -= 1
                        # print(clause + " clause tokens: ")
                        # for tok in where_tokens:
                        #     print("Value: " + str(tok.value))
                        # print("End of " + clause + " tokens")
                        curr_query, clause_list = self.extract_where_having(curr_query, curr_query_components, where_tokens, [], last_keyword_token, query_alias, curr_level)
                        curr_component.extend(clause_list)
                        last_comma_index = i + 1
                        continue

                    if (t.value.lower() == "union" or t.value.lower() == "intersect" or t.value.lower() == "minus"):
                        print("reached set op")
                        i += 1
                        for t in tokens[i:]:
                            print(t.value.lower())
                        set_op = t.value.lower()
                        t = tokens[i]
                        self.curr_query_num[query_alias][curr_level] += 1
                        next_query_alias = query_alias.split("_")[0] + "_" + str(self.curr_query_num[query_alias][curr_level])
                        decomposed_query[next_query_alias] = {"subqueries": {}}
                        query_components[next_query_alias] = {"subqueries": {}}
                        decomposed_query, query_components = self.extract_keywords(tokens[i:], decomposed_query, query_components, parenthesis_level, curr_level, next_query_alias)
                        curr_query[set_op] = next_query_alias
                        curr_query_components[set_op] = set_op + " " + next_query_alias
                        # print("finished recursive handling")
                        # print(decomposed_query)
                        # print(query_components)
                        break

                # subquery
                else:
                    # print("Reached subquery case")
                    if query_alias not in self.curr_query_num:
                        self.curr_query_num[query_alias] = {}
                    if (curr_level + 1) not in self.curr_query_num[query_alias]:
                        self.curr_query_num[query_alias][curr_level + 1] = 1
                    else:
                        self.curr_query_num[query_alias][curr_level + 1] += 1
                    # self.curr_subquery_num += 1
                    curr_query, curr_query_components, i, subquery_alias = self.extract_subquery(curr_query, curr_query_components, tokens, i, last_keyword_token, t, self.curr_query_num[query_alias][curr_level + 1], curr_level + 1)

            # reached a table alias; then add it to as mapping and remove
            if (t.value in self.tables_aliases):
                # print("reached table alias")
                prev_token: token.SQLToken = tokens[i - 1]
                if (prev_token.is_as_keyword):
                    prev_token = tokens[i - 2]

                if "as" not in curr_query:
                    curr_query["as"] = {}
                curr_query["as"][prev_token.value.lower()] = t.value.lower()
                tokens.pop(i)
                continue

            # region
            # aggregates
            # elif t.value.lower() in self.aggregate_list:
            #     agg = t.value.lower()
            #     i += 1
            #     t = tokens[i]
            #     # a ")" token with same parenthesis level as parent query marks end of aggregate
            #     while t.value != ")" or t.parenthesis_level != last_keyword_token.parenthesis_level:
            #         agg += t.value.lower()
            #         i += 1
            #         t = tokens[i]
            #     i -= 1
            #     agg += t.value.lower()
            #     print("Aggregate: " + agg)
            #     curr_query[last_keyword_token.value].append(agg)
            # endregion

            # region
            # default
            # elif (not t.is_comment and not t.is_punctuation):
            #     print(last_keyword_token.value)
            #     print(curr_query)
            #     curr_query[last_keyword_token.value].append(t.value)
            # endregion

            i += 1

        if (not semicolon_present and not set_op):
            curr_query, last_comma_index = self.collapse_from_last_comma(curr_query, last_comma_index, last_keyword_token, tokens, i, curr_level)
            last_comma_index = i
            if (last_keyword_token is not None):
                curr_query_components[last_keyword_token.value.lower()] = " ".join(curr_component)

        if ("order by" in curr_query):
            if (curr_query["order by"][-1] not in ["asc", "desc"]):
                curr_query["order by"][-1] += " asc"

        if '' in curr_query_components:
            del curr_query_components['']

        decomposed_query[query_alias] = curr_query
        query_components[query_alias] = curr_query_components

        return decomposed_query, query_components

if __name__ == "__main__":
    complex_query = "select l_returnflag, l_linestatus, sum(l_quantity) as sum_qty, sum(l_extendedprice) as sum_base_price,sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge from lineitem where l_shipdate <= date '1998-12-01' group by sum_disc_price, l_linestatus order by sum_disc_price, l_linestatus"
    long_query = """
        select
            ps_partkey PS,
            sum(ps_supplycost * ps_availqty) as value
            from
            partsupp P,
            supplier,
            nation
            where
            P.ps_suppkey = s_suppkey
            and not s_nationkey = n_nationkey
            and n_name = 'GERMANY'
            and ps_supplycost > 20
            and s_acctbal > 10
            group by
            ps_partkey having
                sum(ps_supplycost * ps_availqty) > (
                select
                    sum(ps_supplycost * ps_availqty) * 0.0001000000
                from
                    partsupp P1,
                    supplier S,
                    nation
                where
                    ps_suppkey = s_suppkey
                    and s_nationkey = n_nationkey
                    and n_name = 'GERMANY'
                )
            order by
            value;
    """
    sql_query = "SELECT * FROM customer C, orders O WHERE C.c_custkey = O.o_custkey"
    primary_key_query = "select * from nation where nation.n_nationkey = 3;"
    set_query = "select n_nationkey from nation union select s_nationkey from supplier"
    q20 = """
        SELECT
        s_name,
        s_address
    FROM
        supplier,
        nation
    WHERE
        s_suppkey IN (
            SELECT
                ps_suppkey
            FROM
                partsupp
            WHERE
                ps_partkey IN (
                    SELECT
                        p_partkey
                    FROM
                        part
                    WHERE
                        p_name LIKE 'forest%'
                )
                AND ps_availqty > (
                    SELECT
                        0.5 * SUM(l_quantity)
                    FROM
                        lineitem
                    WHERE
                        l_partkey = ps_partkey
                        AND l_suppkey = ps_suppkey
                        AND l_shipdate >= date '1-1-1994'
                        AND l_shipdate < date '1-1-1994' + '1 year'
                )
        )
        AND s_nationkey = n_nationkey
        AND n_name = 'CANADA'
    ORDER BY
        s_name
        """
    where_subquery = """
    select * from part where p_brand = 'Brand#13' and p_size <> (select max(p_size) from part);
    """

    db = DB()

    preprocessor = PreProcessor(where_subquery, db)
    # preprocessor.print_query_debug_info()
    print(preprocessor.tables)
    print(preprocessor.columns)
    print(preprocessor.decomposed_query)
    print(preprocessor.query_components)
    print(preprocessor.parser.columns_aliases_dict)
    print(preprocessor.parser.columns_aliases_names)
    print(preprocessor.parser.columns_aliases)
    print(preprocessor.parser.tables_aliases)
    print(preprocessor.tables)
    # print(preprocessor.query)

    db.close()

# from decomposed query has aliases due to the table name to alias mapping in as dictionary and working collapse form last comma.


from __future__ import annotations
import glob
import shutil

import subprocess
import sys

import semver

def main(args):
    assert len(args) == 2
    check_cleanliness()
    new_version = bump_version(args[1])
    update_changelog(new_version)
    make_publication_commit(new_version)
    publish_to_pypi()
    prepare_next_version(new_version)

def check_cleanliness():
    not_indexed = subprocess.run(["git", "diff", "--stat", "--exit-code"])
    if not_indexed.returncode != 0:
        print("ERROR: there are non-staged changes. Please 'git add' them and retry.")
        exit(1)

    branch = subprocess.run(["git", "branch", "--show-current"], stdout=subprocess.PIPE, universal_newlines=True, check=True)
    if branch.stdout.strip() != "main":
        input("WARNING: you're not on branch 'main'. Press enter to proceed, Ctrl+C to cancel.")

    not_commited = subprocess.run(["git", "diff", "--stat", "--staged", "--exit-code"], stdout=subprocess.DEVNULL)
    if not_commited.returncode != 0:
        input("WARNING: Some changes are staged but not committed. They will be included in the publication commit. Press enter to proceed, Ctrl+C to cancel.")

def bump_version(part):
    with open("setup.py") as f:
        setup_lines = f.readlines()
    for line in setup_lines:
        if line.startswith("version = "):
            dev_version = semver.VersionInfo.parse(line[11:-2])

    assert dev_version.prerelease == "dev"
    assert dev_version.build is None

    print("Development version:", dev_version)
    if part == "patch":
        new_version = dev_version.replace(prerelease=None)
    elif part == "minor":
        new_version = dev_version.bump_minor()
    elif part == "major":
        new_version = dev_version.bump_major()
    else:
        assert False
    print("New version:", new_version)

    with open("setup.py", "w") as f:
        for line in setup_lines:
            if line.startswith("version = "):
                f.write(f"version = \"{new_version}\"\n")
            else:
                f.write(line)

    return new_version

def update_changelog(new_version):
    # @todo(later) Fix order of changelog: put recent versions first

    tags = subprocess.run(
        ["git", "tag"],
        stdout=subprocess.PIPE, universal_newlines=True,
        check=True,
    ).stdout.splitlines()
    last_tag = None
    for tag in tags:
        assert tag.startswith("v")
        if last_tag is None or semver.compare(tag[1:], last_tag[1:]) > 0:
            last_tag = tag

    log_lines = subprocess.run(
        ["git", "log", "--oneline", "--no-decorate", f"{last_tag}.."],
        stdout=subprocess.PIPE, universal_newlines=True,
        check=True,
    ).stdout.splitlines()

    with open("CHANGELOG.md", "a") as f:
        f.write(f"\n# Version {new_version}\n\n")
        for line in log_lines:
            f.write(f"- {line.split(' ', 1)[1]}\n")

    input("Please edit CHANGELOG.md then press enter to proceed, Ctrl+C to cancel.")

def make_publication_commit(new_version):
    subprocess.run(["git", "add", "setup.py", "CHANGELOG.md"], check=True)
    subprocess.run(["git", "commit", "-m", f"Publish version {new_version}"], stdout=subprocess.DEVNULL, check=True)
    subprocess.run(["git", "tag", f"v{new_version}"], check=True)
    subprocess.run(["git", "push", f"--tags"], check=True)

def publish_to_pypi():
    shutil.rmtree("dist", ignore_errors=True)
    shutil.rmtree("build", ignore_errors=True)
    shutil.rmtree("Chrones.egg-info", ignore_errors=True)
    subprocess.run(["python3", "-m", "build"])
    # The --repository option on next line assumes ~/.pypirc contains:
    # [distutils]
    #   index-servers=
    #     ...
    #     Chrones
    # [Chrones]
    #   repository = https://upload.pypi.org/legacy/
    #   username = __token__
    #   password = ... a token for package Chrones
    subprocess.run(["twine", "upload", "--repository", "Chrones"] + glob.glob("dist/*"), check=True)

def prepare_next_version(new_version):
    with open("setup.py") as f:
        setup_lines = f.readlines()

    next_version = new_version.bump_patch().replace(prerelease="dev")

    with open("setup.py", "w") as f:
        for line in setup_lines:
            if line.startswith("version = "):
                f.write(f"version = \"{next_version}\"\n")
            else:
                f.write(line)

    subprocess.run(["git", "add", "setup.py"], check=True)
    subprocess.run(["git", "commit", "-m", f"Start working on next version"], stdout=subprocess.DEVNULL, check=True)
    subprocess.run(["git", "push"], check=True)

if __name__ == "__main__":
    main(sys.argv)


import plotly.figure_factory as ff
import pandas
import statistics
# read csv
df= pandas.read_csv("data.csv")

# finding mean,median,mode of the height
height=df["Height(Inches)"].tolist()
mean=statistics.mean(height)
mode=statistics.mode(height)
median=statistics.median(height)

# calculating std _deviation
std_deviation=statistics.stdev(height)

#printing mean.median.mode.std_deviation
print("Mean of height of students:",mean)
print("Median of height of students:",median)
print("Mode of height of students:",mode)
print("Standard Deviation of height of students:",std_deviation)
# plot the graph
#figure=ff.create_distplot([height],["height of students"])
#figure.show()
# finding the first std deviation
upper_limit=mean+std_deviation
lower_limit=mean-std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list1=[]
for i in height:
    if lower_limit<i<upper_limit:
        list1.append(i)

per1=(len(list1)/len(height))*100
print("percentage of data lying in 1 st std deviation:", per1)

# finding 2 nd std deviation
upper_limit=mean+2*std_deviation
lower_limit=mean-2*std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list2=[]
for i in height:
    if lower_limit<i<upper_limit:
        list2.append(i)

per2=(len(list2)/len(height))*100
print("percentage of data lying in 2st std deviation:", per2)

# finding 3 nd std deviation
upper_limit=mean+3*std_deviation
lower_limit=mean-3*std_deviation
print (upper_limit,lower_limit)
#run a loop on height if the height lies betwween upper_limit and lower_limit
list3=[]
for i in height:
    if lower_limit<i<upper_limit:
        list3.append(i)

per3=(len(list3)/len(height))*100
print("percentage of data lying in 3 rd std deviation:", per3)