import logging
import math
import numpy as np

import util.annotation
import util.metadata_service
import util.provenance_metadata_store
from util.annotation import AnnotationStore
from engine import app
from ooi_data.postgres.model import Parameter, Stream, NominalDepth
from util.asset_management import AssetManagement
from util.cass import fetch_l0_provenance
from util.common import log_timing, StreamEngineException, StreamKey, MissingDataException, read_size_config, \
    isfillvalue, QC_SUFFIXES
from util.metadata_service import build_stream_dictionary, get_available_time_range
from util.qc_executor import QcExecutor
from util.qartod_qc_executor import QartodQcExecutor
from util.stream_dataset import StreamDataset

log = logging.getLogger()

PRESSURE_DPI = app.config.get('PRESSURE_DPI')
GPS_STREAM_ID = app.config.get('GPS_STREAM_ID')
GPS_LAT_PARAM_ID = app.config.get('GPS_LAT_PARAM_ID')
GPS_LON_PARAM_ID = app.config.get('GPS_LON_PARAM_ID')
LAT_PARAM_ID = app.config.get('LAT_PARAM_ID')
LON_PARAM_ID = app.config.get('LON_PARAM_ID')
INTERP_LAT_PARAM_ID = app.config.get('INTERP_LAT_PARAM_ID')
INTERP_LON_PARAM_ID = app.config.get('INTERP_LON_PARAM_ID')
PRESSURE_DEPTH_PARAM_ID = app.config.get('PRESSURE_DEPTH_PARAM_ID')
INT_PRESSURE_NAME = app.config.get('INT_PRESSURE_NAME')
MAX_DEPTH_VARIANCE = app.config.get('MAX_DEPTH_VARIANCE')
MAX_DEPTH_VARIANCE_METBK = app.config.get('MAX_DEPTH_VARIANCE_METBK')
ASSET_HOST = app.config.get('ASSET_HOST')
SIZE_ESTIMATES = read_size_config(app.config.get('SIZE_CONFIG'))
DEFAULT_PARTICLE_DENSITY = app.config.get('PARTICLE_DENSITY', 1000)  # default bytes/particle estimate
SECONDS_PER_BYTE = app.config.get('SECONDS_PER_BYTE', 0.0000041)  # default bytes/sec estimate
MINIMUM_REPORTED_TIME = app.config.get('MINIMUM_REPORTED_TIME')
DEPTH_FILL = app.config['DEPTH_FILL']
PRESSURE_DEPTH_APPLICABLE_STREAM_KEYS = app.config['PRESSURE_DEPTH_APPLICABLE_STREAM_KEYS']
DEPTH_PARAMETER_NAME = app.config.get('DEPTH_PARAMETER_NAME')

class StreamRequest(object):
    """
    Stores the information from a request, and calculates the required
    parameters and their streams
    """

    def __init__(self, stream_key, parameters, time_range, uflags, qc_parameters=None,
                 limit=None, include_provenance=False, include_annotations=False, strict_range=False,
                 request_id='', collapse_times=False, execute_dpa=True, require_deployment=True):

        if not isinstance(stream_key, StreamKey):
            raise StreamEngineException('Received no stream key', status_code=400)

        # Inputs
        self.request_id = request_id
        self.stream_key = stream_key
        self.requested_parameters = parameters
        self.time_range = time_range
        self.uflags = uflags
        self.qc_executor = QcExecutor(qc_parameters, self)
        self.qartod_qc_executor = QartodQcExecutor(self)
        self.limit = limit
        self.include_provenance = include_provenance
        self.include_annotations = include_annotations
        self.strict_range = strict_range
        self.execute_dpa = execute_dpa
        self.require_deployment = require_deployment

        # Internals
        self.asset_management = AssetManagement(ASSET_HOST, request_id=self.request_id)
        self.stream_parameters = {}
        self.unfulfilled = set()
        self.datasets = {}
        self.external_includes = {}
        self.annotation_store = AnnotationStore()

        self._initialize()

        if collapse_times:
            self._collapse_times()

    def __repr__(self):
        return str(self.__dict__)

    @property
    def needs_cc(self):
        """
        Return the list of calibration coefficients necessary to compute all data products for this request
        :return:
        """
        stream_list = []
        for sk in self.stream_parameters:
            needs = list(sk.stream.needs_cc)
            d = sk.as_dict()
            d['coefficients'] = needs
            stream_list.append(d)
        return stream_list

    @log_timing(log)
    def fetch_raw_data(self):
        """
        Fetch the source data for this request
        :return:
        """
        # Start fetching calibration data from Asset Management
        am_events = {}
        am_futures = {}
        for stream_key in self.stream_parameters:
            refdes = '-'.join((stream_key.subsite, stream_key.node, stream_key.sensor))
            am_futures[stream_key] = self.asset_management.get_events_async(refdes)

        # Resolve calibration data futures and attach to instrument data
        for stream_key in am_futures:
            events = am_futures[stream_key].result()
            am_events[stream_key] = events

        # Start fetching instrument data
        for stream_key, stream_parameters in self.stream_parameters.iteritems():
            other_streams = set(self.stream_parameters)
            other_streams.remove(stream_key)

            # Pull a data point on either side of the requested time range only for supporting streams
            # to improve interpolation
            should_pad = stream_key != self.stream_key

            if not stream_key.is_virtual:
                log.debug('<%s> Fetching raw data for %s', self.request_id, stream_key.as_refdes())
                sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id)
                sd.events = am_events[stream_key]
                try:
                    sd.fetch_raw_data(self.time_range, self.limit, should_pad)
                    self.datasets[stream_key] = sd
                except MissingDataException as e:
                    if stream_key == self.stream_key:
                        raise MissingDataException("Query returned no results for primary stream")
                    elif stream_key.stream in self.stream_key.stream.source_streams:
                        raise MissingDataException("Query returned no results for source stream")
                    else:
                        log.error('<%s> %s', self.request_id, e.message)

            else:
                log.debug('<%s> Creating empty dataset for virtual stream: %s',
                          self.request_id, stream_key.as_refdes())
                sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id)
                sd.events = am_events[stream_key]
                self.datasets[stream_key] = sd

        self._exclude_flagged_data()
        self._exclude_nondeployed_data()

        # Verify data still exists after masking virtual
        message = 'Query returned no results for %s stream (due to deployment or annotation mask)'
        if self.stream_key.is_virtual:
            found_streams = [stream.stream for stream in self.datasets
                             if self.datasets[stream]]
            if not any(stream in self.stream_key.stream.source_streams for stream in found_streams):
                raise MissingDataException(message % 'source')
        # real
        else:
            primary_stream_dataset = self.datasets[self.stream_key]
            if not primary_stream_dataset.datasets:
                raise MissingDataException(message % 'primary')

        # Remove any empty, non-virtual supporting datasets
        for stream_key in list(self.datasets):
            if not stream_key.is_virtual:
                if not self.datasets[stream_key].datasets:
                    del self.datasets[stream_key]

        # Remove pressure_depth if it is not applicable to prevent misguided uses of rubbish
        # pressure_depth data when pressure should be interpolated from the CTD stream
        for stream_key in list(self.datasets):
            if not self._is_pressure_depth_valid(stream_key) and self.datasets[stream_key].datasets:
                for _, ds in self.datasets[stream_key].datasets.iteritems():
                    pressure_depth = Parameter.query.get(PRESSURE_DEPTH_PARAM_ID)
                    if pressure_depth.name in ds:
                        del ds[pressure_depth.name]

    @staticmethod
    def missing_params_of_dataset_depend_on_another(dataset, another):
        if not (dataset.missing and another.missing):
            return False
        missing_param_dependencies = set()
        for param_dependencies_dict in dataset.missing.values():
            for param_dependencies in param_dependencies_dict.values():
                for dependency in param_dependencies.values():
                    missing_param_dependencies.add(dependency)

        for param_list in another.params.values():
            for param in param_list:
                if (another.stream_key.stream, param) in missing_param_dependencies:
                    return True
        return False

    @staticmethod
    def compare_datasets_by_missing(ds1, ds2):
        if ds1.stream_key.is_virtual:
            return 1
        if ds2.stream_key.is_virtual:
            return -1
        if StreamRequest.missing_params_of_dataset_depend_on_another(ds1, ds2):
            return 1
        if StreamRequest.missing_params_of_dataset_depend_on_another(ds2, ds1):
            return -1
        return 0

    def calculate_derived_products(self):
        # Calculate all internal-only data products
        for sk in self.datasets:
            if not sk.is_virtual:
                self.datasets[sk].calculate_all(ignore_missing_optional_params=False)

        # Sort the datasets in case a derived parameter requires an another external parameter to be calculated first
        sorted_datasets = sorted(self.datasets.values(), cmp=StreamRequest.compare_datasets_by_missing)
        sorted_stream_keys = [ds.stream_key for ds in sorted_datasets]

        # Allow each StreamDataset to interpolate any needed non-virtual parameters from the other datasets
        # Then calculate any data products which required only non-virtual external input.
        for sk in sorted_stream_keys:
            if not sk.is_virtual:
                self.datasets[sk].interpolate_needed(self.datasets, interpolate_virtual=False)
                self.datasets[sk].calculate_all(ignore_missing_optional_params=True)

        for sk in self.datasets:
            if sk.is_virtual:
                for poss_source in self.datasets:
                    if poss_source.stream in sk.stream.source_streams:
                        self.datasets[sk].calculate_virtual(self.datasets[poss_source])
                        break

        # Allow each StreamDataset to interpolate any needed virtual parameters from the other datasets
        # Then calculate any data products which required virtual external input.
        for sk in self.datasets:
            if not sk.is_virtual:
                self.datasets[sk].interpolate_needed(self.datasets, interpolate_virtual=True)
                self.datasets[sk].calculate_all()

        for sk in self.datasets:
            self.datasets[sk].fill_missing()

    def execute_qc(self):
        self._run_qc()

    def execute_qartod_qc(self):
        self._run_qartod_qc()

    def insert_provenance(self):
        self._insert_provenance()
        self._add_location()

    @log_timing(log)
    def _run_qc(self):
        # execute any QC
        for sk, stream_dataset in self.datasets.iteritems():
            for param in sk.stream.parameters:
                for dataset in stream_dataset.datasets.itervalues():
                    self.qc_executor.qc_check(param, dataset)

    @log_timing(log)
    def _run_qartod_qc(self):
        self.qartod_qc_executor.execute_qartod_tests()

    # noinspection PyTypeChecker
    def _insert_provenance(self):
        """
        Insert all source provenance for this request. This is dependent on the data already having been fetched.
        :return:
        """
        if self.include_provenance:
            for stream_key in self.stream_parameters:
                if stream_key in self.datasets:
                    self.datasets[stream_key].insert_instrument_attributes()
                    prov_metadata = self.datasets[stream_key].provenance_metadata
                    prov_metadata.add_query_metadata(self, self.request_id, 'JSON')
                    prov_metadata.add_instrument_provenance(
                        stream_key,
                        [deployment_event for deployment_event in self.datasets[stream_key].events.events
                            if deployment_event["deploymentNumber"] in self.datasets[stream_key].datasets.keys()])
                    for deployment, dataset in self.datasets[stream_key].datasets.iteritems():
                        if 'provenance' in dataset:
                            provenance = dataset.provenance.values.astype('str')
                            prov = fetch_l0_provenance(stream_key, provenance, deployment)
                            prov_metadata.update_provenance(prov, deployment)

    def insert_annotations(self):
        """
        Insert all annotations for this request.
        """
        for stream_key in self.stream_parameters:
            self.annotation_store.add_query_annotations(stream_key, self.time_range)

    def _exclude_flagged_data(self):
        """
        Exclude data from datasets based on annotations
        TODO: Future optimization, avoid querying excluded data when possible
        :return:
        """
        for stream_key, stream_dataset in self.datasets.iteritems():
            stream_dataset.exclude_flagged_data(self.annotation_store)

    def _exclude_nondeployed_data(self):
        """
        Exclude data from datasets that are outside of deployment dates
        :return:
        """
        for stream_key, stream_dataset in self.datasets.iteritems():
            stream_dataset.exclude_nondeployed_data(self.require_deployment)

    def _is_pressure_depth_valid(self, stream_key):
        """
        Returns true if the stream key corresponds to an instrument which should use pressure_depth instead of
        int_ctd_pressure. Many streams have a pressure_depth parameter which is filled with unusable data. This
        function handles determining when the pressure_depth parameter is usable based on a lookup.
        """
        stream_key = stream_key.as_dict()

        for candidate_key in PRESSURE_DEPTH_APPLICABLE_STREAM_KEYS:
            # ignore fields in candidate_key which are set to None as None means wildcard
            fields_to_match = {k: candidate_key[k] for k in candidate_key if candidate_key[k] != None}
            # compute the difference in the non-None fields
            mismatch = {k: stream_key[k] for k in fields_to_match if stream_key[k] != candidate_key[k]}
            if not mismatch:
                return True
        return False

    def import_extra_externals(self):
        # import any other required "externals" into all datasets
        for source_sk in self.external_includes:
            if source_sk in self.datasets:
                for param in self.external_includes[source_sk]:
                    for target_sk in self.datasets:
                        self.datasets[target_sk].interpolate_into(source_sk, self.datasets[source_sk], param)

        # determine if there is a pressure parameter available (9328) - should be none when _is_pressure_depth_valid evaluates to True
        pressure_params = [(sk, param) for sk in self.external_includes for param in self.external_includes[sk]
                           if param.data_product_identifier == PRESSURE_DPI]

        if not pressure_params:
            return

        # integrate the pressure parameter into the stream
        pressure_key, pressure_param = pressure_params.pop()
        pressure_name = '-'.join((pressure_key.stream.name, pressure_param.name))

        if pressure_key not in self.datasets:
            return

        # interpolate CTD pressure
        self.datasets[self.stream_key].interpolate_into(pressure_key, self.datasets.get(pressure_key), pressure_param)

        for deployment in self.datasets[self.stream_key].datasets:
            ds = self.datasets[self.stream_key].datasets[deployment]

            # If we used the CTD pressure, then rename it to the configured final name (e.g. 'int_ctd_pressure')
            if pressure_name in ds.data_vars:
                pressure_value = ds.data_vars[pressure_name]
                del ds[pressure_name]
                pressure_value.name = INT_PRESSURE_NAME
                self.datasets[self.stream_key].datasets[deployment][INT_PRESSURE_NAME] = pressure_value

        # determine if there is a depth parameter available
        # depth is computed from pressure, so look for it in the same stream
        depth_key, depth_param = self.find_stream(self.stream_key, tuple(Parameter.query.filter(Parameter.name == DEPTH_PARAMETER_NAME)), pressure_key.stream)

        if not depth_param:
            return

        if depth_key not in self.datasets:
            return

        # update external_includes for any post processing that looks at it - pressure was already handled, but depth was not
        self.external_includes.setdefault(depth_key, set()).add(depth_param)

        # interpolate depth computed from CTD pressure
        self.datasets[self.stream_key].interpolate_into(depth_key, self.datasets.get(depth_key), depth_param)

    def rename_parameters(self):
        """
        Some internal parameters are not well suited for output data files (e.g. NetCDF). To get around this, the
        Parameter class has a netcdf_name attribute for use in output files. This function performs the translations
        from internal name (Parameter.name) to output name (Parameter.netcdf_name).
        """
        # build a mapping from original parameter name to netcdf_name
        parameter_name_map = {x.name: x.netcdf_name for x in self.requested_parameters if x.netcdf_name != x.name}
        for external_stream_key in self.external_includes:
            for parameter in [x for x in self.external_includes[external_stream_key] if x.netcdf_name != x.name]:
                long_parameter_name = external_stream_key.stream_name + "-" + parameter.name
                # netcdf_generator.py is expecting the long naming scheme
                parameter_name_map[long_parameter_name] = external_stream_key.stream_name + "-" + parameter.netcdf_name
                #make sure netcdf_name parameter is used for co-located CTD's (15486)
                parameter_name_map[parameter.name] = parameter.netcdf_name
        # pass the parameter mapping to the annotation store for renaming there
        if self.include_annotations:
            self.annotation_store.rename_parameters(parameter_name_map)

        # generate possible qc/qartod renamings too so they will be handled in the update loop below
        qartod_name_map = {}
        for suffix in ['_qc_executed', '_qc_results', '_qartod_executed', '_qartod_results']:
            qartod_name_map.update({name + suffix: netcdf_name + suffix for name, netcdf_name in
                                       parameter_name_map.iteritems()})
        parameter_name_map.update(qartod_name_map)

        # update parameter names
        for stream_key, stream_dataset in self.datasets.iteritems():
            for deployment, ds in stream_dataset.datasets.iteritems():
                for key in [x for x in parameter_name_map.keys() if x in ds]:
                    # add an attribute to help users associate the renamed variable with its original name
                    ds[key].attrs['alternate_parameter_name'] = key
                    # rename
                    ds.rename({key: parameter_name_map[key]}, inplace=True)

    def _add_location(self):
        log.debug('<%s> Inserting location data for all datasets', self.request_id)
        for stream_dataset in self.datasets.itervalues():
            stream_dataset.add_location()

    def _locate_externals(self, parameters):
        """
        Locate external data sources for the given list of parameters
        :param parameters: list of type Parameter
        :return: found parameters as dict(StreamKey, Parameter), unfulfilled parameters as set(Parameter)
        """
        log.debug('<%s> _locate_externals: %r', self.request_id, parameters)
        # A set of tuples of the dependant stream and the required parameters that it depends on.
        # Initially, it is just the requested stream and external parameters that it needs.
        external_to_process = set()
        for param in parameters:
            external_to_process.add((self.stream_key, param))
        found = {}
        external_unfulfilled = set()
        stream_parameters = {}

        def process_found_stream(stream_key, parameter):
            """
            Internal subroutine to process each found stream/parameter
            :param stream_key: StreamKey found by find_stream
            :param parameter: Parameter inside found stream
            :return: None
            """
            found.setdefault(stream_key, set()).add(parameter)
            sk_needs_internal = stream_key.stream.needs_internal([parameter])
            sk_needs_external = stream_key.stream.needs_external([parameter])
            log.debug('<%s> _locate_externals FOUND INT: %r %r', self.request_id,
                      stream_key.as_refdes(), sk_needs_internal)
            log.debug('<%s> _locate_externals FOUND EXT: %r %r', self.request_id,
                      stream_key.as_refdes(), sk_needs_external)

            # Add externals not yet processed to the to_process set
            for sub_need in sk_needs_external:
                if sub_need not in external_unfulfilled:
                    external_to_process.add((stream_key, sub_need))
            # Add internal parameters to the corresponding stream set
            stream_parameters.setdefault(stream_key, set()).update(sk_needs_internal)

        while external_to_process:
            # Pop an external from the list of externals to process
            stream_key, external = external_to_process.pop()
            stream, poss_params = external
            # all non-virtual streams define PD7, skip
            if poss_params[0].id == 7:
                continue
            log.debug('<%s> _locate_externals: STREAM: %r POSS_PARAMS: %r', self.request_id, stream, poss_params)
            found_sk, found_param = self.find_stream(stream_key, poss_params, stream=stream)
            if found_sk:
                process_found_stream(found_sk, found_param)
            else:
                external_unfulfilled.add(external)

        return stream_parameters, found, external_unfulfilled

    @log_timing(log)
    def _get_mobile_externals(self):
        """
        For mobile assets, build the set of externals necessary to provide location data
        :return: set((Stream, (Parameter,)))
        """
        external_to_process = set()
        if self.stream_key.is_mobile and not self._is_pressure_depth_valid(self.stream_key):
            # add pressure parameter
            external_to_process.add((None, tuple(Parameter.query.filter(
                Parameter.data_product_identifier == PRESSURE_DPI).all())))
            # do NOT add depth parameter here; we want to make sure it comes from the
            # same stream as the pressure parameter (which has not been determined yet)
        if self.stream_key.is_glider:
            gps_stream = Stream.query.get(GPS_STREAM_ID)
            external_to_process.add((gps_stream, (Parameter.query.get(GPS_LAT_PARAM_ID),)))
            external_to_process.add((gps_stream, (Parameter.query.get(GPS_LON_PARAM_ID),)))
            external_to_process.add((gps_stream, (Parameter.query.get(LAT_PARAM_ID),)))
            external_to_process.add((gps_stream, (Parameter.query.get(LON_PARAM_ID),)))
            external_to_process.add((gps_stream, (Parameter.query.get(INTERP_LAT_PARAM_ID),)))
            external_to_process.add((gps_stream, (Parameter.query.get(INTERP_LON_PARAM_ID),)))
        return external_to_process

    @log_timing(log)
    def _initialize(self):
        """
        Initialize stream request. Computes data sources / parameters
        :return:
        """
        # Build our list of internally requested parameters
        if self.requested_parameters:
            internal_requested = [p for p in self.stream_key.stream.parameters if p.id in self.requested_parameters]
        else:
            internal_requested = self.stream_key.stream.parameters

        pressure_depth = Parameter.query.get(PRESSURE_DEPTH_PARAM_ID)
        if pressure_depth in internal_requested and not self._is_pressure_depth_valid(self.stream_key):
            log.debug('<%s> removing invalid pressure_depth from requested parameters', self.request_id)
            internal_requested.remove(pressure_depth)
            log.debug('<%s> removing invalid depth computed from invalid pressure_depth from requested parameters', self.request_id)
            for param in internal_requested:
                if param.name == DEPTH_PARAMETER_NAME:
                    internal_requested.remove(param)

        self.requested_parameters = internal_requested

        # Identify internal parameters needed to support this query
        primary_internals = self.stream_key.stream.needs_internal(internal_requested)
        log.debug('<%s> primary stream internal needs: %r', self.request_id, primary_internals)
        self.stream_parameters[self.stream_key] = primary_internals

        if self.execute_dpa:
            # Identify external parameters needed to support this query
            external_to_process = self.stream_key.stream.needs_external(internal_requested)
            log.debug('<%s> primary stream external needs: %r', self.request_id, external_to_process)
            if external_to_process:
                stream_parameters, found, external_unfulfilled = self._locate_externals(external_to_process)
                for sk in stream_parameters:
                    self.stream_parameters.setdefault(sk, set()).update(stream_parameters[sk])
                self.unfulfilled = external_unfulfilled
                for sk in found:
                    self.external_includes.setdefault(sk, set()).update(found[sk])

            # Now identify any parameters needed for mobile assets
            external_to_process = self._get_mobile_externals()
            if external_to_process:
                stream_parameters, found, external_unfulfilled = self._locate_externals(external_to_process)
                for sk in stream_parameters:
                    self.stream_parameters.setdefault(sk, set()).update(stream_parameters[sk])
                self.unfulfilled = self.unfulfilled.union(external_unfulfilled)
                for sk in found:
                    self.external_includes.setdefault(sk, set()).update(found[sk])

            if self.unfulfilled:
                log.warn('<%s> Unable to find sources for the following params: %r',
                         self.request_id, self.unfulfilled)

    @log_timing(log)
    def _collapse_times(self):
        """
        Collapse request times to match available data
        :return:
        """
        if self.stream_key.is_virtual:
            # collapse to smallest of all source streams
            tr = self.time_range.copy()
            for sk in self.stream_parameters:
                if sk.is_virtual:
                    continue
                tr = tr.collapse(get_available_time_range(sk))
            new_time_range = self.time_range.collapse(tr)
            if new_time_range != self.time_range:
                log.info('<%s> Collapsing requested time range: %s to available time range: %s',
                         self.request_id, self.time_range, new_time_range)
                self.time_range = new_time_range

        else:
            # collapse to primary stream
            new_time_range = self.time_range.collapse(get_available_time_range(self.stream_key))
            if new_time_range != self.time_range:
                log.info('<%s> Collapsing requested time range: %s to available time range: %s',
                         self.request_id, self.time_range, new_time_range)
                self.time_range = new_time_range

    @log_timing(log)
    def find_stream(self, stream_key, poss_params, stream=None):
        log.debug('find_stream(%r, %r, %r)', stream_key, poss_params, stream)
        subsite = stream_key.subsite
        node = stream_key.node
        sensor = stream_key.sensor
        stream_dictionary = build_stream_dictionary()

        param_streams = []
        for p in poss_params:
            if stream is None:
                param_streams.append((p, [s.name for s in p.streams]))
            else:
                param_streams.append((p, [stream.name]))

        # First, try to find the stream on the same sensor
        for param, search_streams in param_streams:
            sk = self._find_stream_same_sensor(stream_key, search_streams, stream_dictionary)
            if sk:
                return sk, param

        # Attempt to find an instrument at the same depth (if not mobile)
        if not stream_key.is_mobile:
            nominal_depth = NominalDepth.get_nominal_depth(subsite, node, sensor)
            if nominal_depth is not None:
                co_located = nominal_depth.get_colocated_subsite()
                for param, search_streams in param_streams:
                    sk = self._find_stream_from_list(stream_key, search_streams, co_located, stream_dictionary)
                    if sk:
                        return sk, param

        # Attempt to find an instrument on the same node
        for param, search_streams in param_streams:
            sk = self._find_stream_same_node(stream_key, search_streams, stream_dictionary)
            if sk:
                return sk, param

        # Not found at same depth, attempt to find nearby (if not mobile)
        if not stream_key.is_mobile:
            nominal_depth = NominalDepth.get_nominal_depth(subsite, node, sensor)
            if nominal_depth is not None:
                max_depth_var = MAX_DEPTH_VARIANCE_METBK if 'METBK' in sensor else MAX_DEPTH_VARIANCE
                nearby = nominal_depth.get_depth_within(max_depth_var)
                for param, search_streams in param_streams:
                    sk = self._find_stream_from_list(stream_key, search_streams, nearby, stream_dictionary)
                    if sk:
                        return sk, param

        return None, None

    @staticmethod
    def _find_stream_same_sensor(stream_key, streams, stream_dictionary):
        """
        Given a primary source, attempt to find one of the supplied streams from the same instrument
        :param stream_key:
        :param streams:
        :return:
        """
        log.debug('_find_stream_same_sensor(%r, %r, STREAM_DICTIONARY)', stream_key, streams)
        method = stream_key.method
        subsite = stream_key.subsite
        node = stream_key.node
        sensor = stream_key.sensor

        # Search the same reference designator
        for stream in streams:
            sensors = stream_dictionary.get(stream, {}).get(method, {}).get(subsite, {}).get(node, [])
            if sensor in sensors:
                return StreamKey.from_dict({
                    "subsite": subsite,
                    "node": node,
                    "sensor": sensor,
                    "method": method,
                    "stream": stream
                })

    @staticmethod
    def _find_stream_from_list(stream_key, streams, sensors, stream_dictionary):
        log.debug('_find_stream_from_list(%r, %r, %r, STREAM_DICTIONARY)', stream_key, streams, sensors)
        method = stream_key.method
        subsite = stream_key.subsite
        designators = [(c.subsite, c.node, c.sensor) for c in sensors]

        for stream in streams:
            for method in StreamRequest._get_potential_methods(method, stream_dictionary):
                subsite_dict = stream_dictionary.get(stream, {}).get(method, {}).get(subsite, {})
                for _node in subsite_dict:
                    for _sensor in subsite_dict[_node]:
                        des = (subsite, _node, _sensor)
                        if des in designators:
                            return StreamKey.from_dict({
                                "subsite": subsite,
                                "node": _node,
                                "sensor": _sensor,
                                "method": method,
                                "stream": stream
                            })

    @staticmethod
    def _find_stream_same_node(stream_key, streams, stream_dictionary):
        """
        Given a primary source, attempt to find one of the supplied streams from the same instrument,
        same node or same subsite
        :param stream_key: StreamKey - defines the source of the primary stream
        :param streams: List - list of target streams
        :return: StreamKey if found, otherwise None
        """
        log.debug('_find_stream_same_node(%r, %r, STREAM_DICTIONARY)', stream_key, streams)
        method = stream_key.method
        subsite = stream_key.subsite
        node = stream_key.node

        for stream in streams:
            for method in StreamRequest._get_potential_methods(method, stream_dictionary):
                sensors = stream_dictionary.get(stream, {}).get(method, {}).get(subsite, {}).get(node, [])
                if sensors:
                    return StreamKey.from_dict({
                        "subsite": subsite,
                        "node": node,
                        "sensor": sensors[0],
                        "method": method,
                        "stream": stream
                    })

    @staticmethod
    def _get_potential_methods(method, stream_dictionary):
        """
        When trying to resolve streams, an applicable stream may have a subtlely different method
        (e.g. 'recovered_host' vs. 'recovered_inst'). This function is used to identify all related methods
        within a stream dictionary so that streams can be resolved properly despite these minor differences.
        """
        method_category = None
        if "streamed" in method:
            method_category = "streamed"
        elif "recovered" in method:
            method_category = "recovered"
        elif "telemetered" in method:
            method_category = "telemetered"

        if not method_category:
            log.warn("<%s> Unexpected method, %s, encountered during stream resolution."
                     " Only resolving streams whose methods match exactly.", method)
            return method

        valid_methods = []
        for stream in stream_dictionary:
            for method in stream_dictionary[stream]:
                if method_category in method and "bad" not in method:
                    valid_methods.append(method)
        return valid_methods

    def interpolate_from_stream_request(self, stream_request):
        source_sk = stream_request.stream_key
        target_sk = self.stream_key
        if source_sk in stream_request.datasets and target_sk in self.datasets:
            for param in stream_request.requested_parameters:
                self.datasets[target_sk].interpolate_into(source_sk, stream_request.datasets[source_sk], param)
                self.external_includes.setdefault(source_sk, set()).add(param)

    def compute_request_size(self, size_estimates=SIZE_ESTIMATES):
        """
        Estimate the time and size of a NetCDF request based on previous data.
        :param size_estimates:  dictionary containing size estimates for each stream
        :return:  size estimate (in bytes) - also populates self.size_estimate
        """
        default_size = DEFAULT_PARTICLE_DENSITY  # bytes / particle
        size_estimate = sum((size_estimates.get(stream.stream_name, default_size) *
                             util.metadata_service.get_particle_count(stream, self.time_range)
                             for stream in self.stream_parameters))

        return int(math.ceil(size_estimate))

    @staticmethod
    def compute_request_time(file_size):
        return max(MINIMUM_REPORTED_TIME, file_size * SECONDS_PER_BYTE)


#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Author    : Nam Zeng
# @Time      : 2018/12/6 10:38
# @Desc      : 用牛顿插值法拟合经过给定点的曲线

#  x0 | y0
#  x1 | y1  y10
#  x2 | y2  y21   y210

import matplotlib.pyplot as plt
import numpy as np

def newton_interpolate(x, y):
    """
    :param x: 点集的x坐标集x[]
    :param y: 点集的y坐标集y[]
    :return: 拟合出的曲线系数列表coe[]
    """

    n = len(x)
    mat = np.zeros((n, n), dtype=float)

    # 将矩阵的第一列元素分别赋值为y[i]
    for i in range(n):
        mat[i][0] = y[i]

    for i in range(1, n):  # 对于第1~n-1列
        for j in range(i, n):  # 每列从上到下填入各差商值，即从三角形的斜边到直角边
            mat[j][i] = (mat[j][i - 1] - mat[j - 1][i - 1]) / (x[j] - x[j - i])

    # 输出对角线系数
    coe = []
    for i in range(n):
        coe.append(mat[i][i])

    return coe

def draw(x, y, coe):
    # 画图
    plt.title("Newton Interpolation")
    plt.scatter(x, y, label="discrete data", color='green')
    ltx = np.linspace(-5, 5, 300)
    lty = [coe[0] + coe[1] * (xi - x[0]) + coe[2] * (xi - x[0]) * (xi - x[1])
           + coe[3] * (xi - x[0]) * (xi - x[1]) * (xi - x[2])
           for xi in ltx]
    plt.plot(ltx, lty, label="fitting curve", color='red')
    plt.legend(loc="upper right")
    plt.show()

if __name__ == '__main__':
    # 设置拟合数据,求三次牛顿多项式
    x = [3, 2, -1, -2]
    y = [-23, -4, 5, 12]
    coe = newton_interpolate(x, y)
    print("拟合曲线的系数(b0~bn-1)：\n", coe)
    draw(x, y, coe)


#!/usr/bin/python
import sys
import numpy as np
import scipy.signal as signal
from scipy.io import wavfile
import sklearn as sk
import librosa
import heartpy as hp
import soundfile as sf
from scipy.fft import fft, ifft, fftfreq
import matplotlib.pyplot as plt

src_signal = librosa.load("LAB5_500HzFHR.wav", sr=None)

sample_rate = src_signal[1]
source_signal = src_signal[0]

N = source_signal.size

Ts = 1.0 / sample_rate
fft_src = fft(source_signal)

yf = 2.0/N * np.abs(fft_src[:N//2])

x = np.linspace(0.0, N*Ts, N)
xf = np.linspace(0.0, 1.0/(2.0*Ts), N//2)

b, a =  signal.iirdesign(1, 3, gpass=1, gstop=40, fs=sample_rate)
signal_filtered = signal.lfilter(b, a, source_signal)

freq_s = fftfreq(N, Ts)[:N//2]
fft_flt = fft(signal_filtered)

plt.plot(freq_s, yf)
plt.show(block=False)

# print(b)
# print(a)
plt.plot(freq_s, 2.0/N * np.abs(fft_flt[0:N//2]))
plt.show(block=False)

plt.plot(x, signal_filtered)
plt.show(block=False)

# print(signal_filtered)

sf.write("test.wav", signal_filtered, sample_rate)

working_data, measures = hp.process(source_signal, 500.0)
working_data_flt, measures_flt = hp.process(signal_filtered, 500.0)

bpm_flt = hp.plotter(working_data, measures, show=True)
plt.show(block=False)

# hp.plotter(working_data, measures)
bpm_flt = hp.plotter(working_data_flt, measures_flt, show=True)
plt.show(block=False)

y1 = 0.01 * np.sin(1.2 * x * 2 * np.pi)
y2 = -0.005 * np.sin(1.6 * x * 2 * np.pi)
Y = y1+y2
working_data_t, measures_t = hp.process(Y, 500.0)
bpm_flt = hp.plotter(working_data_t, measures_t, show=True)
plt.show(block=False)
fig, (ax1, ax2) = plt.subplots(2)

ax1.plot(x, y1+y2)
ax2.plot(x, signal_filtered)

plt.show()


import numpy as np
import torch
from torch._C import dtype
import torch.nn as nn
import random
from filters import BP_filter
import matplotlib.pyplot as plt

class Random_shift(nn.Module):
# Randomly shifts the track
    def __init__(self, shift_max):
        super().__init__()
        self.shift_max = shift_max

    def forward(self, noisy_track, clean_track):
        shift = random.randint(0, self.shift_max)
        noisy_track = torch.roll(noisy_track, shift, dims=-1)
        clean_track = torch.roll(clean_track, shift, dims=-1)
        return noisy_track, clean_track

class Remix(nn.Module):
# Randomly shifts the noise in the track
    def forward(self, noisy_track, clean_track):
        noise = noisy_track - clean_track
        perm = torch.randperm(noise.size()[-1])
        noisy_track[-1][-1][:] = clean_track[-1][-1][:] + noise[-1][-1][perm]
        return noisy_track, clean_track

class Band_mask(nn.Module):
# Maskes bands of frequencies
    def __init__(self, sample_rate, min_freq, max_freq):
        super().__init__()
        self.sample_rate = sample_rate
        self.filter = BP_filter(min_freq, max_freq)

    def forward(self, noisy_track, clean_track):
        noisy_track = self.filter.forward(noisy_track)
        clean_track = self.filter.forward(clean_track)
        return noisy_track, clean_track

def test():
    x = torch.randn(1,1,1000)
    y = torch.randn(1,1,1000)
    t = np.linspace(0,1000, num=1000, dtype=np.float32)

    shift = Random_shift(10)
    remix = Remix()
    band_mask = Band_mask(48000, 0.1, 0.15)
    shifted_x, shifted_y = shift.forward(x, y)
    remixed_x, remixed_y = remix.forward(x, y)
    masked_x, masked_y = band_mask.forward(x, y)

    plt.figure()
    plt.title('x')
    plt.plot(t, x[-1][-1])
    plt.figure()
    plt.title('shifted x')
    plt.plot(t, shifted_x[-1][-1])
    plt.figure()
    plt.title('remixed x')
    plt.plot(t, remixed_x[-1][-1])
    plt.figure()
    plt.title('masked x')
    plt.plot(t, masked_x[-1][-1])
    plt.show()

if __name__ == '__main__':
    test()


import random
import json

import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader, random_split, ConcatDataset
import pandas as pd
import os

import torchaudio

from dataset import snoring_preprocess
from dataset import dataset_utils

# TODO: some inspections for arguments
# TODO: ABC? (to a meta dataset)
# TODO: split rate for testing --> train and testing dataset building need to be separated
# TODO: Considering a reusing dataset builder (Can we minimize the building effort?)

class MetaDataset(Dataset):
    def __init__(self, loader, data_refs, dataset_name=None, transform=None, target_loader=None):
        self.loader = loader
        self.data_refs = data_refs
        self.dataset_name = dataset_name
        self.transform = transform
        self.target_loader = target_loader
        self.input_key = 'input'
        self.target_key = 'target'
        self.log_dataset()
        self.dataflow_func = self.get_dataflow_format()

    def __len__(self):
        return len(self.data_refs)

    def __getitem__(self, idx):
        return self.dataflow_func(idx)

    def get_dataflow_format(self):
        first_sample = self.data_refs[0]
        if isinstance(first_sample, dict):
            assert self.input_key in first_sample, 'No input data reference'
            if self.target_key in first_sample:
                return self.get_datapair_from_dict_seq
            else:
                return self.get_input_from_dict_seq
        elif isinstance(first_sample, str):
            return self.get_input_from_str_seq
        else:
            raise ValueError('Unknown reference format.')

    def get_input_from_str_seq(self, idx):
        input_data = self.loader(self.data_refs[idx])
        if self.transform is not None:
            input_data = self.transform(input_data)
        return input_data

    def get_input_from_dict_seq(self, idx):
        input_data = self.loader(self.data_refs[idx]['input'])
        if self.transform is not None:
            input_data = self.transform(input_data)
        return input_data

    def get_datapair_from_dict_seq(self, idx):
        input_data = self.loader(self.data_refs[idx]['input'])
        if self.target_loader is not None:
            target = self.target_loader(self.data_refs[idx]['target'])
        else:
            target = self.data_refs[idx]['target']
        if self.transform is not None:
            input_data, target = self.transform(input_data, target)
        return {'input': input_data, 'target': target}

    def log_dataset(self):
        pass

    def get_ref_mapping():
        pass

def wav_loader(filename, sr=None, channels=1):
    y = dataset_utils.get_pydub_sound(
        filename, 'wav', sr, channels)
    sr = y.frame_rate
    waveform = np.array(y.get_array_of_samples(), np.float32)
    return waveform, sr

# def build_single_snoring_dataset(
#     data, data_refs, loader, data_transform=None, dataset_name=None):
#     data_config = {
#         'data': data,
#         'data_refs': data_refs,
#         'loader': loader,
#         'dataset_name': dataset_name,
#         'transform': data_transform,
#         'target_loader': None,
#     }

#     data_config['data_refs'] = train_refs
#     train_dataset = SnoringDataset(**data_config)

#     data_config['data_refs'] = valid_refs
#     data_config['transform'] = None
#     valid_dataset = SnoringDataset(**data_config)
#     return train_dataset, valid_dataset

# def get_snoring_refs(label_path, split_json):
#     local_ref = json.load(split_json)
#     label_table = pd.read_csv(label_path)
#     return path_refs

# XXX: change to JSON format
def get_snoring_refs(data_ref_path, data_root):
    data_refs = pd.read_csv(data_ref_path)
    path_refs = snoring_preprocess.get_path_refs_fast(data_root, data_refs, suffix='wav')
    return path_refs

# XXX: Put in here currently
def snoring_transform(waveform):
    waveform = wavform_transform(waveform)
    spec = wavform_to_spec(waveform)
    spec = spec_transform(spec)
    return spec

# XXX: return sr?
def torchaudio_loader(path):
    waveform, sr = torchaudio.load(path, normalize=True)
    return waveform

def build(data_csv, train_json, valid_json, dataset_name):
    train_data_refs = get_snoring_refs(data_csv, train_json)
    valid_data_refs = get_snoring_refs(data_csv, valid_json)
    loader = torchaudio_loader

    train_dataset = MetaDataset(
        loader, train_data_refs, dataset_name, snoring_transform, target_loader=None)
    valid_dataset = MetaDataset(
        loader, valid_data_refs, dataset_name, snoring_transform, target_loader=None)
    return train_dataset, valid_dataset

def build_dataloader(total_data_config, train_batch_size):
    total_train_dataset = []
    total_valid_dataset = []
    for data_config in total_data_config:
        train_dataset, valid_dataset = build(**data_config)
        total_train_dataset.append(train_dataset)
        total_valid_dataset.append(valid_dataset)

    concat_train_dataset = ConcatDataset(total_train_dataset)
    concat_valid_dataset = ConcatDataset(total_valid_dataset)

    train_loader = DataLoader(concat_train_dataset, train_batch_size, shuffle=True)
    valid_loader = DataLoader(concat_valid_dataset, 1, shuffle=False)
    return train_loader, valid_loader

def main():
    data_root = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\pp\Samsung_Note10Plus_night\wave_split'
    data_ref_path = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\pp\Samsung_Note10Plus_night\data.csv'
    data_refs = pd.read_csv(data_ref_path)
    path_refs = snoring_preprocess.get_path_refs_fast(data_root, data_refs, suffix='wav')
    pass
    # build_snoring_dataset(data_roots, data_ref_paths, train_batch_size, data_transform=None)

if __name__ == '__main__':
    main()


# Project) 오락실 Pang 게임 만들기

# [게임 조건]
# 1. 캐릭터는 화면 아래에 위치, 좌우로만 이동 가능
# 2. 스페이스를 누르면 무기를 쏘아 올림
# 3. 큰 공 1 개가 나타나서 바운스
# 4. 무기에 닿으면 공은 작은 크기 2 개로 분할, 가장 작은 크기의 공은 사라짐
# 5. 모든 공을 없애면 게임 종료 => 성공
# 6. 캐릭터는 공에 닿으면 게임 종료 => 실패
# 7. 시간 제한 99 초 초과 시 게임 종료 => 실패
# 8. FPS 는 30 으로 고정 => 필요 시 speed 값을 조정

# [게임 이미지]
# 1. 배경 : 640 * 480 (가로 세로) - background.png
# 2. 무대 : 640 * 50 - stage.png
# 3. 캐릭터 : 60 * 33 - character.png
# 4. 무기 : 20 * 430 - weapon.png
# 5. 공 : 160 * 160, 80 * 80, 40 * 40, 20 * 20 - balloon1.png ~ balloon4.png
######################################################
import os
import pygame
######################################################
# 기본 초기화 (반드시 해야 하는 것들)
pygame.init()

# 화면 크기 설정
screen_width = 640  # 가로 크기
screen_height = 480 # 세로 크기
screen = pygame.display.set_mode((screen_width, screen_height))

# 화면 타이틀 설정
pygame.display.set_caption("*** Pang pang ***")

# FPS 초당 프레임 수
clock = pygame.time.Clock()
######################################################

# 1. 사용자 게임 초기화 (배경 화면, 게임 이미지, 좌표, 속도, 폰트 등)
current_path = os.path.dirname(__file__)    # 현재 파일의 위치 반환
image_path = os.path.join(current_path, "images")   # 현재 폴더 위치에 images 폴더 위치를 더해 이미지가 있는 위치 반환

# 배경화면 만들기
background = pygame.image.load(os.path.join(image_path, "background.png"))  # image_path 경로의 background.png 파일을 불러오기

# 스테이지 만들기
stage = pygame.image.load(os.path.join(image_path, "stage.png"))
stage_size = stage.get_rect().size
stage_height = stage_size[1]    # 스테이지의 높이 위에 캐릭터를 두기 위해 사용

# 캐릭터 만들기
character = pygame.image.load(os.path.join(image_path, "character.png"))
character_size = character.get_rect().size
character_width = character_size[0]
character_height = character_size[1]
character_x_pos = (screen_width / 2) - (character_width / 2)
character_y_pos = screen_height - character_height - stage_height

running = True
while running:
    dt = clock.tick(30)

    # 2. 이벤트 처리 (키보드, 마우스 등)
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

    # 3. 게임 캐릭터 위치 정의

    # 4. 충돌 처리

    # 5. 화면에 그리기
    screen.blit(background, (0, 0))
    screen.blit(stage, (0, screen_height - stage_height))
    screen.blit(character, (character_x_pos, character_y_pos))

    pygame.display.update()

pygame.quit()


import ipywidgets as widgets

def make_model_list_dropdown() -> widgets.Widget:
    model_list_dropdown = widgets.Dropdown(
        options=['Model 1', "Click to add new model..."],
        value='Model 1',
        description='Select Model',
        style={
            'description_width': '100px'
        },
        layout={
            'width': '400px'
        }
    )

    return model_list_dropdown

def make_model_list_box() -> widgets.Widget:
    model_list_box = widgets.HBox(
        children=[make_model_list_dropdown()],
        layout={}
    )

    return model_list_box

def make_clear_button() -> widgets.Widget:
    clear_button = widgets.Button(
        description='Clear All Plots',
        icon='chart-line',
        style={
            'button_color': '#9ecae1'
        },
        layout={'width': '400px'},
        tooltip="Removes all traces from the small-angle scattering and dark field plots on the right."
    )

    return clear_button

def make_export_button() -> widgets.Widget:
    export_button = widgets.Button(
        description='Export Plot Data',
        icon='save',
        style={
            'button_color': '#4292c6'
        },
        layout={'width': '400px'},
        tooltip="Exports SAS and dark field data from the plots to csv files in the working directory."
    )

    return export_button

class ModelHeaderBox:

    def __init__(self):
        model_list = make_model_list_box()

        clear_button = make_clear_button()
        # clear_button.on_click(clear_plots_func)

        export_button = make_export_button()
        # export_button.on_click(export_data_func)

        self.model_header_box = widgets.VBox(
            children=[
                model_list,
                widgets.HBox(children=[clear_button, export_button])
            ]
        )


import pyttsx3
import speech_recognition as sr
import datetime
import wikipedia
import webbrowser

print("READ THIS")
print("WHAT YOUR JARVIS CAN DO")
print("Your Jarvis can search in chrome\n can summarize wikipedia\n can open google and youtube\n  can show you IPL score\n can open National Geographic\n can recommend the best educational channel \n can open the famous coding YT Channel Code With Harry\n can show you recommended Hindi news channel\n Can plat latest music playlist having more than 1000 songs from T Series\n can show you recommended marathi news channel\n can open Adobe Reader 9\n can open amazon\n can also open recommended coding website of stack overflow\n can also tell present time\n can also open SonyLiv website")
print("Jarvis can also answer your questions as\n How are you?\n Who are you?\n How do you do?\n What are you doing?\n What is your name?\n Are you a robot? ")
print("Your Jarvis can also reply on your comments like\n Really!\n Hello\n I am fine\n")
print("Your Jarvis can quit by saying\n Quit \n Bye \n Goodbye \n Exit \n \n \n")
engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[0].id)

print("Enter your Name:")
username = str(input())

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

def wishMe():
    hour = int(datetime.datetime.now().hour)
    if hour>=0 and hour<12:
        speak('Good Morning Sir')
    elif hour>=12 and hour<18:
        speak('Good Afternoon sir')
    else:
        speak('Good Evening sir')

    speak('I am Jarvis. How can I help you?')
def takeCommand():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        print('Listening...')
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print('Recognizing...')
        query = r.recognize_google(audio, language='en')
        print(f"User said: {query}\n")

    except Exception as e:

        print("say that again pls sir")
        return "None"
    return query

if __name__ == "__main__":
   wishMe()
   while True:
        query = takeCommand().lower()
   #executing task based on query

        if'wikipedia' in query:
            speak('Searching wikipedia')
            query = query.replace("wikipedia", "")
            results = wikipedia.summary(query, sentences=2)
            speak("According to Wikipedia")
            print(results)
            speak(results)
        elif'youtube' in query:
            webbrowser.open("www.youtube.com")
            speak("This is the official website of YouTube")
        elif'google' in query:
            webbrowser.open("www.google.com")
            speak("This is the official website of Google.")
        elif'how are you' in query:
            speak("I am fine sir")
            print("I am fine sir")
        elif'what are you doing' in query:
            speak("I am spending time with you")
            print("I am spending time with you")
        elif'your name' in query:
            speak("My name is Jarvis sir")
            print("My name is Jarvis, sir")
        elif'who are you' in query:
            speak("I am Jarvis sir")
            print("I am Jarvis sir")
        elif'well done' in query:
            speak("Thank you its my pleasure")
            print("Thank you its my pleasure")
        elif'thank you' in query:
            speak("You are welcome sir")
            print("You are welcome sir")
        elif'sports' in query:
            webbrowser.open("https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRFp1ZEdvU0FtVnVHZ0pWVXlnQVAB?hl=en-IN&gl=IN&ceid=IN:en")
            speak("These are sports news from google news.")
        elif'exit' in query:
            speak(f"Goodbye {username}, Take care")
            print(f"Goodbye {username}, Take care")
            exit()
        elif'bye' in query:
            speak("Bye sir see you next time")
            exit()
        elif'goodbye' in query:
            speak("Goodbye sir, see you next time")
            exit()
        elif'music'in query:
            webbrowser.open("https://www.youtube.com/aashiqui2/videos")
            speak("These are the music from T-Series YouTube Channel")
        elif'really' in query:
            speak("yeah sir")
        elif'are you robot' in query:
            speak("yes sir, I am Jarvis, a desktop assistant.")
            print("yes sir, I am Jarvis, a desktop assistant.")
        elif'scientific hindi' in query:
            webbrowser.open("https://www.youtube.com/c/FactTechz/videos")
            speak("This is the most famous science fiction YouTube channel in hindi named Facttechz")
        elif'hindi science' in query:
            webbrowser.open("https://www.youtube.com/c/FactTechz/videos")
        elif'hindi scientific' in query:
            webbrowser.open("https://www.youtube.com/c/FactTechz/videos")
            speak("This is the most famous science fiction YouTube channel in hindi named Facttechz")
        elif'play latest music' in query:
            webbrowser.open("https://www.youtube.com/watch?v=b8--JS9lRnI&list=PL095CA373D64B70C0")
            speak("This is the playlist of latest song by T series")
        elif'hello' in query:
            speak(f"hello {username}")
            print(f"hello {username}")
        elif'quit' in query:
            speak("Bye")
            print("Bye")
            exit()
        elif'scientific channel' in query:
            webbrowser.open("https://www.youtube.com/c/NatGeo/videos")
            speak("This is the best scientific YouTube channel names National Geographic")
        elif'codewithharry' in query:
            webbrowser.open("https://www.youtube.com/c/CodeWithHarry/videos")
            speak("This is the best coding YouTube channel named Code With Harry")
        elif'best educational channel' in query:
            webbrowser.open("https://www.youtube.com/c/DearSir/videos")
            speak("This is the most best educational youtube channel, who teaches English and Mathematics for seondary, high secondary and students who are preparing themselves for competitive exams and also teaches us English speaking and short tricks for Maths. This channel is named as Dear Sir")
        elif'time' in query:
            strTime = datetime.datetime.now().strftime("%H:%M:%S")
            speak(f"Sir, the time is{strTime}")
        elif'stack overflow' in query:
            webbrowser.open("www.stackoverflow.com")
            speak("This is one of the recommended website named stack overflow which teaches you coding")
        elif'how do you do' in query:
            speak("I am fine sir, what about you")
            print("I am fine sir, what about you?")
        elif'am fine' in query:
            speak("Nice to hear that from you sir")
            print("Nice to hear that from you sir")
        elif'hindi news' in query:
            webbrowser.open("https://www.youtube.com/watch?v=SmQqAnKG6zs")
            speak("This is the live news from hindi")
        elif'live news' in query:
            webbrowser.open("https://www.youtube.com/watch?v=9Auq9mYxFEE")
            speak("This is the live news from Sky News in English")
        elif'marathi news' in query:
            webbrowser.open("https://www.youtube.com/watch?v=itJLgnqZ3U8")
            speak("This is the live news from ABP maza in marathi")
        elif'amazon' in query:
            webbrowser.open("https://www.amazon.in/?ext_vrnc=hi&tag=googhydrabk-21&ascsubtag=_k_EAIaIQobChMIipy3m8iB7QIVBpZLBR3Lqwg6EAAYASAAEgJNlfD_BwE_k_&ext_vrnc=hi&gclid=EAIaIQobChMIipy3m8iB7QIVBpZLBR3Lqwg6EAAYASAAEgJNlfD_BwE")
            speak("This the shopping website named amazon")
        elif'shopping' in query:
            webbrowser.open("https://www.amazon.in/?ext_vrnc=hi&tag=googhydrabk-21&ascsubtag=_k_EAIaIQobChMIipy3m8iB7QIVBpZLBR3Lqwg6EAAYASAAEgJNlfD_BwE_k_&ext_vrnc=hi&gclid=EAIaIQobChMIipy3m8iB7QIVBpZLBR3Lqwg6EAAYASAAEgJNlfD_BwE")
            speak("This the shopping website named amazon")
        elif'sonyliv' in query:
            webbrowser.open("https://www.sonyliv.com/")
            speak("This is the sony leave website. You can use it for watching TV shows")
        elif'best entertaining' in query:
            webbrowser.open("https://www.sonyliv.com/")
            speak("This is the sony leave website. You can use it for watching TV shows")
        elif'entertainment' in query:
            webbrowser.open("https://www.sonyliv.com/")
            speak("This is the sonyliv website. You can use it for watching TV shows")
        elif'national geographic'in query:
            webbrowser.open("https://www.youtube.com/c/NatGeo/videos")
            speak("This is the best scientific YouTube channel names National Geographic")
        elif'hindi news' in query:
            webbrowser.open("https://www.youtube.com/watch?v=SmQqAnKG6zs")
            speak("This is the live news from hindi")
        elif'gmail' in query:
            webbrowser.open("https://mail.google.com/mail/u/0/#inbox")
            speak("Starting Google Mail")
        elif'meet' in query:
            webbrowser.open("https://meet.google.com/")
            speak("Starting Google Meet")


#!/usr/bin/env python3

import os
import datetime

# Complete the time_delta function below.
def time_delta(t1, t2):
    pattern = '%a %d %b %Y %H:%M:%S %z'
    date1 = int(datetime.datetime.strptime(t1, pattern).timestamp())
    date2 = int(datetime.datetime.strptime(t2, pattern).timestamp())

    print(abs(date1 - date2))
    return abs(date1 - date2)

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    t = int(input())

    for t_itr in range(t):
        t1 = input()

        t2 = input()

        delta = time_delta(t1, t2)

        fptr.write(str(delta) + '\n')

    fptr.close()


from django.urls import path
from . import views

urlpatterns = [
        path('', views.index, name='index'),
        path('event/', views.event, name='event'),
        path('place/', views.place, name='place'),
        path('imply/', views.imply, name='imply'),
        path('insert_event/', views.insert_event, name='insert_event'),
        path('insert_place/', views.insert_place, name='insert_place'),
        path('insert_imply/', views.insert_imply, name='insert_imply'),
        path('insert_picture/', views.insert_picture, name='insert_picture'),
        path('participate_event/', views.participate_event, name='participate_event'),
        path("insert_user/", views.insert_user, name="insert_user"),
        path("insert_user_direct/", views.insert_user_direct, name="insert_user_direct"),
        path("id_duplicate_check/", views.id_duplicate_check, name="id_duplicate_check"),
        path("login/", views.login, name="login"),
        path("auth/", views.auth, name="auth"),
        path("ranking/", views.ranking, name="ranking"),
        path("participating/", views.participating, name="participating"),
        path('select_auth_method/', views.select_auth_method, name='select_auth_method'),
        path('finish_auth_check/', views.finish_auth_check, name='finish_auth_check'),
        path('place_auth_info/', views.place_auth_info, name='place_auth_info'),
]


import re
import sys
from io import StringIO

from colorama import Fore, Style
from diff_match_patch import diff_match_patch

def highlight_differences(text1, text2):
    dmp = diff_match_patch()
    diffs = dmp.diff_main(text1, text2)
    dmp.diff_cleanupSemantic(diffs)

    highlighted_diff = ""
    for diff in diffs:
        if diff[0] == -1:
            highlighted_diff += Fore.RED + diff[1] + Style.RESET_ALL
        elif diff[0] == 1:
            highlighted_diff += Fore.GREEN + diff[1] + Style.RESET_ALL
        else:
            highlighted_diff += diff[1]

    return highlighted_diff

def capture_output(func, *args, **kwargs) -> str:
    original_stdout = sys.stdout
    sys.stdout = StringIO()

    func(*args, **kwargs)

    output_value = sys.stdout.getvalue()

    sys.stdout = original_stdout

    return output_value

def compare_output(output_value, expected_output_file) -> bool:
    with open(expected_output_file, 'r') as f:
        expected_output = f.read().strip()

    output_value = output_value.strip()

    output_value = re.sub(r"ic\|.*\n", "", output_value)

    if output_value != expected_output:
        print("Output does not match the expected output.")
        print()
        highlighted_diff = highlight_differences(expected_output, output_value)
        print(highlighted_diff)
        print()

    return output_value == expected_output


import argparse
import sys
import os, os.path
import redis
import argparse
import logging, logging.config
from hashlib import md5

argparser = argparse.ArgumentParser()
argparser.add_argument("--data", type=str, required=True)
argparser.add_argument("--db-host", type=str, required=True)
argparser.add_argument("--db-port", type=int, required=True)
args = argparser.parse_args()

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

def main():
    db_conn = redis.Redis(host=args.db_host, port=args.db_port, db=0)
    set_root_segments(args.data, db_conn)
    for dir in os.listdir(args.data):
        process_data_segment(args.data, dir, db_conn)

def set_root_segments(data_path, db_conn):
    root_dirs = os.listdir(data_path)
    logging.debug("Setting root dirs: {}".format(root_dirs))
    db_conn.sadd("/root", *root_dirs)

def process_data_segment(data_root, data_path, db_conn):
    logging.debug("Processing data entry {}".format(data_path))
    data_entries = os.listdir(os.path.join(data_root, data_path))

    key_hash = md5(data_path.encode('utf-8')).hexdigest()
    db_conn.set(key_hash, data_path)

    if data_entries == ["_data"]:
        logging.debug("Saving data for entry {}".format(data_path))
        db_conn.set("{}:data".format(data_path),
                    open(os.path.join(data_root, data_path, "_data")).read())
    else:
        logging.debug("Setting data child entries {}".format(data_entries))
        db_conn.sadd("{}:next".format(data_path), *data_entries)
        for entry in os.listdir(os.path.join(data_root, data_path)):
            new_data_path = os.path.join(data_path, entry)
            process_data_segment(data_root, new_data_path, db_conn)

if __name__ == "__main__":
    main()


"""============================================================================
   Có 60% người mua xe thể thao là nam giới.
      1. Chọn loại phân phối. Tạo ra 10 mẫu (ngẫu nhiên) theo mô tả trên
         với số lần lặp lại các thí nghiệm là 1000
      2. Vẽ histogram quan sát. Nhận xét.
      3. Trong 10 chủ xe thể thao được chọn ngẫu nhiên, tính xs có 7 nam giới.
============================================================================"""
import math
import seaborn as sns

from scipy.stats import binom

#------------------------------------------------------------------------------
#  Hàm tính tổ hợp n chập k
#  Vơi Python 3.8 thì dùng math.comb(n, k)
#------------------------------------------------------------------------------
def combination(n, k):
    # n!/(k!)(n-k)!
    numerator   = math.factorial(n)
    denominator = math.factorial(k) * math.factorial(n - k)

    return (numerator / denominator)

# 1. Chọn loại phân phối. Tạo ra 10 mẫu với 1000 lần lặp lại thí nghiệm
#     X ~ Binomial(n, p)
#     n: số mẫu thử nghiệm
#     p: xác suất thành công
#     size: số lần lặp lại (trials) thí nghiệm.
n          = 10
p          = 0.6
size       = 1000
data_binom = binom.rvs(n=n, p=p, size = size)

# 2. Vẽ histogram quan sát. Nhận xét.
ax = sns.distplot(data_binom, kde = False, color = 'blue',
                  hist_kws = {'linewidth': 15, 'alpha': 1})
ax.set(xlabel = 'Binomial Distribution', ylabel = 'Frequency')

# 3. Trong 10 chủ xe thể thao được chọn ngẫu nhiên, tính xs có 7 nam giới.
#    f(k) = P(X = k) = P_x = n!/(k!)(n-k)! x p^k x (1-p)^(n-k)
k     = 7
P_X_k = combination(n, k) * math.pow(p, k) * math.pow(1 - p, n - k)
print('P(X = 7) =', P_X_k)


"""
Constraints:
    - the position of the container has to be in range of device capacity
    based on device version
    - on creation it checks there is a proper amount of chambers associated
    with this object
    - there cannot be any other container at the same place in the same device
"""
from django.db import models
import uuid as UUID
from django.core.exceptions import ValidationError
from django.utils.translation import gettext as _
from django.db.models import Q
from API.managers import ContainerManager

class Container(models.Model):
    # universal identifier
    uuid = models.UUIDField(
        primary_key=True,
        default=UUID.uuid4,
        editable=False,
        unique=True
    )

    # version that specifies capacity
    version = models.ForeignKey(
        "ContainerVersion",
        on_delete=models.PROTECT,
        related_name="existing_containers"
    )

    # reference to device this container belongs to
    device = models.ForeignKey(
        "Device",
        on_delete=models.CASCADE,
        related_name="containers"
    )

    # position in the container
    position = models.PositiveSmallIntegerField()

    # time this container was refilled at
    last_refill = models.DateTimeField(null=True, blank=True)

    # custom manager
    objects = ContainerManager()

    @property
    def capacity(self):
        return self.version.capacity

    @property
    def fill_status(self):
        """
        Returns number of percents this container is filled in calculated over
        every chamber inside.
        """
        chambers = self.chambers.filter(container=self)
        totalCapacity = self.capacity
        fullChambers = 0
        for chamb in chambers:
            if chamb.is_full:
                fullChambers += 1

        if totalCapacity == 0:
            return 0

        return round(100 * fullChambers / totalCapacity)

    def clean(self):
        # validate if the position is in the correct range
        if self.position < 0 or self.position >= self.device.capacity:
            raise ValidationError(
                _("Position value outside of container's capacity"),
                code="invalid_value"
            )

        # make sure there are no two containers at the same place
        if Container.objects.filter(
            ~Q(uuid=self.uuid) &
            Q(device=self.device) &
            Q(position=self.position)
        ).exists():
            raise ValidationError(
                _("Two containers at the same place."),
                code="integrity_error"
            )

    def __str__(self):
        return f"In {self.device}, at pos. {self.position}"


from django.conf.urls import patterns, include, url, static
from django.conf import settings
from django.views.generic import TemplateView
from django.contrib import admin
from django.views.generic import RedirectView
from accounts.views import UserProfileUpdateView
from django.views.defaults import permission_denied

admin.autodiscover()

urlpatterns = []

colab_plugins = settings.COLAB_APPS

for plugin in colab_plugins:
    colab_plugin = colab_plugins.get(plugin)
    plugin_blacklist = colab_plugin.get('blacklist')
    if plugin_blacklist:
        for plugin_url in plugin_blacklist:
            final_url = colab_plugin.get('urls').get('prefix')
            final_url += plugin_url
            urlpatterns += patterns(
                '', url(final_url, permission_denied))

if hasattr(settings, 'BLACKLIST'):
    core_blacklist = settings.BLACKLIST
    for core_url in core_blacklist:
        urlpatterns += patterns('', url(core_url, permission_denied))

urlpatterns += patterns(
    '',
    url(r'^$', RedirectView.as_view(url=settings.COLAB_HOME_URL), name='home'),
    url(r'^robots.txt$', 'colab.home.views.robots', name='robots'),
    url(r'^dashboard$', 'colab.home.views.dashboard', name='dashboard'),
    url(r'^search/', include('colab.search.urls')),
    url(r'^rss/', include('colab.rss.urls')),

    url(r'^account/', include('colab.accounts.urls')),
    url(r'^myaccount/(?P<route>.*)$',
        'colab.accounts.views.myaccount_redirect', name='myaccount'),

    url(r'^colab/admin/', include(admin.site.urls)),

    url(r'', include('colab.plugins.urls')),
)


import requests

from xml.etree import ElementTree

class CTAInterface():
    def __init__(self, api_key):
        self.api_key = api_key
        self.base_url = 'http://lapi.transitchicago.com/api/1.0/ttarrivals.aspx'

    def get_next_arrivals(self, station_id_list):
        next_arrivals = list()
        parameters = {'key': self.api_key}

        for station_id in station_id_list:
            parameters['mapid'] = station_id
            response = requests.get(self.base_url, params=parameters)
            next_arrivals += self.parse_xml_response(response.content)

        return next_arrivals

    @staticmethod
    def parse_xml_response(xml_content):
        next_arrivals = list()
        xml_tree = ElementTree.fromstring(xml_content)

        for node in xml_tree.iter('eta'):
            this_arrival = [None] * 4
            for child in node._children:
                if child.tag == 'staNm':
                    this_arrival[0] = child.text
                elif child.tag == 'rt':
                    this_arrival[1] = child.text
                elif child.tag == 'stpDe':
                    this_arrival[2] = child.text
                elif child.tag == 'arrT':
                    this_arrival[3] = child.text
            next_arrivals.append(this_arrival)

        return next_arrivals


import requests
from bs4 import BeautifulSoup
from time import sleep
from lxml import html

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
                         " AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}

def get_url():
    for count in range(0,121,20):

        url = f"http://www.spisszkol.eu/typ/?wojewodztwo=dolnoslaskie&powiat=wroclawski&start={count}"

        responce = requests.get(url, headers=headers,)
        responce.encoding = "utf8"
        soup = BeautifulSoup(responce.text, "lxml")
        data = soup.find_all("div", class_="doc_entry")

        for i in data:
            card_url = i.find("a").get("href")
            yield card_url

def array():
    for list_card in get_url():
        responce = requests.get(list_card, headers=headers)
        responce.encoding = "utf8"
        sleep(3)
        soup = BeautifulSoup(responce.text, "lxml")
        parsed = html.fromstring(responce.text)
        emails = [e.attrib['title'] for e in parsed.xpath('//a[contains(@href, "email") and @title]')]
        s = " ".join(emails)
        data = soup.find("div", class_="page_body")
        name = data.find("p", class_="map_title red").text
        adres = data.find("p", itemprop="address").text
        try:
            telefon = data.find("span", itemprop="telephone").text
        except Exception as e:
            print(e)
            telefon = None
        yield name, adres, telefon,s


from irc.client import Event, ServerConnection

from timmy.db_access import settings

class AuthHandler:
    def __init__(self):
        self.auth_on_welcome = False
        self.auth_type = ""
        self.auth_data = ""
        self.post_identify = ""
        self.post_auth_sent = False

    def init(self) -> None:
        self.auth_on_welcome = settings.get_setting('auth_on_welcome') == "1"
        self.auth_type = settings.get_setting('auth_type')
        self.auth_data = settings.get_setting('auth_data')
        self.post_identify = settings.get_setting('post_identify')

    def on_welcome(self, connection: ServerConnection, event: Event) -> None:
        if self.auth_on_welcome:
            self.handle_auth(connection)

    def handle_auth(self, connection: ServerConnection) -> None:
        if self.auth_type == 'nickserv':
            connection.privmsg('nickserv', 'IDENTIFY ' + self.auth_data)

    def on_umode(self, connection: ServerConnection, event: Event) -> None:
        if event.target == connection.nickname and not self.post_auth_sent:
            self.post_auth_sent = True
            if self.post_identify != '':
                connection.send_raw(self.post_identify)


from lxml import html
import requests
from time import sleep
import json
import argparse
from collections import OrderedDict
from time import sleep

def parse(ticker):
    # Code to get the stock price
    url = "http://finance.yahoo.com/quote/%s?p=%s" % (ticker, ticker)
    response = requests.get(url, verify=False)
    print("Parsing %s" % (url))
    parser = html.fromstring(response.text)
    summary_table = parser.xpath('//div[contains(@data-test,"summary-table")]//tr')
    summary_data = OrderedDict()
    try:
        for table_data in summary_table:
            raw_table_key = table_data.xpath('.//td[contains(@class,"C(black)")]//text()')
            raw_table_value = table_data.xpath('.//td[contains(@class,"Ta(end)")]//text()')
            table_key = ''.join(raw_table_key).strip()
            table_value = ''.join(raw_table_value).strip()
            summary_data.update({table_key: table_value})
    except:
        print("Failed to parse json response")
        return {"error": "Failed to parse json response"}
    # code to get the current avg estimate, next avg estimate, and next 5 years growth estimate
    url = "http://finance.yahoo.com/quote/%s/analysis?p=%s" % (ticker, ticker)
    response = requests.get(url, verify=False)
    print("Parsing %s" % (url))
    parser = html.fromstring(response.text)
    summary_table = parser.xpath('//section[contains(@data-test,"qsp-analyst")]//tr')
    # summary_data = OrderedDict()
    try:
        for table_data in summary_table:
            raw_table_key = table_data.xpath('.//td[contains(@class,"Py(10px)")]//text()')
            raw_table_value = table_data.xpath('.//td[contains(@class,"Ta(end)")]//text()')
            table_key = ' '.join(raw_table_key).strip()
            table_value = ' '.join(raw_table_value).strip()
            summary_data.update({table_key: table_value})
        summary_data.update({'ticker': ticker})
        return summary_data
    except:
        print("Failed to parse json response")
        return {"error": "Failed to parse json response"}

def parseWiki():
    # code to get all stocks from wiki page
    url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    response = requests.get(url, verify=False)
    print("Parsing")
    parser = html.fromstring(response.text)
    summary_table = parser.xpath('//table[contains(@id,"constituents")]//tr')
    summary_data = OrderedDict()
    n = 0
    try:
        for table_data in summary_table:
            raw_table_value = table_data.xpath('.//a[contains(@class,"external text")]//text()')
            table_value = ' '.join(raw_table_value).strip()
            x = table_value.split(" reports")
            # table_value = table_value[0:5]
            summary_data.update({n: x[0]})
            n = n + 1
        return summary_data
    except:
        print("Failed to parse json response")
        return {"error": "Failed to parse json response"}

def parseAll():
    arr = []
    for item in list(parseWiki().items())[1:]:
        arr.append(parse(item[1]))
    return arr

if __name__ == "__main__":
    """argparser = argparse.ArgumentParser()
    argparser.add_argument('ticker', help='')
    args = argparser.parse_args()
    ticker = args.ticker
    print ("Fetching data for %s" % (ticker))"""
    scraped_data = parseAll()
    print ("Writing data to output file")
    with open('%s-analysis.json', 'w') as fp:
        json.dump(scraped_data, fp, indent=4)
    f = open("done.txt", "w+")
    for i in range(10):
        f.write("This is line %d\r\n" % (i + 1))
    f.close()


# mypy: disable-error-code=arg-type
import asyncio

import discord
import validators
import wavelink
from discord import app_commands
from wavelink.ext import spotify

from .. import config
from ..client import CustomClient

def add_streaming_commands(client: CustomClient) -> None:
    @client.tree.command(
        name="youtube",
        description="play audio from a YouTube video",
    )
    @app_commands.describe(query="search request or URL")
    async def queue_youtube(
        interaction: discord.Interaction, *, query: str
    ) -> None:
        player = await ensure_voice_channel(interaction)
        tracks = await wavelink.YouTubeTrack.search(query)

        if not tracks:
            await interaction.response.send_message(
                f"No search results for query *{query}*"
            )

        track = tracks[0]
        await player.queue.put_wait(track)
        await interaction.response.send_message(f"**{track}** added to queue")

        await start_playing(interaction, player)

    @client.tree.command(
        name="spotify",
        description="play spotify tracks, playlists and albums from a URL",
    )
    @app_commands.describe(
        url=(
            "spotify URL "
            "(https://open.spotify.com/track|album|playlist/...)"
        )
    )
    async def queue_spotify(
        interaction: discord.Interaction, *, url: str
    ) -> None:
        player = await ensure_voice_channel(interaction)
        decoded = spotify.decode_url(url)

        if not validators.url(url) or decoded is None:
            await interaction.response.send_message("Invalid URL provided")
            return

        if decoded["type"] in (
            spotify.SpotifySearchType.album,
            spotify.SpotifySearchType.playlist,
        ):
            await interaction.response.send_message(
                "Loading tracks into the queue"
            )
            tracks = spotify.SpotifyTrack.iterator(query=url)

            if not player.is_playing():
                first_item = await anext(tracks)
                await player.play(first_item)
                await interaction.followup.send(
                    f"Playing **{first_item.title}**"
                )

            async for track in tracks:
                await player.queue.put_wait(track)
                await interaction.followup.send(
                    f"**{track.title}** added to queue"
                )

            await interaction.followup.send(
                f"Loading done. Items in queue: {len(player.queue)}"
            )

        elif decoded["type"] == spotify.SpotifySearchType.track:
            tracks = await spotify.SpotifyTrack.search(url)

            if not tracks:
                await interaction.response.send_message(
                    f"No search results for *{url}*"
                )

            track = tracks[0]
            await player.queue.put_wait(track)
            await interaction.response.send_message(
                f"**{track.title}** added to queue"
            )

        await start_playing(interaction, player)

    @client.tree.command(name="outro", description="epic disconnect")
    async def play_n_leave(interaction: discord.Interaction) -> None:
        player = await ensure_voice_channel(interaction)
        await interaction.response.send_message("It's time to go to sleep")

        url = config.OUTRO_VIDEO["url"]
        tracks = await wavelink.YouTubeTrack.search(url)

        if not tracks:
            await interaction.response.send_message(
                f"Couldn't find the video, please verify that link is correct: '{url}'"
            )

        track = tracks[0]
        await player.play(track)

        while player.current == track and player.is_playing():
            await asyncio.sleep(0.25)

            if player.position >= config.OUTRO_VIDEO["timestamp_ms"]:
                await player.disconnect()
                await client.change_presence(status=discord.Status.idle)

    async def ensure_voice_channel(
        interaction: discord.Interaction,
    ) -> wavelink.Player:
        if not interaction.user:
            raise discord.DiscordException("interaction.user is None")

        if not isinstance(interaction.user, discord.Member):
            raise discord.DiscordException(
                "interaction.user is not a discord.Member object"
            )

        author_voice = interaction.user.voice

        if not author_voice:
            await interaction.response.send_message(
                "You're not in a voice channel"
            )
            raise discord.DiscordException("interaction.user.voice is None")

        if not interaction.guild:
            raise discord.DiscordException("interaction.guild is None")

        player: wavelink.Player = interaction.guild.voice_client

        if not player and author_voice.channel:
            player = await author_voice.channel.connect(cls=wavelink.Player)
            return player

        if author_voice.channel != player.channel:
            await interaction.response.send_message(
                "You're in a different channel"
            )
            raise discord.DiscordException(
                "user is in a different voice channel"
            )

        return player

    async def start_playing(
        interaction: discord.Interaction, player: wavelink.Player
    ) -> None:
        if player.is_playing() or player.queue.is_empty:
            return

        next_item = player.queue.get()

        await player.play(next_item)
        await interaction.followup.send(f"Playing **{next_item.title}**")


from selenium import webdriver
from lxml import etree
import time,re,random,csv

driver_path = r'D:\chromedriver\chromedriver.exe'
class BossSpider(object):
    def __init__(self,writer):
        self.base_url = 'https://www.zhipin.com'
        self.url = 'https://www.zhipin.com/job_detail/?query=python&city=101010100&industry=&position='
        self.driver = webdriver.Chrome(executable_path=driver_path)
        self.writer = writer

    def run(self):
        self.driver.get(url=self.url)
        while True:
            page_source = self.driver.page_source
            self.request_catalog_page(page_source=page_source)
            next_btn = self.driver.find_element_by_xpath("//div["
                                                         "@class='page']//a[last()]")
            if "next disabled" in next_btn.get_attribute("class"):
                return
            else:
                next_btn.click()

    def request_catalog_page(self,page_source):
        html = etree.HTML(page_source)
        urls = html.xpath("//div[@class='info-primary']//a/@href")
        for url in urls:
            url = self.base_url + url
            self.request_detail_page(url=url)
            time.sleep(random.randint(1,3))

    def request_detail_page(self,url):
        self.driver.execute_script("window.open('%s')" %url)
        self.driver.switch_to.window(self.driver.window_handles[1])
        detail_page_source = self.driver.page_source
        detail_html = etree.HTML(detail_page_source)
        self.prase_detail_page(detail_html=detail_html)

    def prase_detail_page(self,detail_html):
        all_datas = []
        job_status = detail_html.xpath("//div[@class='job-status']/text()")[0]
        job_name = detail_html.xpath("//div[@class='info-primary']//h1/text("
                                    ")")[0]
        salary = detail_html.xpath("//span[@class='salary']/text()")[0]
        salary = re.sub(r"[\s]","",salary)
        dates = detail_html.xpath("//div[@class='job-banner']//p/text()")
        city = dates[0]
        work_years = dates[1]
        education = dates[2]
        job_description = detail_html.xpath("//div[@class='job-sec']//div["
                                            "@class='text']//text()")
        job_description = "".join(list(map(lambda x:re.sub(r"(\n|●|  )","",x),
                                  job_description)))
        company_introduction = detail_html.xpath("//div[@class='job-sec company-info']//div["
                                            "@class='text']//text()")
        company_introduction = "".join(list(map(lambda x: re.sub(r"(\n|●| )", "", x),
                                          company_introduction)))
        com = detail_html.xpath("//div[@class='job-sec company-info']//a/@href")
        if len(com):
            company_url = self.base_url + com[0]
        else:
            company_url = self.base_url
        work_address = detail_html.xpath("//div["
                                         "@class='location-address']/text()")[0]
        map_url = detail_html.xpath("//div["
                                         "@class='job-location']//img/@src")[0]
        contacts = detail_html.xpath("//h2[@class='name']/text()")
        if len(contacts):
            contacts = contacts[0]
        datas_dict = {
           'job_name':job_name,
           'job_status':job_status,
           'salary':salary,
           'city':city,
           'work_years':work_years,
           'education':education,
           'job_description':job_description,
           'company_introduction':company_introduction,
           'company_url':company_url,
           'work_address':work_address,
           'map_url':map_url,
           'contacts': contacts
         }
        all_datas.append(datas_dict)
        self.driver.close()
        self.driver.switch_to.window(self.driver.window_handles[0])
        self.writer.writerows(all_datas)
        print('保存成功！')

def writer_func():
    headers = ['job_name', 'job_status', 'salary', 'city', 'work_years',
               'education', 'job_description', 'company_introduction',
               'company_url', 'work_address', 'map_url', 'contacts']
    fp = open("boss.csv", 'a', newline='', encoding='utf-8')
    writer = csv.DictWriter(fp, headers)
    writer.writeheader()
    return writer

if __name__ == '__main__':
    writer = writer_func()
    bossspider = BossSpider(writer)
    bossspider.run()


from tqdm import tqdm

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from group4_banker import Group4Banker

from plot_config import setup, set_fig_size, set_arrowed_spines

setup()

def prep_data():

    features = ['checking account balance', 'duration', 'credit history',
                'purpose', 'amount', 'savings', 'employment', 'installment',
                'marital status', 'other debtors', 'residence time',
                'property', 'age', 'other installments', 'housing', 'credits',
                'job', 'persons', 'phone', 'foreign']

    target = 'repaid'

    df_train = pd.read_csv("../../data/credit/D_train.csv", sep=' ', names=features+[target])
    df_test = pd.read_csv("../../data/credit/D_test.csv", sep=' ', names=features+[target])

    numerical_features = ['duration', 'age', 'residence time', 'installment', 'amount', 'persons', 'credits']
    quantitative_features = list(filter(lambda x: x not in numerical_features, features))
    D_train = pd.get_dummies(df_train, columns=quantitative_features, drop_first=True)
    D_test = pd.get_dummies(df_test, columns=quantitative_features, drop_first=True)
    encoded_features = list(filter(lambda x: x != target, D_train.columns))

    return D_train, D_test, encoded_features, target

def oob_sampling(model, X, y, n_samples, size_sample, seed):

    np.random.seed(seed)

    Y_true = np.zeros((n_samples, size_sample))
    Y_pred = np.zeros((n_samples, size_sample))

    for i in tqdm(range(n_samples)):

        test_idx = np.random.choice(np.arange(X.shape[0]), replace=True, size=size_sample)

        y_test = y.iloc[test_idx]
        X_test = X.iloc[test_idx]

        Y_true[i] = y_test.values - 1
        Y_pred[i] = [int(model.get_best_action(x)) for _, x in X_test.iterrows()]

    return Y_true, Y_pred

def posterior_hypothesis(X, p):

    n_samples, n_trails = np.shape(X)

    posterior = np.zeros_like(X, dtype=float)
    for i in range(n_trails):

        # Parameters for Beta prior.
        alpha = 1
        beta = 1

        log_p, log_p_marginal = 0, 0
        for j in range(n_samples):

            x = X[j, i]
            p_x = (x * alpha + (1 - x) * beta) / (alpha + beta)

            alpha = alpha + x
            beta = beta + 1 - x

            # Prior for the null hypothesis being true.
            log_p = log_p + np.log(p)
            log_p_marginal = log_p_marginal + np.log(p_x)

            posterior[j, i] = np.exp(log_p - np.log(np.exp(log_p_marginal) + np.exp(log_p)))

    return posterior

def plot_posteriors(Y_true, Y_pred):

    delta = 0.05

    # H0:
    p_not_grant = 1 - np.mean(Y_true)
    p_not_grant_low = p_not_grant - np.sqrt(np.log(2) * delta / (2 * Y_true.shape[0]))
    p_not_grant_high = p_not_grant + np.sqrt(np.log(2) * delta / (2 * Y_true.shape[0]))

    # Posterior hypothesis of H0.
    post_grant_true = posterior_hypothesis(Y_true, p=p_not_grant)
    post_grant_pred = posterior_hypothesis(Y_pred, p=p_not_grant)
    post_grant_pred_low = posterior_hypothesis(Y_pred, p=p_not_grant_low)
    post_grant_pred_high = posterior_hypothesis(Y_pred, p=p_not_grant_high)

    fig, ax = plt.subplots(1, 1, figsize=set_fig_size(500, fraction=1))
    ax.set_title(r"Posterior probability of accepting $H_0$")
    ax.plot(np.mean(post_grant_true, axis=1), label="Test data", c="darkorange", alpha=0.7)
    ax.plot(np.mean(post_grant_pred, axis=1), label="Model predictions", alpha=0.7)
    ax.fill_between(np.arange(Y_true.shape[0]),
                    np.mean(post_grant_pred_low, axis=1),
                    np.mean(post_grant_pred_high, axis=1), alpha=0.2)
    ax.legend()
    ax.set_ylabel("Posterior probability")
    ax.set_xlabel("Number of bootstrap samples")
    set_arrowed_spines(fig, ax)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.tight_layout()
    fig.savefig("posterior_null_hypo.pdf")

    bayes_reject_grant_true = np.mean(post_grant_true < p_not_grant, axis=1)
    bayes_reject_grant_pred = np.mean(post_grant_pred < p_not_grant, axis=1)
    bayes_reject_grant_pred_low = np.mean(post_grant_pred_low < p_not_grant, axis=1)
    bayes_reject_grant_pred_high = np.mean(post_grant_pred_high < p_not_grant, axis=1)

    fig, ax = plt.subplots(1, 1, figsize=set_fig_size(500, fraction=1))
    ax.set_title(r"Posterior probability of rejecting $H_0$")
    ax.plot(bayes_reject_grant_true, label="Test data", c="darkorange", alpha=0.7)
    ax.plot(bayes_reject_grant_pred, label="Model predictions", alpha=0.7)
    ax.fill_between(np.arange(Y_true.shape[0]),
                    bayes_reject_grant_pred_low,
                    bayes_reject_grant_pred_high, alpha=0.2)
    ax.legend()
    ax.set_ylabel("Posterior probability")
    ax.set_xlabel("Number of bootstrap samples")
    set_arrowed_spines(fig, ax)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.tight_layout()
    fig.savefig("reject_null_hypo.pdf")

def main():

    D_train, D_test, encoded_features, target = prep_data()

    X_train = D_train.loc[:, encoded_features]
    y_train = D_train.loc[:, target]

    model = Group4Banker(optimize=False, random_state=42)
    model.set_interest_rate(0.05)
    model.fit(X_train, y_train)

    X_test = D_test.loc[:, encoded_features]
    y_test = D_test.loc[:, target]

    Y_true, Y_pred = oob_sampling(model, X_test, y_test, 500, 50, 42)

    np.save("Y_true.npy", Y_true)
    np.save("Y_pred.npy", Y_pred)

    Y_true = np.load("Y_true.npy")
    Y_pred = np.load("Y_pred.npy")

    plot_posteriors(Y_true, Y_pred)

if __name__ == "__main__":
    main()


from django.urls import include, path
from rest_framework import routers
from . import views

router = routers.DefaultRouter()
router.register(r'reto', views.RetoViewSet)
router.register(r'jugador', views.JugadoresViewSet)
router.register(r'usuarios', views.UsuarioViewSet, basename='usuario')
router.register(r'partidas', views.PartidaViewSet, basename='partida')
#Reto
router.register(r'estudiantes', views.EstudianteViewSet)
router.register(r'juegos', views.JuegoViewSet)
router.register(r'intentos', views.IntentosViewSet)

urlpatterns = [
    path('api/',include(router.urls)),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    path('',views.index, name='index'),
    path('procesamiento',views.procesamiento, name='procesamiento'),
    path('lista',views.lista,name='lista'),
    path('suma',views.suma,name='suma'),
    path('resta',views.resta,name='resta'),
    path('multiplicacion',views.multiplicacion,name='multiplicacion'),
    path('division',views.division,name='division'),
    path('usuario_lista',views.usuarios,name='usuarios'),
    path('usuario_pos',views.usuario_pos,name='usuario_pos'),
    path('usuario_del',views.usuario_del,name='usuario_del'),
    path('usuario_updt',views.usuario_updt,name='usuario_updt'),
    path('login', views.login, name='login'),
    path('procesologin', views.procesologin, name='procesologin'),
    path('valida_usuario',views.valida_usuario,name='valida_usuario'),
    path('grafica',views.grafica,name='grafica'),
    path('barras',views.barras,name='barras'),
    path('gauge',views.gauge_chart,name='gauge'),
    path('datos_estudiantes',views.datos_estudiantes,name='datos_estudiantes'),
]


# NN ライブラリを使いやすいように変更
# 7763c160b4ec1caa99718cd3c865339227a1908e

import numpy as np
import matplotlib.pyplot as plt

def main():
    # ファイルからプロットする値を読み込む
    true_x = []
    predict_x = []
    with open("data/tmp_result.csv") as fileobj:
        while True:
            line = fileobj.readline()
            if line:
                line = line.rstrip()
                splitted = line.split(",")
                true_x.append(float(splitted[0]))
                predict_x.append(float(splitted[1]))
            else:
                break

    # 相関係数
    coef = np.corrcoef(true_x, predict_x)[0, 1]
    print(coef)

    # 散布図を描画
    font_size = 15
    fig = plt.figure(figsize=(12, 6))
    # x
    xy_max = np.max(true_x+predict_x)
    xy_min = np.min(true_x+predict_x)
    g_x = fig.add_subplot(1, 2, 1)
    plt.plot([xy_min, xy_max], [xy_min, xy_max], color="orange")
    plt.scatter(true_x, predict_x)
    plt.title("xt (相関係数："+f'{coef:.3f}'+")",
                fontname="MS Gothic", fontsize=font_size)
    plt.xlabel("xt の真値 [px]", fontname="MS Gothic", fontsize=font_size)
    plt.ylabel("xt の予測値 [px]", fontname="MS Gothic", fontsize=font_size)
    plt.grid(True)
    # t
    # xy_max = np.max(result[:, 2:4])
    # xy_min = np.min(result[:, 2:4])
    # g_t = fig.add_subplot(1, 2, 2)
    # plt.plot([xy_min, xy_max], [xy_min, xy_max], color="orange")
    # plt.scatter(result[:, 2:3], result[:, 3:4])
    # plt.title("tt (相関係数："+f'{coef_t:.3f}'+")",
    #             fontname="MS Gothic", fontsize=font_size)
    # plt.xlabel("tt の真値 [sec]", fontname="MS Gothic", fontsize=font_size)
    # plt.ylabel("tt の予測値 [sec]", fontname="MS Gothic", fontsize=font_size)
    # plt.grid(True)
    plt.show()

if __name__ == '__main__':
    main()