text:
# -*- coding: utf-8 -*-
# Copyright Hannah von Reth <vonreth@kde.org>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

### fetch functions

from CraftCore import CraftCore
from CraftDebug import deprecated
import utils

import io
import os
import urllib
import subprocess
import sys
import re

def getFile(url, destdir, filename='', quiet=None) -> bool:
    """download file from 'url' into 'destdir'"""
    if quiet is None:
        quiet = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False)
    CraftCore.log.debug("getFile called. url: %s" % url)
    if url == "":
        CraftCore.log.error("fetch: no url given")
        return False

    pUrl = urllib.parse.urlparse(url)
    if not filename:
        filename = os.path.basename(pUrl.path)

    utils.createDir(destdir)

    if pUrl.scheme == "s3":
        return s3File(url, destdir, filename)
    elif pUrl.scheme == "minio":
        return minioGet(pUrl.netloc + pUrl.path, destdir, filename)

    # curl and wget basically only work when we have a cert store on windows
    if not CraftCore.compiler.isWindows or os.path.exists(os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")):
        if not CraftCore.settings.getboolean("General", "NoWget"):
            if CraftCore.cache.findApplication("wget"):
                return wgetFile(url, destdir, filename, quiet)

        if CraftCore.cache.findApplication("curl"):
            return curlFile(url, destdir, filename, quiet)

    if os.path.exists(os.path.join(destdir, filename)):
        return True

    powershell = CraftCore.cache.findApplication("powershell")
    if powershell:
        filename = os.path.join(destdir, filename)
        return utils.system([powershell, "-NoProfile", "-ExecutionPolicy", "ByPass", "-Command",
                       f"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (new-object net.webclient).DownloadFile(\"{url}\", \"{filename}\")"])
    else:
        def dlProgress(count, blockSize, totalSize):
            if totalSize != -1:
                percent = int(count * blockSize * 100 / totalSize)
                utils.printProgress(percent)
            else:
                sys.stdout.write(("\r%s bytes downloaded" % (count * blockSize)))
                sys.stdout.flush()

        try:
            urllib.request.urlretrieve(url, filename=os.path.join(destdir, filename),
                                       reporthook=dlProgress if CraftCore.debug.verbose() >= 0 else None)
        except Exception as e:
            CraftCore.log.warning(e)
            return False

    if CraftCore.debug.verbose() >= 0:
        sys.stdout.write("\n")
        sys.stdout.flush()
    return True

def curlFile(url, destdir, filename, quiet):
    """download file with curl from 'url' into 'destdir', if filename is given to the file specified"""
    curl = CraftCore.cache.findApplication("curl")
    command = [curl, "-C", "-", "--retry", "10", "-L", "--ftp-ssl", "--fail"]
    cert = os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")
    if os.path.exists(cert):
        command += ["--cacert", cert]
    # the default of 20 might not be enough for sourceforge ...
    command += ["--max-redirs",  "50"]
    command += ["-o", os.path.join(destdir, filename)]
    command += [url]
    CraftCore.log.debug("curlfile called")

    if CraftCore.debug.verbose() < 1:
        if quiet:
            with io.StringIO() as tmp:
                ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False)
                if ciMode:
                    command += ["-v"]
                if not utils.system(command, logCommand=ciMode, stdout=tmp, stderr=subprocess.STDOUT):
                    CraftCore.log.warning(tmp.getvalue())
                    return False
                if ciMode:
                    loc = re.findall(r"Host: ([^\s]+)", tmp.getvalue())
                    if loc:
                        CraftCore.log.info(f"Downloaded from: {loc[-1]}")
                return True
        elif CraftCore.cache.checkCommandOutputFor(curl, "--progress-bar"):
            command += ["--progress-bar"]
            CraftCore.log.info(f"curl {url}")
            return utils.system(command, displayProgress=True, logCommand=False, stderr=subprocess.STDOUT)
    command += ["-v"]
    return utils.system(command)

def wgetFile(url, destdir, filename, quiet):
    """download file with wget from 'url' into 'destdir', if filename is given to the file specified"""
    wget = CraftCore.cache.findApplication("wget")
    command = [wget, "-c", "-t", "10"]
    cert = os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")
    if os.path.exists(cert):
        command += ["--ca-certificate", cert]
    # the default of 20 might not be enough for sourceforge ...
    command += ["--max-redirect",  "50"]
    if CraftCore.settings.getboolean("General", "EMERGE_NO_PASSIVE_FTP", False):
        command += ["--no-passive-ftp"]
    if not filename:
        command += ["-P", destdir]
    else:
        command += ["-O", os.path.join(destdir, filename)]
    command += [url]

    if CraftCore.debug.verbose() < 1:
        if quiet:
            with io.StringIO() as tmp:
                ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False)
                if not utils.system(command, logCommand=ciMode, stdout=tmp, stderr=subprocess.STDOUT):
                    CraftCore.log.warning(tmp.getvalue())
                    return False
                if ciMode:
                    loc = re.findall(r"Location: ([^\s]+)", tmp.getvalue())
                    if loc:
                        CraftCore.log.info(f"Downloaded from: {loc[-1]}")
                return True
        elif CraftCore.cache.checkCommandOutputFor(wget, "--show-progress"):
            command += ["-q", "--show-progress"]
            CraftCore.log.info(f"wget {url}")
            return utils.system(command, displayProgress=True, logCommand=False, stderr=subprocess.STDOUT)
    return utils.system(command)

def s3File(url : str, destdir : str, filename : str) ->bool:
    aws = CraftCore.cache.findApplication("aws")
    if not aws:
        CraftCore.log.critical("aws not found, please install awscli. \"pip install awscli\" ")
        return False
    return utils.system([aws, "s3", "cp", url, os.path.join(destdir, filename)])

def minioGet(url : str, destdir : str, filename : str) ->bool:
    minio = None
    if CraftCore.compiler.isWindows:
        minio = CraftCore.cache.findApplication("minio")
    if not minio:
        minio = CraftCore.cache.findApplication("mc")
    if not minio:
        CraftCore.log.critical("minio client not found, please install minio")
        return False
    return utils.system([minio, "cp", url, os.path.join(destdir, filename)])


text:
import os

def app_config(app):
    app.config.from_object('src.conf.flask_api_conf')
    if os.environ.get('FLASK_API_CONF') is not None:
        app.config.from_envvar('FLASK_API_CONF')

    sqlalchemy_database_uri_str = os.environ.get('FLASK_API_SQLALCHEMY_DATABASE_URI')
    if sqlalchemy_database_uri_str is not None:
        app.config['SQLALCHEMY_DATABASE_URI'] = sqlalchemy_database_uri_str

    init_sample_data_str = os.environ.get('FLASK_API_INIT_SAMPLE_DATA')
    if init_sample_data_str is not None:
        app.config['INIT_SAMPLE_DATA'] = eval(init_sample_data_str)

    cleanup_invalidated_tokens_interval_seconds_str = os.environ.get('FLASK_API_CLEANUP_INVALIDATED_TOKENS_INTERVAL_SECONDS')
    if cleanup_invalidated_tokens_interval_seconds_str is not None:
        app.config['CLEANUP_INVALIDATED_TOKENS_INTERVAL_SECONDS'] = eval(cleanup_invalidated_tokens_interval_seconds_str)

    return app

domain: python


text:
import datetime
from typing import Dict, Tuple, Any

import boto3
from botocore.stub import Stubber
from dateutil.tz import tzutc

from dassana.common.aws_client import LambdaTestContext
from json import dumps
import pytest

@pytest.fixture()
def input_s3_with_website(s3_public_bucket_with_website, region):
    return {
        'bucketName': s3_public_bucket_with_website,
        'region': region
    }

@pytest.fixture()
def iam_policy():
    return {
        "Version": "2012-10-17",
        "Statement": [
            {
                "Sid": "VisualEditor0",
                "Effect": "Allow",
                "Action": [
                    "ec2:GetDefaultCreditSpecification",
                    "ec2:GetEbsEncryptionByDefault",
                    "ec2:ExportClientVpnClientConfiguration",
                    "ec2:GetCapacityReservationUsage",
                    "ec2:DescribeVolumesModifications",
                    "ec2:GetHostReservationPurchasePreview",
                    "ec2:GetSubnetCidrReservations",
                    "ec2:GetConsoleScreenshot",
                    "ec2:GetConsoleOutput",
                    "ec2:ExportClientVpnClientCertificateRevocationList",
                    "ec2:GetLaunchTemplateData",
                    "ec2:GetSerialConsoleAccessStatus",
                    "ec2:GetFlowLogsIntegrationTemplate",
                    "ec2:DescribeScheduledInstanceAvailability",
                    "ec2:GetEbsDefaultKmsKeyId",
                    "ec2:GetManagedPrefixListEntries",
                    "ec2:DescribeVpnConnections",
                    "ec2:DescribeTags",
                    "ec2:GetCoipPoolUsage",
                    "ec2:DescribeFastSnapshotRestores",
                    "ec2:GetReservedInstancesExchangeQuote",
                    "ec2:GetAssociatedEnclaveCertificateIamRoles",
                    "ec2:GetPasswordData",
                    "ec2:GetAssociatedIpv6PoolCidrs",
                    "ec2:DescribeScheduledInstances",
                    "ec2:GetManagedPrefixListAssociations",
                    "ec2:DescribeElasticGpus"
                ],
                "Resource": "*"
            }
        ]
    }

@pytest.fixture()
def iam_role_name():
    return 'ec2-iam-role'

@pytest.fixture()
def instance_profile_name():
    return 'ec2-instance-profile-role'

@pytest.fixture()
def iam_role_arn(iam_client, iam_policy, iam_role_name, instance_profile_name) -> Tuple[Any, Dict[str, Any]]:
    resp = iam_client.create_role(RoleName=iam_role_name, AssumeRolePolicyDocument=dumps(iam_policy))
    instance_profile_resp = iam_client.create_instance_profile(
        InstanceProfileName=instance_profile_name
    )
    iam_client.add_role_to_instance_profile(
        InstanceProfileName=instance_profile_name,
        RoleName=iam_role_name
    )

    instance_profile_resp = instance_profile_resp.get('InstanceProfile')
    return resp['Role']['Arn'], {
        'Name': instance_profile_resp.get('InstanceProfileName'),
        'Arn': instance_profile_resp.get('Arn')
    }

@pytest.fixture()
def ec2_instance_with_role(ec2_client, iam_role_arn, instance_profile_name):
    instances = ec2_client.run_instances(ImageId='ami-1234',
                                         MinCount=1,
                                         MaxCount=1,
                                         InstanceType='t2.micro',
                                         IamInstanceProfile=iam_role_arn[1])
    instance_id = instances.get('Instances')[0].get('InstanceId')
    assoc_resp = ec2_client.associate_iam_instance_profile(IamInstanceProfile=iam_role_arn[1], InstanceId=instance_id)
    return instance_id

@pytest.fixture()
def ec2_instance_without_role(ec2_client):
    ec2_client.run_instances(ImageId='ami-1234-foobar',
                             MinCount=1,
                             MaxCount=1)
    instances = ec2_client.describe_instances(
        Filters=[
            {
                'Name': 'image-id',
                'Values': ['ami-1234-foobar']
            }
        ]
    )['Reservations'][0]['Instances']
    return instances[0]['InstanceId']

def test_ec2_instance_with_role(ec2_instance_with_role, iam_role_arn, region):
    from handler_ec2_role import handle

    result: Dict = handle({'instanceId': ec2_instance_with_role, 'region': region},
                          LambdaTestContext('foobar', env={},
                                            custom={}))
    assert result.get('result').get('roleName') == iam_role_arn[1].get('Name')
    assert str.split(result.get('result').get('roleArn'), ':role/') == str.split(iam_role_arn[1].get(
        'Arn'), ':instance-profile/')

def test_ec2_instance_without_role(ec2_instance_without_role, region):
    from handler_ec2_role import handle
    result: Dict = handle({'instanceId': ec2_instance_without_role, 'region': region},
                          LambdaTestContext('foobar', env={}, custom={}))
    assert result.get('result').get('roleArn') == ''
    assert result.get('result').get('roleName') == ''

def test_ec2_instance_does_not_exist(ec2_instance_without_role, region):
    from handler_ec2_role import handle
    result: Dict = handle({'instanceId': 'i-abcd', 'region': region},
                          LambdaTestContext('foobar', env={}, custom={}))
    assert result.get('result').get('roleArn') == ''
    assert result.get('result').get('roleName') == ''


text:
#!/usr/bin/env python

# This example uses Uvicorn package that must be installed. However, it can be
# replaced with any other ASGI-compliant server.
#
# NOTE: Python 3.6 requires aiocontextvars package to be installed.
#
# Run: python app_global_request.py

import rollbar
import uvicorn

from rollbar.contrib.starlette import LoggerMiddleware
from starlette.applications import Starlette
from starlette.responses import JSONResponse

# Integrate Rollbar with Starlette application
app = Starlette()
app.add_middleware(LoggerMiddleware)  # should be added as the last middleware

async def get_user_agent():
    # Global access to the current request object
    request = rollbar.get_request()

    user_agent = request.headers['User-Agent']
    return user_agent

# $ curl -i http://localhost:8888
@app.route('/')
async def root(request):
    user_agent = await get_user_agent()
    return JSONResponse({'user-agent': user_agent})

if __name__ == '__main__':
    uvicorn.run(app, host='localhost', port=8888)


text:
from infoblox_netmri.utils.utils import locate, to_snake
from infoblox_netmri.api.exceptions.netmri_exceptions import NotImplementedException

class Broker(object):
    """ Base class for broker instances, provides methods for API requests.
        And return responces wrapped with specific class

        :param client: InfobloxNetMRI client
    """

    controller = None

    def __init__(self, client):
        self.client = client

    def api_request(self, method_name, params):
        """ Make api request and return single wrapped object

            :param method_name: name of API methods
            :param params: dict-wrapped params for specific API call
        """
        data = self.client.api_request(method_name, params)

        if isinstance(data, dict) and len(data) > 1:
            for x in data.keys():
                data[x] = self._get_return_object_type(data.get(x))
            return data

        class_name = to_snake(self.__class__.__name__.replace("Broker", ""))

        if class_name in data:
            result_name = class_name
        else:
            result_name = method_name.split('/')[-1]
            if result_name not in data:
                return data

        return self._get_return_object_type(data.get(result_name))

    # See NETMRI-31545
    def api_mixed_request(self, method_name, params):
        """ Make api request and download a file and return
            JSON response or request status dictionary

            :param method_name: name of API methods
            :param params: dict-wrapped params for specific API call
        """
        data = self.client.api_request(method_name, params, downloadable=True)

        class_name = to_snake(self.__class__.__name__.replace("Broker", ""))

        if class_name in data:
            result_name = class_name
        else:
            result_name = method_name.split('/')[-1]
            if result_name not in data:
                return data

        return self._get_return_object_type(data.get(result_name))

    def api_list_request(self, method_name, params):
        """ Make api request and return list of wrapped objects

            :param method_name: name of API methods
            :param params: dict-wrapped params for specific API call
        """
        data = self.client.api_request(method_name, params)

        if not data:
            return None
        try:
            return [self._get_return_object_type(x) for x in data[self.controller]]
        except KeyError:
            print("Sorry, this method will be implemented in the\
                  future versions of NetMRI")
            raise NotImplementedException(self.controller, method_name)

    def _get_method_fullname(self, method):
        """ Returns full API method name using controller name

            **Input**
            :param method: method name
            :return: full API path
        """
        return "{}/{}".format(self.controller, method)

    def _get_return_object_type(self, data):
        """ Returns wrapped response which inherits from RemoteModel class

            :param data: API responce data
            :return: RemoteModel child class
        """
        if not data or type(data) != dict:
            return data

        class_name = data.get("_class")
        obj_class = locate(self._get_remote_class_name(class_name))

        return obj_class(data, self.client)

    def _get_remote_class_name(self, name):
        """ Generate full path to specific RemoteModel instance

            :param name: name of model
            :return: full path for model
        """
        return "infoblox_netmri.api.remote.models.{pckg}_remote.{name}Remote".format(
            pckg=to_snake(name),
            name=name
        )


text:
from flask_wtf import FlaskForm
from flask_wtf.file import FileRequired
from wtforms import (
    StringField,
    SubmitField,
    PasswordField,
    FileField,
    SelectField,
    TextAreaField,
    BooleanField,
)
from wtforms.validators import DataRequired, Length, ValidationError

from models import RoomBGMTypes

"""
Reference:
        movie_id = form.movie_id.data
        place = form.place.data
        imageid = form.imageid.data
        title = form.title.data
"""

class RoomMovieForm(FlaskForm):
    movie_id = StringField("Movie ID")
    place = StringField("Place in Room")
    imageid = StringField("Image ID")
    title = StringField("Movie Title")
    submit = SubmitField("Done!")

class LoginForm(FlaskForm):
    username = StringField("Username")
    password = PasswordField("Password")
    submit = SubmitField("Enter the underground")

class NewsForm(FlaskForm):
    news = TextAreaField("News Contents", validators=[DataRequired()])
    create = SubmitField("Create!")

class MiiUploadForm(FlaskForm):
    mii = FileField("Mii Selection", validators=[FileRequired()])
    name = StringField("Mii Name", validators=[DataRequired()])
    color1 = StringField("Shirt Color (Hex)", validators=[DataRequired()])
    color2 = StringField("Pants Color (Hex)", validators=[DataRequired()])
    upload = SubmitField("Add Mii")

class NewUserForm(FlaskForm):
    username = StringField("Username", validators=[DataRequired()])
    password1 = PasswordField("Password", validators=[DataRequired()])
    password2 = PasswordField("Confirm Password", validators=[DataRequired()])
    upload = SubmitField("Complete")

    def validate_password1(self, _):
        if self.password1.data != self.password2.data:
            return ValidationError("New passwords must be the same")

class ChangePasswordForm(FlaskForm):
    current_password = PasswordField("Password", validators=[DataRequired()])
    new_password = PasswordField("New Password", validators=[DataRequired()])
    new_password_confirmation = PasswordField(
        "Confirm New Password", validators=[DataRequired()]
    )
    complete = SubmitField("Complete")

    def validate_current_password(self, _):
        if self.current_password.data == self.new_password.data:
            return ValidationError("New password cannot be the same as current!")

    def validate_new_password(self, _):
        if self.new_password.data != self.new_password_confirmation.data:
            return ValidationError("New passwords must be the same")

class MovieUploadForm(FlaskForm):
    movie = FileField("Movie", validators=[FileRequired()])
    title = StringField("Movie title", validators=[DataRequired(), Length(max=48)])
    thumbnail = FileField("Movie thumbnail", validators=[FileRequired()])
    # Choices for the select field are only evaluated once, so we must set it when necessary.
    category = SelectField("Movie category", validators=[DataRequired()])
    upload = SubmitField("Add Movie")

class CategoryAddForm(FlaskForm):
    category_name = StringField("Category Name", validators=[DataRequired()])
    thumbnail = FileField("Movie thumbnail", validators=[FileRequired()])
    submit = SubmitField("Add")

class CategoryEditForm(FlaskForm):
    category_name = StringField("Category Name", validators=[DataRequired()])
    thumbnail = FileField("Movie thumbnail")
    submit = SubmitField("Edit")

class ParadeForm(FlaskForm):
    news = StringField("News", validators=[DataRequired()])
    company = StringField("Company", validators=[DataRequired()])
    image = FileField("Parade Banner", validators=[FileRequired()])
    submit = SubmitField("Create")

class RoomForm(FlaskForm):
    bgm = SelectField(
        "Background Music",
        choices=RoomBGMTypes.choices(),
        coerce=RoomBGMTypes.coerce,
    )
    room_logo = FileField("Room Logo", validators=[FileRequired()])
    has_mascot = BooleanField("Mascot Enabled")
    has_contact = BooleanField("Show Contact Information")
    intro_msg = StringField("Intro Message", validators=[DataRequired()])
    mii_msg = StringField("Mii Message", validators=[DataRequired()])
    submit = SubmitField("Create")

class KillMii(FlaskForm):
    given_id = StringField("Item ID", validators=[DataRequired()])
    submit = SubmitField("Delete!")

class ConciergeForm(FlaskForm):
    prof = StringField("Profession", validators=[DataRequired()])
    message1 = StringField("Message 1", validators=[DataRequired()])
    message2 = StringField("Message 2", validators=[DataRequired()])
    message3 = StringField("Message 3", validators=[DataRequired()])
    message4 = StringField("Message 4", validators=[DataRequired()])
    message5 = StringField("Message 5", validators=[DataRequired()])
    message6 = StringField("Message 6", validators=[DataRequired()])
    message7 = StringField("Message 7", validators=[DataRequired()])
    movieid = StringField("Movie ID", validators=[DataRequired()])
    submit = SubmitField("Create!")

class PosterForm(FlaskForm):
    file = FileField("Poster Image", validators=[FileRequired()])
    title = StringField("Title", validators=[DataRequired()])
    msg = StringField("Message", validators=[DataRequired()])
    upload = SubmitField("Create Poster!")


text:
import pandas as pd
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics.pairwise import linear_kernel

def email_mapper(df):
    coded_dict = dict()
    cter = 1
    email_encoded = []

    for val in df['email']:
        if val not in coded_dict:
            coded_dict[val] = cter
            cter+=1

        email_encoded.append(coded_dict[val])
    return email_encoded

def create_user_item_matrix(df):
    '''
    INPUT:
    df - pandas dataframe with article_id, title, user_id columns

    OUTPUT:
    user_item - user item matrix

    Description:
    Return a matrix with user ids as rows and article ids on the columns with 1 values where a user interacted with
    an article and a 0 otherwise
    '''
    # Fill in the function here
    user_item = df.groupby('user_id')['article_id'].value_counts().unstack()
    user_item[user_item.isna() == False] = 1

    return user_item # return the user_item matrix

def get_top_articles(n, df):
    '''
    INPUT:
    n - (int) the number of top articles to return
    df - (pandas dataframe) df as defined at the top of the notebook

    OUTPUT:
    top_articles - (list) A list of the top 'n' article titles

    '''
    article_id_grouped_df = df.groupby(['title'])
    top_articles = article_id_grouped_df['user_id'].count().sort_values(ascending=False).iloc[:n].index.tolist()

    return top_articles # Return the top article titles from df (not df_content)

def get_top_article_ids(n, df):
    '''
    INPUT:
    n - (int) the number of top articles to return
    df - (pandas dataframe) df as defined at the top of the notebook

    OUTPUT:
    top_articles - (list) A list of the top 'n' article titles

    '''
    article_id_grouped_df = df.groupby(['article_id'])
    top_articles_ids = article_id_grouped_df['user_id'].count().sort_values(ascending=False).iloc[:n].index.tolist()

    return top_articles_ids # Return the top article ids

def user_user_recs(user_id, user_item, df, m=10):
    '''
    INPUT:
    user_id - (int) a user id
    m - (int) the number of recommendations you want for the user

    OUTPUT:
    recs - (list) a list of recommendations for the user by article id
    rec_names - (list) a list of recommendations for the user by article title

    Description:
    Loops through the users based on closeness to the input user_id
    For each user - finds articles the user hasn't seen before and provides them as recs
    Does this until m recommendations are found

    Notes:
    * Choose the users that have the most total article interactions
    before choosing those with fewer article interactions.

    * Choose articles with the articles with the most total interactions
    before choosing those with fewer total interactions.

    '''
    def get_user_articles_names_ids(user_id):
        '''
        INPUT:
        user_id

        OUTPUT:
        article_ids - (list) a list of the article ids seen by the user
        article_names - (list) a list of article names associated with the list of article ids
                        (this is identified by the doc_full_name column in df_content)

        Description:
        Provides a list of the article_ids and article titles that have been seen by a user
        '''
        # Your code here
        article_ids = user_item.loc[user_id][user_item.loc[user_id] ==1].index.tolist()
        article_names = []
        for i in article_ids:
            try:
                title = df[df['article_id'] == i]['title'].unique()[0]
            except IndexError:
                title ="None"

            article_names.append(title)
        article_ids = list(map(str, article_ids))

        return article_ids, article_names # return the ids and names

    def find_similar_users():
        '''
        OUTPUT:
        similar_users - (list) an ordered list where the closest users (largest dot product users)
                        are listed first

        Description:
        Computes the similarity of every pair of users based on the dot product
        Returns an ordered

        '''
        # compute similarity of each user to the provided user
        user_item_tmp = user_item.copy()
        user_item_tmp[user_item_tmp.isna() == True] = 0 # 1. Make Nan to 0
        row = user_item_tmp.loc[user_id] # 2. Select a row
        result_dot = row@user_item_tmp.T # 3. Dot product of each of row of the matrix
        result_dot.drop(labels = [user_id], inplace=True) # remove the own user's id
        most_similar_users = result_dot.sort_values(ascending=False).index.tolist()  # sort by similarity # create list of just the ids

        return most_similar_users # return a list of the users in order from most to least similar

    def get_top_sorted_users(most_similar_users):

        '''
        INPUT:
        most_similar_users - (list) an ordered list where the closest users (largest dot product users)
                        are listed first

        OUTPUT:
        neighbors_df - (pandas dataframe) a dataframe with:
                        neighbor_id - is a neighbor user_id
                        similarity - measure of the similarity of each user to the provided user_id
                        num_interactions - the number of articles viewed by the user - if a u

        Other Details - sort the neighbors_df by the similarity and then by number of interactions where
                        highest of each is higher in the dataframe

        '''
        # Make neighbor_id column
        df_user_id_grouped =df.groupby("user_id")
        df_user_id_grouped['article_id'].count().sort_values(ascending=False)
        neighbors_df = pd.DataFrame()
        neighbors_df['neighbor_id'] = most_similar_users

        # make similarity column
        user_item_tmp = user_item.copy()
        user_item_tmp[user_item_tmp.isna() == True] = 0 # 1. Make Nan to 0
        row = user_item_tmp.loc[user_id] # Select a row
        result_dot = row@user_item_tmp.T # Dot product of each of row of the matrix
        result_dot.drop(labels = [user_id], inplace=True) # remove the own user's id
        similarity = result_dot.sort_values(ascending=False).values.tolist()[0:10]
        neighbors_df['similarity'] = similarity

        # Make num_interactions column
        num_interactions = []
        for i in neighbors_df['neighbor_id']:
            counted_interaction = df_user_id_grouped['article_id'].count().loc[i]
            num_interactions.append(counted_interaction)
        neighbors_df['num_interactions'] = num_interactions
        neighbors_df = neighbors_df.sort_values(by=['similarity', 'num_interactions'], ascending=False)

        return neighbors_df # Return the dataframe specified in the doc_string

    recs = []
    rec_names =[]
    counter = 0
    # Get seen article ids and names from selected user id
    article_ids, article_names = get_user_articles_names_ids(user_id)
    # Make set to find unseen articles
    seen_ids_set = set(article_ids)
    most_similar_users = find_similar_users()[0:10]
    neighbors_df = get_top_sorted_users(most_similar_users)
    # Find similar users of the selected user
    similar_users_list = neighbors_df['neighbor_id'] # Get neighbor_df

    # Make recommendation list
    for sim_user in similar_users_list:
         if counter < m:
            # Get seen article ids and names from similar users
            sim_article_ids, sim_article_names = get_user_articles_names_ids(sim_user)
            # Make dict (key: article_ids, value:article_names)
            sim_user_dict = dict(zip(sim_article_ids, sim_article_names))
            # Make set to find unseen articles
            sim_seen_ids_set = set(sim_article_ids)
            # Create set of unseen articles_ids
            unseen_ids_set = sim_seen_ids_set.difference(seen_ids_set)

            for i in unseen_ids_set:
                if counter < m:
                    recs.append(i)
                    rec_names.append(sim_user_dict[i])
                    counter += 1

    return recs, rec_names

###

def make_Tfidf_array(df_content):
    def tokenize(text):
        '''
        Function splits text into separate words and gets a word lowercased and removes whitespaces at the ends of a word.
        The funtions also cleans irrelevant stopwords.
        Input:
        1. text: text message
        Output:
        1. Clean_tokens : list of tokenized clean words
        '''
        # Get rid of other sepcial characters
        text = re.sub(r"[^a-zA-Z0-9]", " ", text)
        # Tokenize
        tokens = word_tokenize(text)
        # Lemmatize
        lemmatizer = WordNetLemmatizer()
        clean_tokens = []
        for tok in tokens:
            clean_tok = lemmatizer.lemmatize(tok, pos='v').lower().strip()
            clean_tokens.append(clean_tok)

        # Remove stop words
        stopwords = nltk.corpus.stopwords.words('english')
        clean_tokens = [token for token in clean_tokens if token not in stopwords]

        return clean_tokens

    corpus = df_content['doc_description']
    df_content['doc_description'].fillna(df_content['doc_full_name'], inplace=True)
    stop_words = stopwords.words("english")
    lemmatizer = WordNetLemmatizer()
    # Text Processing, Feature Extraction
    vect = TfidfVectorizer(tokenizer=tokenize)
    # get counts of each token (word) in text data
    X = vect.fit_transform(corpus)
    X = X.toarray()

    return vect, X

def make_content_recs(article_id, df_content, df, m=10):
    '''
    INPUT:
    article_id = (int) a article id in df_content
    m - (int) the number of recommendations you want for the user
    df_content -  (pandas dataframe) df_content as defined at the top of the notebook
    df - (pandas dataframe) df as defined at the top of the notebook

    OUTPUT:
    recs - (list) a list of recommendations for the user by article id
    rec_names - (list) a list of recommendations for the user by article title
    '''
    def tokenize(text):
        '''
        Function splits text into separate words and gets a word lowercased and removes whitespaces at the ends of a word.
        The funtions also cleans irrelevant stopwords.
        Input:
        1. text: text message
        Output:
        1. Clean_tokens : list of tokenized clean words
        '''
        # Get rid of other sepcial characters
        text = re.sub(r"[^a-zA-Z0-9]", " ", text)
        # Tokenize
        tokens = word_tokenize(text)
        # Lemmatize
        lemmatizer = WordNetLemmatizer()
        clean_tokens = []
        for tok in tokens:
            clean_tok = lemmatizer.lemmatize(tok, pos='v').lower().strip()
            clean_tokens.append(clean_tok)

        # Remove stop words
        stopwords = nltk.corpus.stopwords.words('english')
        clean_tokens = [token for token in clean_tokens if token not in stopwords]

        return clean_tokens

    vect, X = make_Tfidf_array(df_content)

    if article_id in df_content.article_id:
        cosine_similarity = linear_kernel(X, X)
        df_similarity = pd.DataFrame(cosine_similarity[article_id], columns=['similarity'])
        df_similarity_modified = df_similarity.drop(article_id)
        recs = df_similarity_modified.similarity.sort_values(ascending=False).index[0:10].tolist()
        rec_names = []

        for i in recs:
            name = df_content[df_content['article_id'] == i]['doc_full_name'].values[0]
            rec_names.append(name)

    else:
        tfidf_feature_name = vect.get_feature_names()
        # Get title of the document of interest
        booktitle = df[df['article_id'] == article_id]['title'].values[0]
        # Tokenize the title
        booktitle_tokenized = tokenize(booktitle)

        X_slice_list = []
        for i in booktitle_tokenized:
            if i in tfidf_feature_name:
                X_slice_list.append(tfidf_feature_name.index(i))

        X_slice_list.sort()
        X_sliced = X[:,X_slice_list]
        check_df = pd.DataFrame(X_sliced, columns=X_slice_list)
        check_df['sum'] = check_df.sum(axis=1)
        recs = check_df.sort_values("sum", ascending=False)[0:10].index.tolist()
        rec_names = []
        for i in recs:
            name = df_content[df_content['article_id'] == i]['doc_full_name'].values[0]
            rec_names.append(name)

    return recs, rec_names


text:
import sys
import argparse
import logging

from ml_crop.version import __version__
from ml_crop.utils_train import train
from ml_crop.utils_predict import predict

logger = logging.getLogger(__name__)

def parse_args(args):
    desc = 'ml_crop (v%s)' % __version__
    dhf = argparse.ArgumentDefaultsHelpFormatter
    parser0 = argparse.ArgumentParser(description=desc)

    pparser = argparse.ArgumentParser(add_help=False)
    pparser.add_argument('--version', help='Print version and exit', action='version', version=__version__)
    pparser.add_argument('--log', default=2, type=int,
                         help='0:all, 1:debug, 2:info, 3:warning, 4:error, 5:critical')

    subparsers = parser0.add_subparsers(dest='command')

    parser = subparsers.add_parser('train', parents=[pparser], help='train the model', formatter_class=dhf)
    parser.add_argument('-model', '--model_id', help='classifer model to apply, e.g. randomforest', default='randomforest', type=str, required=True)
    parser.add_argument('-tif', '--raster_file', help='path to the satellite image in raster', default='trans_nzoia.tif', type=str, required=True)
    parser.add_argument('-crop', '--train_geo', help='path to the crop type vector in geojson or shp', default='training_combined.geojson', type=str, required=True)
    parser.add_argument('-out', '--out_model', help='the train model output as pickle file', default="model.sav", type=str, required=True)

    parser = subparsers.add_parser('predict', parents=[pparser], help='used a pretrained model for prediction', formatter_class=dhf)
    parser.add_argument('-out', '--out_model', help='the train model output as pickle file', default="model.sav", type=str, required=True)
    parser.add_argument('-ntif', '--pred_tiff', help='path to the satellite image in raster', default='trans_nzoia.tif', type=str, required=True)
    parser.add_argument('-outimg', '--output_image', help='filename to save the output classification images', default='trans_nzoia.tif', type=str, required=True)
    parsed_args = vars(parser0.parse_args(args))

    return parsed_args

def main(cmd, **kwargs):
    if cmd == 'train':
        train(**kwargs)
    elif cmd == 'predict':
        predict(**kwargs)

def cli():
    args = parse_args(sys.argv[1:])
    logger.setLevel(args.pop('log') * 10)
    main(args.pop('command'), **args)

if __name__ == "__main__":
    cli()


text:
#!/usr/bin/env python

"""
python list_bundles.py prod b6dc9b93-929a-45d0-beb2-5cf8e64872fe
python list_bundles.py staging 3b41f062-621c-46ca-abad-bce09427934d
"""

import argparse
import json
import logging
import sys

from ingest.api.ingestapi import IngestApi

logging.getLogger('ingest').setLevel(logging.DEBUG)
format = ' %(asctime)s  - %(name)s - %(levelname)s in %(filename)s:' \
         '%(lineno)s %(funcName)s(): %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.WARNING, format=format)

class BundleManifest:
    def __init__(self, resource):
        self._object = resource

    @property
    def fqid(self):
        uuid = self._object.get('bundleUuid')
        version = self._object.get('bundleVersion')
        return f'{uuid}.{version}'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Generates a bundle fqid list given a project uuid')
    parser.add_argument('env', choices=['dev', 'integration', 'staging', 'prod'], help='environment')
    parser.add_argument('project_uuid', metavar='project-uuid', type=str, help='Project uuid')
    parser.add_argument('--filename', type=str, help='Output filename')
    args = parser.parse_args()

    project_uuid = args.project_uuid
    filename = args.filename or f'{args.project_uuid}.json'
    env = args.env

    infix = f'.{env}' if env != 'prod' else ''
    url = f'https://api.ingest{infix}.data.humancellatlas.org'
    ingest_api = IngestApi(url)
    project = ingest_api.get_project_by_uuid(project_uuid)
    bundle_manifests = ingest_api.get_related_entities("bundleManifests", project, "bundleManifests")

    bundle_fqids = [BundleManifest(obj).fqid for obj in bundle_manifests]

    with open(filename, 'w') as outfile:
        json.dump(bundle_fqids, outfile, indent=4)

    print(f'Total bundle count: {len(bundle_fqids)}')
    print(f'Saved into file: {filename}')


text:
from Tkinter import *
from Tkinter import Text as textcontrol

class StyledTextControl( textcontrol ):
    def spaces(self, val):
        return str(val*8)
    def __screen(self, width, height):
        self.
    def __init__(self, parent, width, height, fontf, fontsize):
        # Predefining Variables
        self.POS = "RIGHT"

        self.app = parent
        self.widget = textcontrol(parent.mainframe)
        self.widget.config(tabs=self.spaces(4), background="#ffffff", foreground='#000000', highlightthickness=0, borderwidth=0)
        if fontf != None:
            if fontsize != None and fontsize != "":
                self.widget.config(font=(fontf, fontsize))
            else:
                self.widget.config(font=fontf)
    def setMargins(self, top, left, right, ):
    def pack(self):
        self.app.configs.append(self.__screen)
        self.widget.pack(side=self.POS)


text:
from celery.loaders.base import BaseLoader

class AppLoader(BaseLoader):

    def on_worker_init(self):
        self.import_default_modules()

    def read_configuration(self):
        return {}


text:
"""Define family of algorithms and make them interchangeable

The algorithms vary independetly from the clients using it.
This class implements to IngestorInterface and dynamically invoke
a suitable algorithm (strategy.algorithm()), through parse()
abstract method. i.e. it is independent of how an algorithm
is implemented.

That means, the behavior can be changed without breaking the classes
that use it, and the classes can switch between behaviors by changing
the specific implementation used without requiring any
significant code changes.
"""

from typing import List

from .IngestorInterface import IngestorInterface
from .QuoteModel import QuoteModel
from .CSVImporter import CSVImporter
from .PDFImporter import PDFImporter
from .DocxImporter import DocxImporter
from .TXTImporter import TXTImporter

class Ingestor(IngestorInterface):
    """Define family of algorithms & dynamically invoke the one of interest"""

    importer_classes = [CSVImporter, PDFImporter, DocxImporter, TXTImporter]

    @classmethod
    def parse(cls, path: str) -> List[QuoteModel]:

        for importer in cls.importer_classes:
            if importer.can_ingest(path):
                return importer.parse(path)


text:
import ast
import math
import copy

with open('input.txt', 'r') as f:
    lines = f.readlines()
lines = [line[:-1] for line in lines]
lines = [ast.literal_eval(line) for line in lines]

def reduce(num):
    explodeCriteriaMet = True
    splitCriteriaMet = True

    while explodeCriteriaMet or splitCriteriaMet:
        explodeCriteriaMet = False
        splitCriteriaMet = False

        # check for pair nested inside four pairs
        for idx1, val1 in enumerate(num):
            if explodeCriteriaMet:
                break

            if type(val1) == list:
                for idx2, val2 in enumerate(val1):
                    if explodeCriteriaMet:
                        break

                    if type(val2) == list:
                        for idx3, val3 in enumerate(val2):
                            if explodeCriteriaMet:
                                break

                            if type(val3) == list:
                                for idx4, val4 in enumerate(val3):
                                    if type(val4) == list:
                                        explodeCriteriaMet = True
                                        num = explode(num, val4, [idx1, idx2, idx3, idx4])
                                        break

        if explodeCriteriaMet:
            continue

        # regular number is 10 or greater
        for idx1, val1 in enumerate(num):
            if splitCriteriaMet:
                break

            if type(val1) == list:
                for idx2, val2 in enumerate(val1):
                    if splitCriteriaMet:
                        break

                    if type(val2) == list:
                        for idx3, val3 in enumerate(val2):
                            if splitCriteriaMet:
                                break

                            if type(val3) == list:
                                for idx4, val4 in enumerate(val3):
                                    if val4 >= 10:
                                        splitCriteriaMet = True
                                        num = split(num, val4, [idx1, idx2, idx3, idx4])
                                        break

                            elif val3 >= 10:
                                splitCriteriaMet = True
                                num = split(num, val3, [idx1, idx2, idx3])
                                break

                    elif val2 >= 10:
                        splitCriteriaMet = True
                        num = split(num, val2, [idx1, idx2])
                        break

            elif val1 >= 10:
                splitCriteriaMet = True
                num = split(num, val1, [idx1])
                break

    return num

def split(num, value, idx):
    # print("SPLIT")
    # print("IDX:", idx)
    # print("VAL:", value)

    pair = [math.floor(value/2), math.ceil(value/2)]

    if len(idx) == 4:
        num[idx[0]][idx[1]][idx[2]][idx[3]] = pair
    elif len(idx) == 3:
        num[idx[0]][idx[1]][idx[2]] = pair
    elif len(idx) == 2:
        num[idx[0]][idx[1]] = pair
    elif len(idx) == 1:
        num[idx[0]] = pair

    return num

def getValueAtIndex(num, idx):
    for i in idx:
        num = num[i]
    return num

def changeValueAtIndex(num, idx, value):
    if len(idx) == 5:
        num[idx[0]][idx[1]][idx[2]][idx[3]][idx[4]] += value
    elif len(idx) == 4:
        num[idx[0]][idx[1]][idx[2]][idx[3]] += value
    elif len(idx) == 3:
        num[idx[0]][idx[1]][idx[2]] += value
    elif len(idx) == 2:
        num[idx[0]][idx[1]] += value
    elif len(idx) == 1:
        num[idx[0]] += value
    return num

def explode(num, item, idx):
    # store values to add:
    left = item[0]
    right = item[1]
    # the exploding pair is replaced with the regular number 0
    num[idx[0]][idx[1]][idx[2]][idx[3]] = 0

    # adding the values to their neighbors
    # left:
    for index, indexValue in enumerate(idx[:: -1]):
        if indexValue != 0:  # there is a neighbour to the left when the indexValue is 1
            idxLeft = idx[: 4-index]
            idxLeft[-1] = 0

            while type(getValueAtIndex(num, idxLeft)) == list:
                idxLeft.append(1)
            num = changeValueAtIndex(num, idxLeft, left)
            break

    # right
    for index, indexValue in enumerate(idx[:: -1]):
        if indexValue != 1:  # there is a neighbour to the right when the indexValue is 0
            idxRight = idx[: 4-index]
            idxRight[-1] = 1

            while type(getValueAtIndex(num, idxRight)) == list:
                idxRight.append(0)
            num = changeValueAtIndex(num, idxRight, right)
            break

    return num

def calculateSum(num1, num2):
    sum = [copy.deepcopy(num1), copy.deepcopy(num2)]
    return reduce(sum)

def calculateMagnitude(num):
    left = num[0]
    right = num[1]
    if type(left) == list:
        left = calculateMagnitude(left)
    if type(right) == list:
        right = calculateMagnitude(right)

    return (3*left + 2*right)

# part 1
sum = lines[0]
for line in lines[1:]:
    sum = calculateSum(sum, line)

print("What is the magnitude of the final sum?", calculateMagnitude(sum))

# part 2
largest = 0

for x in lines:
    for y in lines:
        if x != y:
            magnitude = calculateMagnitude(calculateSum(x, y))
            if magnitude > largest:
                largest = magnitude

print("What is the largest magnitude of any sum of two different snailfish numbers from the homework assignment?", largest)


text:
numeros = [[], []]
for c in range(0, 7):
    n = int(input(f'Digite o {c+1}o. número: '))
    if n % 2 == 0:
        numeros[0].append(n)
    elif n % 2 == 1:
        numeros[1].append(n)
numeros[0].sort()
numeros[1].sort()
print('='*30)
print(f'Números pares digitados: {numeros[0]}')
print(f'Números ímpares digitados: {numeros[1]}')


text:
#!/usr/bin/env python3
# Copyright (c) 2014-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Base class for RPC testing."""

from enum import Enum
from io import BytesIO
import logging
import optparse
import os
import pdb
import shutil
from struct import pack
import sys
import tempfile
import time

from . import coverage
from .address import wif_to_privkey
from .authproxy import JSONRPCException
from .blocktools import (
    create_block,
    create_coinbase_pos,
    create_transaction_from_outpoint,
)
from .key import CECKey
from .messages import (
    COIN,
    COutPoint,
    CTransaction,
    CTxIn,
    CTxOut,
    hash256,
)
from .script import (
    CScript,
    OP_CHECKSIG,
)
from .test_node import TestNode
from .util import (
    MAX_NODES,
    PortSeed,
    assert_equal,
    assert_greater_than,
    assert_greater_than_or_equal,
    check_json_precision,
    connect_nodes,
    connect_nodes_clique,
    disconnect_nodes,
    Decimal,
    DEFAULT_FEE,
    get_datadir_path,
    hex_str_to_bytes,
    bytes_to_hex_str,
    initialize_datadir,
    p2p_port,
    set_node_times,
    SPORK_ACTIVATION_TIME,
    SPORK_DEACTIVATION_TIME,
    vZC_DENOMS,
    wait_until,
)

class TestStatus(Enum):
    PASSED = 1
    FAILED = 2
    SKIPPED = 3

TEST_EXIT_PASSED = 0
TEST_EXIT_FAILED = 1
TEST_EXIT_SKIPPED = 77

TMPDIR_PREFIX = "t_notes_func_test_"

class t_notesTestFramework():
    """Base class for a t_notes test script.

    Individual t_notes test scripts should subclass this class and override the set_test_params() and run_test() methods.

    Individual tests can also override the following methods to customize the test setup:

    - add_options()
    - setup_chain()
    - setup_network()
    - setup_nodes()

    The __init__() and main() methods should not be overridden.

    This class also contains various public and private helper methods."""

    def __init__(self):
        """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method"""
        self.setup_clean_chain = False
        self.nodes = []
        self.mocktime = 0
        self.rpc_timewait = 600  # Wait for up to 600 seconds for the RPC server to respond
        self.supports_cli = False
        self.set_test_params()

        assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()"

    def main(self):
        """Main function. This should not be overridden by the subclass test scripts."""

        parser = optparse.OptionParser(usage="%prog [options]")
        parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
                          help="Leave t_notesds and test.* datadir on exit or error")
        parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true",
                          help="Don't stop t_notesds after the test execution")
        parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"),
                          help="Source directory containing t_notesd/t_notes-cli (default: %default)")
        parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"),
                          help="Directory for caching pregenerated datadirs")
        parser.add_option("--tmpdir", dest="tmpdir", help="Root directory for datadirs")
        parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO",
                          help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.")
        parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true",
                          help="Print out all RPC calls as they are made")
        parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int',
                          help="The seed to use for assigning port numbers (default: current process id)")
        parser.add_option("--coveragedir", dest="coveragedir",
                          help="Write tested RPC commands into this directory")
        parser.add_option("--configfile", dest="configfile",
                          help="Location of the test framework config file")
        parser.add_option('--legacywallet', dest="legacywallet", default=False, action="store_true",
                          help='create pre-HD wallets only')
        parser.add_option('--tiertwo', dest="tiertwo", default=False, action="store_true",
                          help='run tier two tests only')
        parser.add_option('--sapling', dest="sapling", default=False, action="store_true",
                          help='run tier two tests only')
        parser.add_option("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true",
                          help="Attach a python debugger if test fails")
        parser.add_option("--usecli", dest="usecli", default=False, action="store_true",
                          help="use t_notes-cli instead of RPC for all commands")
        self.add_options(parser)
        (self.options, self.args) = parser.parse_args()

        PortSeed.n = self.options.port_seed

        os.environ['PATH'] = self.options.srcdir + ":" + self.options.srcdir + "/qt:" + os.environ['PATH']

        check_json_precision()

        self.options.cachedir = os.path.abspath(self.options.cachedir)

        # Set up temp directory and start logging
        if self.options.tmpdir:
            self.options.tmpdir = os.path.abspath(self.options.tmpdir)
            os.makedirs(self.options.tmpdir, exist_ok=False)
        else:
            self.options.tmpdir = tempfile.mkdtemp(prefix=TMPDIR_PREFIX)
        self._start_logging()

        success = TestStatus.FAILED

        try:
            if self.options.usecli and not self.supports_cli:
                raise SkipTest("--usecli specified but test does not support using CLI")
            self.setup_chain()
            self.setup_network()
            time.sleep(5)
            self.run_test()
            success = TestStatus.PASSED
        except JSONRPCException as e:
            self.log.exception("JSONRPC error")
        except SkipTest as e:
            self.log.warning("Test Skipped: %s" % e.message)
            success = TestStatus.SKIPPED
        except AssertionError as e:
            self.log.exception("Assertion failed")
        except KeyError as e:
            self.log.exception("Key error")
        except Exception as e:
            self.log.exception("Unexpected exception caught during testing")
        except KeyboardInterrupt as e:
            self.log.warning("Exiting after keyboard interrupt")

        if success == TestStatus.FAILED and self.options.pdbonfailure:
            print("Testcase failed. Attaching python debugger. Enter ? for help")
            pdb.set_trace()

        if not self.options.noshutdown:
            self.log.info("Stopping nodes")
            if self.nodes:
                self.stop_nodes()
        else:
            for node in self.nodes:
                node.cleanup_on_exit = False
            self.log.info("Note: t_notesds were not stopped and may still be running")

        if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED:
            self.log.info("Cleaning up")
            shutil.rmtree(self.options.tmpdir)
        else:
            self.log.warning("Not cleaning up dir %s" % self.options.tmpdir)

        if success == TestStatus.PASSED:
            self.log.info("Tests successful")
            exit_code = TEST_EXIT_PASSED
        elif success == TestStatus.SKIPPED:
            self.log.info("Test skipped")
            exit_code = TEST_EXIT_SKIPPED
        else:
            self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir)
            self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir))
            exit_code = TEST_EXIT_FAILED
        logging.shutdown()
        sys.exit(exit_code)

    # Methods to override in subclass test scripts.
    def set_test_params(self):
        """Tests must this method to change default values for number of nodes, topology, etc"""
        raise NotImplementedError

    def add_options(self, parser):
        """Override this method to add command-line options to the test"""
        pass

    def setup_chain(self):
        """Override this method to customize blockchain setup"""
        self.log.info("Initializing test directory " + self.options.tmpdir)
        if self.setup_clean_chain:
            self._initialize_chain_clean()
        else:
            self._initialize_chain()

    def setup_network(self):
        """Override this method to customize test network topology"""
        self.setup_nodes()

        # Connect the nodes as a "chain".  This allows us
        # to split the network between nodes 1 and 2 to get
        # two halves that can work on competing chains.
        #
        # Topology looks like this:
        # node0 <-- node1 <-- node2 <-- node3
        #
        # If all nodes are in IBD (clean chain from genesis), node0 is assumed to be the source of blocks (miner). To
        # ensure block propagation, all nodes will establish outgoing connections toward node0.
        # See fPreferredDownload in net_processing.
        #
        # If further outbound connections are needed, they can be added at the beginning of the test with e.g.
        # connect_nodes(self.nodes[1], 2)
        for i in range(self.num_nodes - 1):
            connect_nodes(self.nodes[i + 1], i)
        self.sync_all()

    def setup_nodes(self):
        """Override this method to customize test node setup"""
        extra_args = None
        if hasattr(self, "extra_args"):
            extra_args = self.extra_args
        self.add_nodes(self.num_nodes, extra_args)
        self.start_nodes()

    def run_test(self):
        """Tests must override this method to define test logic"""
        raise NotImplementedError

    # Public helper methods. These can be accessed by the subclass test scripts.

    def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None):
        """Instantiate TestNode objects"""

        if extra_args is None:
            extra_args = [[]] * num_nodes
        # Check wallet version
        if self.options.legacywallet:
            for arg in extra_args:
                arg.append('-legacywallet')
            self.log.info("Running test with legacy (pre-HD) wallet")
        if binary is None:
            binary = [None] * num_nodes
        assert_equal(len(extra_args), num_nodes)
        assert_equal(len(binary), num_nodes)
        for i in range(num_nodes):
            self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=self.rpc_timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, use_cli=self.options.usecli))

    def start_node(self, i, *args, **kwargs):
        """Start a t_notesd"""

        node = self.nodes[i]

        node.start(*args, **kwargs)
        node.wait_for_rpc_connection()

        time.sleep(10)

        if self.options.coveragedir is not None:
            coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc)

    def start_nodes(self, extra_args=None, *args, **kwargs):
        """Start multiple t_notesds"""

        if extra_args is None:
            extra_args = [None] * self.num_nodes
        assert_equal(len(extra_args), self.num_nodes)
        try:
            for i, node in enumerate(self.nodes):
                node.start(extra_args[i], *args, **kwargs)
            for node in self.nodes:
                node.wait_for_rpc_connection()
        except:
            # If one node failed to start, stop the others
            self.stop_nodes()
            raise

        time.sleep(10)

        if self.options.coveragedir is not None:
            for node in self.nodes:
                coverage.write_all_rpc_commands(self.options.coveragedir, node.rpc)

    def stop_node(self, i):
        """Stop a t_notesd test node"""
        self.nodes[i].stop_node()
        self.nodes[i].wait_until_stopped()

    def stop_nodes(self):
        """Stop multiple t_notesd test nodes"""
        for node in self.nodes:
            # Issue RPC to stop nodes
            node.stop_node()

        for node in self.nodes:
            # Wait for nodes to stop
            time.sleep(5)
            node.wait_until_stopped()

    def restart_node(self, i, extra_args=None):
        """Stop and start a test node"""
        self.stop_node(i)
        self.start_node(i, extra_args)

    def assert_start_raises_init_error(self, i, extra_args=None, expected_msg=None, *args, **kwargs):
        with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr:
            try:
                self.start_node(i, extra_args, stderr=log_stderr, *args, **kwargs)
                self.stop_node(i)
            except Exception as e:
                assert 't_notesd exited' in str(e)  # node must have shutdown
                self.nodes[i].running = False
                self.nodes[i].process = None
                if expected_msg is not None:
                    log_stderr.seek(0)
                    stderr = log_stderr.read().decode('utf-8')
                    if expected_msg not in stderr:
                        raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr)
            else:
                if expected_msg is None:
                    assert_msg = "t_notesd should have exited with an error"
                else:
                    assert_msg = "t_notesd should have exited with expected error " + expected_msg
                raise AssertionError(assert_msg)

    def wait_for_node_exit(self, i, timeout):
        self.nodes[i].process.wait(timeout)

    def split_network(self):
        """
        Split the network of four nodes into nodes 0/1 and 2/3.
        """
        disconnect_nodes(self.nodes[1], 2)
        disconnect_nodes(self.nodes[2], 1)
        self.sync_all(self.nodes[:2])
        self.sync_all(self.nodes[2:])

    def join_network(self):
        """
        Join the (previously split) network halves together.
        """
        connect_nodes(self.nodes[1], 2)
        self.sync_all()

    def sync_blocks(self, nodes=None, wait=1, timeout=60):
        """
        Wait until everybody has the same tip.
        sync_blocks needs to be called with an rpc_connections set that has least
        one node already synced to the latest, stable tip, otherwise there's a
        chance it might return before all nodes are stably synced.
        """
        rpc_connections = nodes or self.nodes
        stop_time = time.time() + timeout
        while time.time() <= stop_time:
            best_hash = [x.getbestblockhash() for x in rpc_connections]
            if best_hash.count(best_hash[0]) == len(rpc_connections):
                return
            # Check that each peer has at least one connection
            assert (all([len(x.getpeerinfo()) for x in rpc_connections]))
            time.sleep(wait)
        raise AssertionError("Block sync timed out after {}s:{}".format(
            timeout,
            "".join("\n  {!r}".format(b) for b in best_hash),
        ))

    def sync_mempools(self, nodes=None, wait=1, timeout=60, flush_scheduler=True):
        """
        Wait until everybody has the same transactions in their memory
        pools
        """
        rpc_connections = nodes or self.nodes
        stop_time = time.time() + timeout
        while time.time() <= stop_time:
            pool = [set(r.getrawmempool()) for r in rpc_connections]
            if pool.count(pool[0]) == len(rpc_connections):
                if flush_scheduler:
                    for r in rpc_connections:
                        r.syncwithvalidationinterfacequeue()
                return
            # Check that each peer has at least one connection
            assert (all([len(x.getpeerinfo()) for x in rpc_connections]))
            time.sleep(wait)
        raise AssertionError("Mempool sync timed out after {}s:{}".format(
            timeout,
            "".join("\n  {!r}".format(m) for m in pool),
        ))

    def sync_all(self, nodes=None):
        self.sync_blocks(nodes)
        self.sync_mempools(nodes)

    def enable_mocktime(self):
        """Enable mocktime for the script.

        mocktime may be needed for scripts that use the cached version of the
        blockchain.  If the cached version of the blockchain is used without
        mocktime then the mempools will not sync due to IBD.

        Sets mocktime to Tuesday, October 31, 2017 6:21:20 PM GMT (1572546080)
        """
        self.mocktime = 1572546080

    def disable_mocktime(self):
        self.mocktime = 0

    # Private helper methods. These should not be accessed by the subclass test scripts.

    def _start_logging(self):
        # Add logger and logging handlers
        self.log = logging.getLogger('TestFramework')
        self.log.setLevel(logging.DEBUG)
        # Create file handler to log all messages
        fh = logging.FileHandler(self.options.tmpdir + '/test_framework.log')
        fh.setLevel(logging.DEBUG)
        # Create console handler to log messages to stderr. By default this logs only error messages, but can be configured with --loglevel.
        ch = logging.StreamHandler(sys.stdout)
        # User can provide log level as a number or string (eg DEBUG). loglevel was caught as a string, so try to convert it to an int
        ll = int(self.options.loglevel) if self.options.loglevel.isdigit() else self.options.loglevel.upper()
        ch.setLevel(ll)
        # Format logs the same as t_notesd's debug.log with microprecision (so log files can be concatenated and sorted)
        formatter = logging.Formatter(fmt='%(asctime)s.%(msecs)03d000 %(name)s (%(levelname)s): %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
        formatter.converter = time.gmtime
        fh.setFormatter(formatter)
        ch.setFormatter(formatter)
        # add the handlers to the logger
        self.log.addHandler(fh)
        self.log.addHandler(ch)

        if self.options.trace_rpc:
            rpc_logger = logging.getLogger("BitcoinRPC")
            rpc_logger.setLevel(logging.DEBUG)
            rpc_handler = logging.StreamHandler(sys.stdout)
            rpc_handler.setLevel(logging.DEBUG)
            rpc_logger.addHandler(rpc_handler)

    def _initialize_chain(self):
        """Initialize a pre-mined blockchain for use by the test."""

        def create_cachedir(cachedir):
            if os.path.isdir(cachedir):
                shutil.rmtree(cachedir)
            os.makedirs(cachedir)

        def copy_cachedir(origin, destination, num_nodes=MAX_NODES):
            for i in range(num_nodes):
                from_dir = get_datadir_path(origin, i)
                to_dir = get_datadir_path(destination, i)
                shutil.copytree(from_dir, to_dir)
                initialize_datadir(destination, i)  # Overwrite port/rpcport in t-notes.conf

        def clone_cache_from_node_1(cachedir, from_num=4):
            """ Clones cache subdir from node 1 to nodes from 'from_num' to MAX_NODES"""
            def copy_and_overwrite(from_path, to_path):
                if os.path.exists(to_path):
                    shutil.rmtree(to_path)
                shutil.copytree(from_path, to_path)
            assert from_num < MAX_NODES
            node_0_datadir = os.path.join(get_datadir_path(cachedir, 0), "regtest")


text:
#     Copyright 2018, Kay Hayen, mailto:kay.hayen@gmail.com
#
#     Python tests originally created or extracted from other peoples work. The
#     parts were too small to be protected.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
from __future__ import print_function

def closureTest1():
    # Assign, but the value is not supposed to be used by the function, instead the later
    # update is effective.
    d = 1

    def subby():
        return d

    d = 22222*2222

    return subby()

def closureTest2():
    # Using a closure variable that is not initialized at the time it is closured should
    # work as well.

    def subby():
        return d

    d = 2222*2222

    return subby()

def closureTest3():
    def subby():
        return undefined_global  # @UndefinedVariable

    try:
        return subby()
    except NameError:
        return 88

d = 1

def scopeTest4():
    try:
        return d

        d = 1
    except UnboundLocalError as e:
        return repr(e)

print("Test closure where value is overwritten:", closureTest1())
print("Test closure where value is assigned only late:", closureTest2())

print("Test function where closured value is never assigned:", closureTest3())

print("Scope test where UnboundLocalError is expected:", scopeTest4())

def function():
    pass

class ClosureLocalizerClass:
    print("Function before assigned in a class:", function)

    function = 1

    print("Function after it was assigned in class:", function)

ClosureLocalizerClass()

def ClosureLocalizerFunction():
    try:
        function = function

        print("Function didn't give unbound local error")
    except UnboundLocalError as e:
        print("Function gave unbound local error when accessing function before assignment:", repr(e))

ClosureLocalizerFunction()

class X:
    def __init__(self, x):
        self.x = x

def changingClosure():
    print("Changing a closure taken value after it was taken.")

    a = 1

    def closureTaker():
        return X(a)

    x = closureTaker()
    a=2
    print("Closure value first time:", x.x)
    x = closureTaker()
    print("Closure value second time:", x.x)

changingClosure()


text:
import disnake
from disnake.ext import commands

# Define a simple View that persists between bot restarts
# In order a view to persist between restarts it needs to meet the following conditions:
# 1) The timeout of the View has to be set to None
# 2) Every item in the View has to have a custom_id set
# It is recommended that the custom_id be sufficiently unique to
# prevent conflicts with other buttons the bot sends.
# For this example the custom_id is prefixed with the name of the bot.
# Note that custom_ids can only be up to 100 characters long.
class PersistentView(disnake.ui.View):
    def __init__(self):
        super().__init__(timeout=None)

    @disnake.ui.button(
        label="Green", style=disnake.ButtonStyle.green, custom_id="persistent_view:green"
    )
    async def green(self, button: disnake.ui.Button, interaction: disnake.MessageInteraction):
        await interaction.response.send_message("This is green.", ephemeral=True)

    @disnake.ui.button(label="Red", style=disnake.ButtonStyle.red, custom_id="persistent_view:red")
    async def red(self, button: disnake.ui.Button, interaction: disnake.MessageInteraction):
        await interaction.response.send_message("This is red.", ephemeral=True)

    @disnake.ui.button(
        label="Grey", style=disnake.ButtonStyle.grey, custom_id="persistent_view:grey"
    )
    async def grey(self, button: disnake.ui.Button, interaction: disnake.MessageInteraction):
        await interaction.response.send_message("This is grey.", ephemeral=True)

class PersistentViewBot(commands.Bot):
    def __init__(self):
        super().__init__(command_prefix=commands.when_mentioned)
        self.persistent_views_added = False

    async def on_ready(self):
        if not self.persistent_views_added:
            # Register the persistent view for listening here.
            # Note that this does not send the view to any message.
            # In order to do this you need to first send a message with the View, which is shown below.
            # If you have the message_id you can also pass it as a keyword argument, but for this example
            # we don't have one.
            self.add_view(PersistentView())
            self.persistent_views_added = True

        print(f"Logged in as {self.user} (ID: {self.user.id})")
        print("------")

bot = PersistentViewBot()

@bot.command()
@commands.is_owner()
async def prepare(ctx: commands.Context):
    """Starts a persistent view."""
    # In order for a persistent view to be listened to, it needs to be sent to an actual message.
    # Call this method once just to store it somewhere.
    # In a more complicated program you might fetch the message_id from a database for use later.
    # However this is outside of the scope of this simple example.
    await ctx.send("What's your favourite colour?", view=PersistentView())

bot.run("token")


text:
"""
Django settings for HedgeFund project.

Generated by 'django-admin startproject' using Django 3.2.8.

For more information on this file, see
https://docs.djangoproject.com/en/3.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""

from pathlib import Path
import os

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
STATIC_ROOT = BASE_DIR / 'staticfiles'

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-v)1&-+#yg2xdd$jddc#$z-c!1gu%v*%_^*_&ppw@pm^x%w-^*8'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'sscapital',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'HedgeFund.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [BASE_DIR / 'templates']
        ,
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'HedgeFund.wsgi.application'

# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': BASE_DIR / 'db.sqlite3',
    }
}

# Password validation
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/3.2/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'America/Los_Angeles'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/

STATIC_URL = '/static/'
MEDIA_URL = '/media/'
MEDIA_ROOT = BASE_DIR / 'static'
STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static'),
)
LOGIN_REDIRECT_URL = '/administrator'

STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
STATICFILES_STORAGE = 'whitenoise.storage.CompressedStaticFilesStorage'

# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'


text:
import pandas as pd
import numpy as np
from numpy.linalg import inv

def get_ffme_returns():
    """
    Load the Fama-French Dataset for the returns of the Top and Bottom Deciles by MarketCap
    """
    me_m = pd.read_csv("data/Portfolios_Formed_on_ME_monthly_EW.csv",
                       header=0, index_col=0, na_values=-99.99)
    rets = me_m[['Lo 10', 'Hi 10']]
    rets.columns = ['SmallCap', 'LargeCap']
    rets = rets/100
    rets.index = pd.to_datetime(rets.index, format="%Y%m").to_period('M')
    return rets

def get_fff_returns():
    """
    Load the Fama-French Research Factor Monthly Dataset
    """
    rets = pd.read_csv("data/F-F_Research_Data_Factors_m.csv",
                       header=0, index_col=0, na_values=-99.99)/100
    rets.index = pd.to_datetime(rets.index, format="%Y%m").to_period('M')
    return rets

def get_hfi_returns():
    """
    Load and format the EDHEC Hedge Fund Index Returns
    """
    hfi = pd.read_csv("data/edhec-hedgefundindices.csv",
                      header=0, index_col=0, parse_dates=True)
    hfi = hfi/100
    hfi.index = hfi.index.to_period('M')
    return hfi

def get_ind_file(filetype, weighting="vw", n_inds=30):
    """
    Load and format the Ken French Industry Portfolios files
    Variant is a tuple of (weighting, size) where:
        weighting is one of "ew", "vw"
        number of inds is 30 or 49
    """
    if filetype is "returns":
        name = f"{weighting}_rets"
        divisor = 100
    elif filetype is "nfirms":
        name = "nfirms"
        divisor = 1
    elif filetype is "size":
        name = "size"
        divisor = 1
    else:
        raise ValueError(f"filetype must be one of: returns, nfirms, size")

    ind = pd.read_csv(f"data/ind{n_inds}_m_{name}.csv", header=0, index_col=0, na_values=-99.99)/divisor
    ind.index = pd.to_datetime(ind.index, format="%Y%m").to_period('M')
    ind.columns = ind.columns.str.strip()
    return ind

def get_ind_returns(weighting="vw", n_inds=30):
    """
    Load and format the Ken French Industry Portfolios Monthly Returns
    """
    return get_ind_file("returns", weighting=weighting, n_inds=n_inds)

def get_ind_nfirms(n_inds=30):
    """
    Load and format the Ken French 30 Industry Portfolios Average number of Firms
    """
    return get_ind_file("nfirms", n_inds=n_inds)

def get_ind_size(n_inds=30):
    """
    Load and format the Ken French 30 Industry Portfolios Average size (market cap)
    """
    return get_ind_file("size", n_inds=n_inds)

def get_ind_market_caps(n_inds=30, weights=False):
    """
    Load the industry portfolio data and derive the market caps
    """
    ind_nfirms = get_ind_nfirms(n_inds=n_inds)
    ind_size = get_ind_size(n_inds=n_inds)
    ind_mktcap = ind_nfirms * ind_size
    if weights:
        total_mktcap = ind_mktcap.sum(axis=1)
        ind_capweight = ind_mktcap.divide(total_mktcap, axis="rows")
        return ind_capweight
    #else
    return ind_mktcap

def get_total_market_index_returns(n_inds=30):
    """
    Load the 30 industry portfolio data and derive the returns of a capweighted total market index
    """
    ind_capweight = get_ind_market_caps(n_inds=n_inds)
    ind_return = get_ind_returns(weighting="vw", n_inds=n_inds)
    total_market_return = (ind_capweight * ind_return).sum(axis="columns")
    return total_market_return

def skewness(r):
    """
    Alternative to scipy.stats.skew()
    Computes the skewness of the supplied Series or DataFrame
    Returns a float or a Series
    """
    r = r[(r!=0) & (r.notnull())]

    demeaned_r = r - r.mean()
    # use the population standard deviation, so set dof=0
    sigma_r = r.std(ddof=0)
    exp = (demeaned_r**3).mean()
    return exp/sigma_r**3

def kurtosis(r):
    """
    Alternative to scipy.stats.kurtosis()
    Computes the kurtosis of the supplied Series or DataFrame
    Returns a float or a Series
    """
    r = r[(r!=0) & (r.notnull())]

    demeaned_r = r - r.mean()
    # use the population standard deviation, so set dof=0
    sigma_r = r.std(ddof=0)
    exp = (demeaned_r**4).mean()
    return exp/sigma_r**4

def compound(r):
    """
    returns the result of compounding the set of returns in r
    """
    return np.expm1(np.log1p(r).sum())

def annualize_rets(r):
    """
    Annualizes a set of returns
    We should infer the periods per year
    but that is currently left as an exercise
    to the reader :-)
    """
    r_valid = r[(r!=0) & (r.notnull())]

    date_beg = r_valid.agg(lambda x: x.first_valid_index())
    date_end = r_valid.agg(lambda x: x.last_valid_index())

    try:
        years_fraction = (date_end-date_beg).dt.days/365.2425
    except:
        years_fraction = (date_end-date_beg).days/365.2425

    compounded_growth = (1+r_valid).prod()

    return compounded_growth**(1/years_fraction)-1

def annualize_vol(r):
    """
    Annualizes the vol of a set of returns
    We should infer the periods per year
    but that is currently left as an exercise
    to the reader :-)
    """
    r_valid = r[(r!=0) & (r.notnull())]

    total_num_periods = r_valid.count()

    date_beg = r_valid.agg(lambda x: x.first_valid_index())
    date_end = r_valid.agg(lambda x: x.last_valid_index())

    try:
        years_fraction = (date_end-date_beg).dt.days/365.2425
    except:
        years_fraction = (date_end-date_beg).days/365.2425

    periods_per_year = total_num_periods/years_fraction

    return r_valid.std()*((periods_per_year)**0.5)

def sharpe_ratio(r, riskfree_rate):
    """
    Computes the annualized sharpe ratio of a set of returns
    """
    # convert the annual riskfree rate to per period
    r_valid = r[(r!=0) & (r.notnull())]

    total_num_periods = r_valid.count()

    date_beg = r_valid.agg(lambda x: x.first_valid_index())
    date_end = r_valid.agg(lambda x: x.last_valid_index())

    try:
        years_fraction = (date_end-date_beg).dt.days/365.2425
    except:
        years_fraction = (date_end-date_beg).days/365.2425

    periods_per_year = total_num_periods/years_fraction

    rf_per_period = (1+riskfree_rate)**(1/periods_per_year)-1
    excess_ret = r - rf_per_period
    ann_ex_ret = annualize_rets(excess_ret)
    ann_vol = annualize_vol(r)

    return ann_ex_ret/ann_vol

import scipy.stats
def is_normal(r, level=0.01):
    """
    Applies the Jarque-Bera test to determine if a Series is normal or not
    Test is applied at the 1% level by default
    Returns True if the hypothesis of normality is accepted, False otherwise
    """
    if isinstance(r, pd.DataFrame):
        return r.aggregate(is_normal)
    else:
        statistic, p_value = scipy.stats.jarque_bera(r)
        return p_value > level

def drawdown(return_series: pd.Series):
    """Takes a time series of asset returns.
       returns a DataFrame with columns for
       the wealth index,
       the previous peaks, and
       the percentage drawdown
    """
    wealth_index = 1000*(1+return_series).cumprod()
    previous_peaks = wealth_index.cummax()
    drawdowns = (wealth_index - previous_peaks)/previous_peaks
    return pd.DataFrame({"Wealth": wealth_index,
                         "Previous Peak": previous_peaks,
                         "Drawdown": drawdowns})

def semideviation(r):
    """
    Returns the semideviation aka negative semideviation of r
    r must be a Series or a DataFrame, else raises a TypeError
    """
    if isinstance(r, pd.Series):
        is_negative = r < 0
        return r[is_negative].std(ddof=0)
    elif isinstance(r, pd.DataFrame):
        return r.aggregate(semideviation)
    else:
        raise TypeError("Expected r to be a Series or DataFrame")

def var_historic(r, level=5):
    """
    Returns the historic Value at Risk at a specified level
    i.e. returns the number such that "level" percent of the returns
    fall below that number, and the (100-level) percent are above
    """
    r = r[(r!=0) & (r.notnull())]

    if isinstance(r, pd.DataFrame):
        return r.aggregate(var_historic, level=level)
    elif isinstance(r, pd.Series):
        return -np.percentile(r, level)
    else:
        raise TypeError("Expected r to be a Series or DataFrame")

def cvar_historic(r, level=5):
    """
    Computes the Conditional VaR of Series or DataFrame
    """
    r = r[(r!=0) & (r.notnull())]

    if isinstance(r, pd.Series):
        is_beyond = r <= -var_historic(r, level=level)
        return -r[is_beyond].mean()
    elif isinstance(r, pd.DataFrame):
        return r.aggregate(cvar_historic, level=level)
    else:
        raise TypeError("Expected r to be a Series or DataFrame")

from scipy.stats import norm
def var_gaussian(r, level=5, modified=False):
    """
    Returns the Parametric Gauusian VaR of a Series or DataFrame
    If "modified" is True, then the modified VaR is returned,
    using the Cornish-Fisher modification
    """
    # compute the Z score assuming it was Gaussian
    r = r[(r!=0) & (r.notnull())]

    z = norm.ppf(level/100)
    if modified:
        # modify the Z score based on observed skewness and kurtosis
        s = skewness(r)
        k = kurtosis(r)
        z = (z +
                (z**2 - 1)*s/6 +
                (z**3 -3*z)*(k-3)/24 -
                (2*z**3 - 5*z)*(s**2)/36
            )
    return -(r.mean() + z*r.std(ddof=0))

def portfolio_return(weights, returns):
    """
    Computes the return on a portfolio from constituent returns and weights
    weights are a numpy array or Nx1 matrix and returns are a numpy array or Nx1 matrix
    """
    return weights.T @ returns

def portfolio_vol(weights, covmat):
    """
    Computes the vol of a portfolio from a covariance matrix and constituent weights
    weights are a numpy array or N x 1 maxtrix and covmat is an N x N matrix
    """
    vol = (weights.T @ covmat @ weights)**0.5
    return vol

def plot_ef2(n_points, er, cov):
    """
    Plots the 2-asset efficient frontier
    """
    if er.shape[0] != 2 or er.shape[0] != 2:
        raise ValueError("plot_ef2 can only plot 2-asset frontiers")
    weights = [np.array([w, 1-w]) for w in np.linspace(0, 1, n_points)]
    rets = [portfolio_return(w, er) for w in weights]
    vols = [portfolio_vol(w, cov) for w in weights]
    ef = pd.DataFrame({
        "Returns": rets,
        "Volatility": vols
    })
    return ef.plot.line(x="Volatility", y="Returns", style=".-")

from scipy.optimize import minimize

def minimize_vol(target_return, er, cov):
    """
    Returns the optimal weights that achieve the target return
    given a set of expected returns and a covariance matrix
    """
    n = er.shape[0]
    init_guess = np.repeat(1/n, n)
    bounds = ((0.0, 1.0),) * n # an N-tuple of 2-tuples!
    # construct the constraints
    weights_sum_to_1 = {'type': 'eq',
                        'fun': lambda weights: np.sum(weights) - 1
    }
    return_is_target = {'type': 'eq',
                        'args': (er,),
                        'fun': lambda weights, er: target_return - portfolio_return(weights,er)
    }
    weights = minimize(portfolio_vol, init_guess,
                       args=(cov,), method='SLSQP',
                       options={'disp': False},
                       constraints=(weights_sum_to_1,return_is_target),
                       bounds=bounds)
    return weights.x

def tracking_error(r_a, r_b):
    """
    Returns the Tracking Error between the two return series
    """
    return np.sqrt(((r_a - r_b)**2).sum())

def msr(riskfree_rate, er, cov):
    """
    Returns the weights of the portfolio that gives you the maximum sharpe ratio
    given the riskfree rate and expected returns and a covariance matrix
    """
    n = er.shape[0]
    init_guess = np.repeat(1/n, n)
    bounds = ((0.0, 1.0),) * n # an N-tuple of 2-tuples!
    # construct the constraints
    weights_sum_to_1 = {'type': 'eq',
                        'fun': lambda weights: np.sum(weights) - 1
    }
    def neg_sharpe(weights, riskfree_rate, er, cov):
        """
        Returns the negative of the sharpe ratio
        of the given portfolio
        """
        r = portfolio_return(weights, er)
        vol = portfolio_vol(weights, cov)
        return -(r - riskfree_rate)/vol

    weights = minimize(neg_sharpe, init_guess,
                       args=(riskfree_rate, er, cov), method='SLSQP',
                       options={'disp': False},
                       constraints=(weights_sum_to_1,),
                       bounds=bounds)
    return weights.x

def gmv(cov):
    """
    Returns the weights of the Global Minimum Volatility portfolio
    given a covariance matrix
    """
    n = cov.shape[0]
    return msr(0, np.repeat(1, n), cov)

def optimal_weights(n_points, er, cov):
    """
    Returns a list of weights that represent a grid of n_points on the efficient frontier
    """
    target_rs = np.linspace(er.min(), er.max(), n_points)
    weights = [minimize_vol(target_return, er, cov) for target_return in target_rs]
    return weights

def plot_ef(n_points, er, cov, style='.-', legend=False, show_cml=False, riskfree_rate=0, show_ew=False, show_gmv=False):
    """
    Plots the multi-asset efficient frontier
    """
    weights = optimal_weights(n_points, er, cov)
    rets = [portfolio_return(w, er) for w in weights]
    vols = [portfolio_vol(w, cov) for w in weights]
    ef = pd.DataFrame({
        "Returns": rets,
        "Volatility": vols
    })
    ax = ef.plot.line(x="Volatility", y="Returns", style=style, legend=legend)
    if show_cml:
        ax.set_xlim(left = 0)
        # get MSR
        w_msr = msr(riskfree_rate, er, cov)
        r_msr = portfolio_return(w_msr, er)
        vol_msr = portfolio_vol(w_msr, cov)
        # add CML
        cml_x = [0, vol_msr]
        cml_y = [riskfree_rate, r_msr]
        ax.plot(cml_x, cml_y, color='green', marker='o', linestyle='dashed', linewidth=2, markersize=10)
    if show_ew:
        n = er.shape[0]
        w_ew = np.repeat(1/n, n)
        r_ew = portfolio_return(w_ew, er)
        vol_ew = portfolio_vol(w_ew, cov)
        # add EW
        ax.plot([vol_ew], [r_ew], color='goldenrod', marker='o', markersize=10)
    if show_gmv:
        w_gmv = gmv(cov)
        r_gmv = portfolio_return(w_gmv, er)
        vol_gmv = portfolio_vol(w_gmv, cov)
        # add EW
        ax.plot([vol_gmv], [r_gmv], color='midnightblue', marker='o', markersize=10)

        return ax

def run_cppi(risky_r, safe_r=None, m=3, start=1000, floor=0.8, riskfree_rate=0.03, drawdown=None):
    """
    Run a backtest of the CPPI strategy, given a set of returns for the risky asset
    Returns a dictionary containing: Asset Value History, Risk Budget History, Risky Weight History
    """
    # set up the CPPI parameters
    dates = risky_r.index
    n_steps = len(dates)
    account_value = start
    floor_value = start*floor
    peak = account_value
    if isinstance(risky_r, pd.Series):
        risky_r = pd.DataFrame(risky_r, columns=["R"])

    if safe_r is None:
        safe_r = pd.DataFrame().reindex_like(risky_r)
        safe_r.values[:] = riskfree_rate/12 # fast way to set all values to a number
    # set up some DataFrames for saving intermediate values
    account_history = pd.DataFrame().reindex_like(risky_r)
    risky_w_history = pd.DataFrame().reindex_like(risky_r)
    cushion_history = pd.DataFrame().reindex_like(risky_r)
    floorval_history = pd.DataFrame().reindex_like(risky_r)
    peak_history = pd.DataFrame().reindex_like(risky_r)

    for step in range(n_steps):
        if drawdown is not None:
            peak = np.maximum(peak, account_value)
            floor_value = peak*(1-drawdown)
        cushion = (account_value - floor_value)/account_value
        risky_w = m*cushion
        risky_w = np.minimum(risky_w, 1)
        risky_w = np.maximum(risky_w, 0)
        safe_w = 1-risky_w
        risky_alloc = account_value*risky_w
        safe_alloc = account_value*safe_w
        # recompute the new account value at the end of this step
        account_value = risky_alloc*(1+risky_r.iloc[step]) + safe_alloc*(1+safe_r.iloc[step])
        # save the histories for analysis and plotting
        cushion_history.iloc[step] = cushion
        risky_w_history.iloc[step] = risky_w
        account_history.iloc[step] = account_value
        floorval_history.iloc[step] = floor_value
        peak_history.iloc[step] = peak
    risky_wealth = start*(1+risky_r).cumprod()
    backtest_result = {
        "Wealth": account_history,
        "Risky Wealth": risky_wealth,
        "Risk Budget": cushion_history,
        "Risky Allocation": risky_w_history,
        "m": m,
        "start": start,
        "floor": floor,
        "risky_r":risky_r,
        "safe_r": safe_r,
        "drawdown": drawdown,
        "peak": peak_history,
        "floor": floorval_history
    }
    return backtest_result

def summary_stats(r, riskfree_rate=0.03):
    """
    Return a DataFrame that contains aggregated summary stats for the returns in the columns of r
    """
    ann_r = annualize_rets(r)
    ann_vol = annualize_vol(r)
    ann_sr = sharpe_ratio(r, riskfree_rate=riskfree_rate)
    dd = r.aggregate(lambda r: drawdown(r).Drawdown.min())
    skew = r.aggregate(skewness)
    kurt = r.aggregate(kurtosis)
    cf_var5 = r.aggregate(var_gaussian, modified=True)
    hist_cvar5 = r.aggregate(cvar_historic)
    return pd.DataFrame({
        "Annualized Return": ann_r,
        "Annualized Vol": ann_vol,
        "Skewness": skew,
        "Kurtosis": kurt,
        "Cornish-Fisher VaR (5%)": cf_var5,
        "Historic CVaR (5%)": hist_cvar5,
        "Sharpe Ratio": ann_sr,
        "Max Drawdown": dd
    })

def gbm(n_years = 10, n_scenarios=1000, mu=0.07, sigma=0.15, steps_per_year=12, s_0=100.0, prices=True):
    """
    Evolution of Geometric Brownian Motion trajectories, such as for Stock Prices through Monte Carlo
    :param n_years:  The number of years to generate data for
    :param n_paths: The number of scenarios/trajectories
    :param mu: Annualized Drift, e.g. Market Return
    :param sigma: Annualized Volatility
    :param steps_per_year: granularity of the simulation
    :param s_0: initial value
    :return: a numpy array of n_paths columns and n_years*steps_per_year rows
    """
    # Derive per-step Model Parameters from User Specifications
    dt = 1/steps_per_year
    n_steps = int(n_years*steps_per_year) + 1
    # the standard way ...
    # rets_plus_1 = np.random.normal(loc=mu*dt+1, scale=sigma*np.sqrt(dt), size=(n_steps, n_scenarios))
    # without discretization error ...
    rets_plus_1 = np.random.normal(loc=(1+mu)**dt, scale=(sigma*np.sqrt(dt)), size=(n_steps, n_scenarios))
    rets_plus_1[0] = 1
    ret_val = s_0*pd.DataFrame(rets_plus_1).cumprod() if prices else rets_plus_1-1
    return ret_val

import statsmodels.api as sm
def regress(dependent_variable, explanatory_variables, alpha=True):
    """
    Runs a linear regression to decompose the dependent variable into the explanatory variables
    returns an object of type statsmodel's RegressionResults on which you can call
       .summary() to print a full summary
       .params for the coefficients
       .tvalues and .pvalues for the significance levels
       .rsquared_adj and .rsquared for quality of fit
    """
    if alpha:
        explanatory_variables = explanatory_variables.copy()
        explanatory_variables["Alpha"] = 1

    lm = sm.OLS(dependent_variable, explanatory_variables).fit()
    return lm

def portfolio_tracking_error(weights, ref_r, bb_r):
    """
    returns the tracking error between the reference returns
    and a portfolio of building block returns held with given weights
    """
    return tracking_error(ref_r, (weights*bb_r).sum(axis=1))

def style_analysis(dependent_variable, explanatory_variables):


text:
"""Utilties for distributed processing"""

import horovod.tensorflow.keras as hvd

def rank():
    try:
        return hvd.rank()
    except ValueError:
        return 0

def barrier():
    try:
        hvd.allreduce([], name='Barrier')
    except ValueError:
        pass