from datetime import datetime, timedelta

import pandas as pd
import six
import ujson
import validators
from sqlalchemy import JSON, Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.orm import relationship
from tornado_sqlalchemy_login.sqla.models import APIKey, Base, User

APIKey = APIKey

class Client(User):
    __mapper_args__ = {"polymorphic_identity": "client"}

    competitions = relationship("Competition", back_populates="user")
    submissions = relationship("Submission", back_populates="user")

class Competition(Base):
    __tablename__ = "competitions"
    competition_id = Column(Integer, primary_key=True)
    title = Column(String(500), nullable=False)
    subtitle = Column(String(500), nullable=False)

    user_id = Column(Integer, ForeignKey("users.id", ondelete="cascade"))
    user = relationship(Client, back_populates="competitions")

    type = Column(String(10), nullable=False)
    expiration = Column(DateTime, nullable=False)
    prize = Column(Integer, nullable=False)
    metric = Column(String(10), nullable=False)

    targets = Column(String(500), nullable=True)

    dataset = Column(JSON, nullable=True)
    dataset_url = Column(String(500), nullable=True)
    dataset_type = Column(String(10), nullable=True)

    dataset_kwargs = Column(JSON, nullable=True)
    dataset_key = Column(String(500), nullable=True)

    num_classes = Column(Integer, nullable=True)
    when = Column(DateTime, nullable=True)

    answer = Column(JSON, nullable=True)
    answer_url = Column(String(500), nullable=True)  # TODO
    answer_type = Column(String(10), nullable=True)
    answer_delay = Column(Integer, nullable=True)  # TODO

    timestamp = Column(DateTime, nullable=False, default=datetime.utcnow)

    submissions = relationship("Submission", back_populates="competition")

    def __repr__(self):
        return "<Competition(id='%s', userId='%s')>" % (self.id, self.userId)

    def to_dict(self, private=False):
        ret = {}
        for item in (
            "competition_id",
            "title",
            "user_id",
            "type",
            "expiration",
            "prize",
            "metric",
            "targets",
            "dataset",
            "dataset_url",
            "dataset_type",
            "dataset_kwargs",
            "num_classes",
            "when",
            "timestamp",
        ):
            ret[item] = getattr(self, item)
        if private:
            for item in (
                "answer",
                "answer_url",
                "answer_type",
            ):
                ret[item] = getattr(self, item)
        return ret

    @staticmethod
    def from_spec(user_id, spec):
        c = Competition(
            user_id=user_id,
            title=spec.title,
            subtitle=spec.subtitle,
            expiration=spec.expiration + timedelta(seconds=spec.answer_delay),
            type=spec.type.value,
            prize=spec.prize,
            metric=spec.metric.value,
            dataset=spec.dataset
            if isinstance(spec.dataset, six.string_types)
            and validators.url(spec.dataset)
            else spec.dataset.to_dict()
            if isinstance(spec.dataset, pd.DataFrame)
            else ""
            if not spec.dataset
            else ujson.loads(spec.dataset),
            dataset_type=spec.dataset_type.value,
            dataset_kwargs=spec.dataset_kwargs,
            dataset_key=spec.dataset_key,
            num_classes=spec.num_classes,
            targets=spec.targets
            if isinstance(spec.targets, six.string_types)
            else ujson.dumps(spec.targets),
            when=spec.when,
            answer=spec.answer
            if isinstance(spec.answer, six.string_types) and validators.url(spec.answer)
            else spec.answer.to_dict()
            if isinstance(spec.answer, pd.DataFrame)
            else ""
            if not spec.answer
            else ujson.loads(spec.answer),
            answer_type=spec.dataset_type.value
            if spec.answer_type.value == "none"
            else spec.answer_type.value,
            answer_delay=spec.answer_delay,
            timestamp=datetime.now(),
        )
        return c

class Submission(Base):
    __tablename__ = "submissions"
    submission_id = Column(Integer, primary_key=True)

    user_id = Column(
        Integer,
        ForeignKey("users.id", ondelete="cascade"),
        nullable=False,
    )
    user = relationship(Client, back_populates="submissions")

    competition_id = Column(
        Integer,
        ForeignKey("competitions.competition_id", ondelete="cascade"),
        nullable=False,
    )
    competition = relationship(Competition, back_populates="submissions")

    score = Column(Integer)

    answer = Column(JSON, nullable=True)
    answer_url = Column(String(500), nullable=True)  # TODO
    answer_type = Column(String(10), nullable=True)

    timestamp = Column(DateTime, nullable=False, default=datetime.utcnow)

    def __repr__(self):
        return "<Submission(id='%s', userId='%s', competitionId='%s')>" % (
            self.id,
            self.userId,
            self.competitionId,
        )

    def to_dict(self, private=False):
        ret = {}
        for item in (
            "submission_id",
            "user_id",
            "competition_id",
            "score",
            "timestamp",
        ):
            ret[item] = getattr(self, item)
        if private:
            for item in (
                "answer",
                "answer_url",
                "answer_type",
            ):
                ret[item] = getattr(self, item)
        return ret

    @staticmethod
    def from_spec(user_id, competition_id, competition, spec):
        c = Submission(
            user_id=user_id,
            competition_id=competition_id,
            score=-1,
            competition=competition,
            answer=spec.answer
            if isinstance(spec.answer, six.string_types) and validators.url(spec.answer)
            else spec.answer.to_json()
            if isinstance(spec.answer, pd.DataFrame)
            else ""
            if not spec.answer
            else ujson.loads(spec.answer),
            answer_type=spec.answer_type.value,
            timestamp=datetime.now(),
        )
        return c


# -*- coding: utf-8 -*-
import scrapy
from scrapy_splash import SplashRequest

script = """
function main(splash)
    local url = splash.args.url
    assert(splash:go(url))
    assert(splash:wait(5))

    splash:set_viewport_full()

    return {
        png=splash:png(),
        cookies = splash:get_cookies(),
        html = splash:html()
    }
end
"""

script2 = """
function main(splash)
    splash:init_cookies(splash.args.cookies)
    local url = splash.args.url
    assert(splash:go(url))
    assert(splash:wait(0.5))
    return {
        cookies = splash:get_cookies(),
        html = splash:html()
    }
end
"""

class FahasaSpider(scrapy.Spider):
    name = 'fahasa'
    allowed_domains = ['fahasa.com']
    start_urls = [
        "https://www.fahasa.com/sach-trong-nuoc/van-hoc-trong-nuoc.html"
    ]

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url, self.parse, endpoint='execute',
                                args={'lua_source': script})

    def parse(self, response):
        # Get the next page and yield Request
        next_selector = response.xpath('//*[@title="Next"]/@href')
        # print('------------------------------------------------1')
        # print(next_selector.extract())
        for url in next_selector.extract():
            yield SplashRequest(url, endpoint='execute',
                                args={'lua_source': script2})

        # Get URL in page and yield Request
        url_selector = response.xpath(
            '//*[@class="product-name p-name-list"]/a/@href')
        # print('------------------------------------------------2')
        # print(url_selector.extract())
        for url in url_selector.extract():
            yield SplashRequest(url, callback=self.parse_item,
                                endpoint='execute',
                                args={'lua_source': script2})

    def parse_item(self, response):
        """
        Handle crawl logic here
        """
        print('------------------------------------------------')
        print('Title', response.xpath('//h1').extract()[0])


from django.contrib.auth import login
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType

from rest_framework import permissions
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from rest_framework.renderers import JSONRenderer

from knox.views import LoginView as KnoxLoginView

from .models import UserFileManager
from .serializers import UserFileManagerSerializer, PermissionSerializer
from .permissions import IsOwnerOrReadOnly

class LoginView(KnoxLoginView):
    permission_classes = (permissions.AllowAny,)
    authentication_classes = []

    def post(self, request, format=None):

        serializer = AuthTokenSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data['user']
        login(request, user)
        return super(LoginView, self).post(request, format=None)

class UserFileManagerListView(APIView):
    """
    List all users or create a new user
    """

    permission_classes = [permissions.IsAuthenticated, IsOwnerOrReadOnly]

    def get(self, request):
        users = UserFileManager.objects.all()
        serializer = UserFileManagerSerializer(users, many=True)
        return Response({'data': serializer.data})

    def post(self, request):
        serializer = UserFileManagerSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(
                {'data': serializer.data},
                status=status.HTTP_201_CREATED
            )
        return Response(
            {'error': serializer.errors},
            status=status.HTTP_400_BAD_REQUEST
        )

class PermissionListView(APIView):
    """
    List all permission or create a new permission.

    The new permission is created only in relation of `UserFileManager` model
    as a `content_type`
    """

    def get(self, request):
        django_permissions = Permission.objects.all()
        serializer = PermissionSerializer(django_permissions, many=True)
        return Response({'data': serializer.data})

    def post(self, request):
        codename = request.data['codename']
        name = request.data['name']
        content_type = ContentType.objects.get_for_model(UserFileManager)

        data = {
            'codename': codename,
            'name': name,
            'content_type': content_type.pk
        }

        serializer = PermissionSerializer(data=data)

        if serializer.is_valid():
            serializer.save()
            return Response(
                {'message': 'A new permission is created', 'created_permission': serializer.data},
                status.HTTP_201_CREATED
            )

        return Response(
            {'error': serializer.errors},
            status=status.HTTP_400_BAD_REQUEST
        )


import plotly.express as px
import pandas as pd

df = pd.read_csv("https://raw.githubusercontent.com/Tymiec/Formula_1_Visualized/master/testing/heatmap_nationalities/nationalities_counted.csv")

fig = px.pie(df,
    values="value",
    names="country_code",
    hole=.3,
    title="Narodowości Formuły 1",
    labels="country_code")
fig.update_traces(textposition='inside', textinfo='percent+label')

fig.show()


import logging
from datetime import date, timedelta
from fints.client import FinTS3PinTanClient
import requests
import json
import schedule
import time
import os
import sys

### Bank data
blz = os.environ['BANK_BLZ'] # Your bank's BLZ
username = os.environ['BANK_USERNAME'] # Your login name
pin = os.environ['BANK_PIN'] # Your banking PIN
bank_url = os.environ['BANK_URL'] # your banks fints url
product_id = os.environ['BANK_PRODUCT_ID'] # your fints product id

### Firefly data
token = os.environ['FIREFLY_TOKEN']
firefly_url = os.environ['FIREFLY_URL']
headers = {'Authorization' : 'Bearer ' + token, 'Accept' : 'application/json', 'Content-Type' : 'application/json'}

req = requests.get(firefly_url + '/api/v1/accounts', headers=headers)

if(req.status_code != 200):
    logging.error("Could not get accounts from Firefly. Aborting.")
    sys.exit()

account_ids = {} # a map of IBAN to Firefly account id

resp = json.loads(req.text)

for a in resp["data"]:
    if a["type"] == "accounts" and a["attributes"]["iban"] is not None:
        account_ids[a["attributes"]["iban"]] = int(a["id"])

logging.info("Retrieved accounts from firefly")

# Establish connection
logging.basicConfig(level=logging.INFO)
f = FinTS3PinTanClient(
    blz,
    username,
    pin,
    bank_url,
    product_id=product_id
)

def create_transactions(account, transactions):
    for transaction in transactions:
        t = {
            "amount": abs(transaction.data['amount'].amount).to_eng_string(),
            "currency_code": transaction.data['amount'].currency,
            "date": transaction.data['date'].strftime('%Y-%m-%dT08:00+02:00'),
            "reconciled": False,
            "description": transaction.data['purpose'],
            "external_id": transaction.data['id']
        }

        if transaction.data['status'] == "C": # a deposit
            t["type"] = "deposit"
            t["source_name"] = transaction.data["applicant_name"]
            t["destination_id"] = account_ids[account.iban]
        else:
            t["type"] = "withdrawal"
            t["source_id"] = account_ids[account.iban]
            t["destination_name"] = transaction.data["applicant_name"]

        data = {
            "error_if_duplicate_hash": True,
            "apply_rules": True,
            "group_title": transaction.data['posting_text'],
            "transactions": [t]
        }
        r = requests.post(firefly_url + '/api/v1/transactions', data=json.dumps(data), headers=headers)

        if(r.status_code != 200 and "Duplicate of transaction" not in r.text):
            logging.error('Failed posting transaction to api with error code: ' + str(r.status_code) + '.')
            logging.error('Error message is: ' + r.text)
            logging.error(json.dumps(data))

def get_transactions():
    with f:
        # Since PSD2, a TAN might be needed for dialog initialization. Let's check if there is one required
        if f.init_tan_response:
            logging.error('Bank access requires TAN which is not supported.')

        # Fetch accounts
        accounts = f.get_sepa_accounts()

        logging.info('Retrieved ' + str(len(accounts)) + ' accounts from bank.')

        # get the transactions for each account
        for account in accounts:
            transactions = f.get_transactions(account, date.today() - timedelta(days=7), date.today())
            logging.info('Retrieved ' + str(len(transactions)) + ' for account ' + account.iban + ' that took place in the last 7 days.')
            create_transactions(account, transactions)

schedule.every().day.at("10:00").do(get_transactions)

logging.info("Started scheduled job")

while True:
    schedule.run_pending()
    time.sleep(1)


import matplotlib.pyplot as plt
import pandas as pd

d = pd.read_csv('Data/276城_3source_by_ct_V3.csv')

df = d.groupby('age_feb20').mean()['locked_down']
plt.bar(x=df.index, height= df)

df = d.groupby('age_feb20').mean()['xc_lockdown']
plt.bar(x=df.index, height= df)

plt.show()

import seaborn as sns
sns.set(style="whitegrid")

# penguins = sns.load_dataset("penguins")

# Draw a nested barplot by species and sex
g = sns.catplot(
    data=d, kind="bar",
    x="age_feb20", y="locked_down",alpha=.6, height=6
)
g.despine(left=True)
g.set_axis_labels("", "Body mass (g)")
# g.legend.set_title("")
plt.show()


import json
import requests
import xlrd
from web3 import Web3
from eth_abi import encode_single, encode_abi

headers = {"Connection": "close", "Cache-Control": "max-age=0", "sec-ch-ua": "\" Not;A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"", "sec-ch-ua-mobile": "?0", "DNT": "1", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Sec-Fetch-Site": "none", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-User": "?1", "Sec-Fetch-Dest": "document", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en,zh-CN;q=0.9,zh;q=0.8"}

w3 = Web3(Web3.HTTPProvider('https://bsc-dataseed1.binance.org'))
abi = json.loads('[{"constant": true,"inputs": [{"name": "who", "type": "address"}],"name": "balanceOf","outputs": [{"name": "", "type": "uint256"}],"payable": false,"stateMutability": "view","type": "function"}, {"inputs": [{"internalType": "uint256", "name": "cumulativePayout", "type": "uint256"},{"internalType": "bytes", "name": "issuerSig", "type": "bytes"}],"name": "cashCheque","outputs": [],"stateMutability": "nonpayable","type": "function"}]')
GPS_ADDRESS = w3.toChecksumAddress('0x5e772acf0f20b0315391021e0884cb1f1aa4545c')
GPS_CONTRACT = w3.eth.contract(address=GPS_ADDRESS, abi=abi)

#表格第一列放钱包地址，第二列放私钥
file_name = "62ABI.xlsx"

## 获取钱包地址
def get_addresses():
    xls = xlrd.open_workbook(file_name)
    addresses = xls.sheets()[0].col_values(0)
    private_secret = xls.sheets()[0].col_values(1)
    #print("一共有 " + str(len(addresses)) + " 地址")
    return addresses,private_secret

## 获取拥有的gps
def get_gps(address):
    address = Web3.toChecksumAddress(address)
    return GPS_CONTRACT.functions.balanceOf(address).call()

## 获取交易所需参数
def get_api(address,private_secret):
    private_secret = private_secret
    url = 'https://api.gpfs.xyz/v1/cheque?address=' + address
    res = requests.get(url, headers=headers)
    text = json.loads(res.text)
    amount = text['data']['amount']
    paid_out = text['data']['paid_out']
    signature = text['data']['signature']
    sedsendSigned(address,amount,signature,private_secret)

def get_count(address):
    nonce = w3.toHex(w3.eth.get_transaction_count(address))
    return nonce

def sedsendSigned(address,amount,signature,private_secret):
    amount_16 = w3.toInt(encode_single('uint256', amount))
    signature_16 = w3.toBytes(hexstr=signature)
    data = GPS_CONTRACT.encodeABI(fn_name="cashCheque", args=[amount_16,signature_16])
    rawTx = {
        'from': address,
        'to': GPS_ADDRESS,
        'value': w3.toHex(w3.toWei('0', 'ether')),
        'nonce': get_count(address),
        'gas': w3.toHex(100000),
        'gasPrice': w3.toHex(w3.toWei('5', 'gwei')),
        'data': data
    }
    signed_txn = w3.eth.account.sign_transaction(rawTx,private_secret)
    txn_hash = w3.toHex(w3.eth.send_raw_transaction(signed_txn.rawTransaction))
    print("地址:{0}\t交易hash:{1}".format(address,txn_hash))
    try:
        status = w3.eth.wait_for_transaction_receipt(txn_hash,timeout=20)['status']
        if status:
            print("交易成功")
    except:
        print("报错")

if __name__ == '__main__':
    key = get_addresses()
    for i in range(len(key[0])):
        address = key[0][i]
        private_secret = key[1][i]
        get_api(w3.toChecksumAddress(address),private_secret)


import AppInfo

# Based on the work of Lena "Teekeks" <info@teawork.de>
import urllib.parse
import uuid
from typing import Union
from json import JSONDecodeError
from aiohttp.web import Request
import webbrowser
from aiohttp import web
import asyncio
from threading import Thread
from time import sleep
from os import path
import requests
from concurrent.futures._base import CancelledError
from logging import getLogger, Logger

class TwitchOauth:
    port: int = 17563
    host: str = "0.0.0.0"
    __state: str = ""

    __server_running: bool = False
    __loop: Union["asyncio.AbstractEventLoop", None] = None
    __runner: Union["web.AppRunner", None] = None
    __thread: Union["threading.Thread", None] = None

    __user_token: Union[str, None] = None

    __can_close: bool = False

    def __build_runner(self):
        app = web.Application()
        app.add_routes([web.get("/", self.__handle_callbackGet)])
        app.add_routes([web.post("/", self.__handle_callbackPost)])
        return web.AppRunner(app)

    async def __run_check(self):
        while not self.__can_close:
            try:
                await asyncio.sleep(1)
            except:
                self.__server_running = False
                pass
        for task in asyncio.all_tasks(self.__loop):
            task.cancel()
            self.__server_running = False

    def __run(self, runner: "web.AppRunner"):
        try:
            self.__runner = runner
            self.__loop = asyncio.new_event_loop()
            asyncio.set_event_loop(self.__loop)
            self.__loop.run_until_complete(runner.setup())
            site = web.TCPSite(runner, self.host, self.port)
            self.__loop.run_until_complete(site.start())
            self.__server_running = True
            self.__loop.run_until_complete(self.__run_check())
        except:
            self.__server_running = False
            pass

    def __start(self):
        self.__thread = Thread(target=self.__run, args=(self.__build_runner(),))
        self.__thread.start()

    def stop(self):
        """Manually stop the flow
        :rtype: None
        """
        self.__can_close = True
        self.__server_running = False

    async def __handle_callbackGet(self, request: "web.Request"):
        fn = path.join(path.dirname(__file__), "Failed.html")
        val = str(request.message)
        if not str.__contains__(val, "access_denied"):
            # Assume that everything else is fine and verify in the post.
            fn = path.join(path.dirname(__file__), "Success.html")
        fd = ""
        with open(fn, "r") as f:
            fd = f.read()

        return web.Response(text=fd, content_type="text/html")

    async def __handle_callbackPost(self, request: "web.Request"):
        val = str(request.content._buffer)
        fn = path.join(path.dirname(__file__), "Failed.html")
        if not str.__contains__(val, "access_denied"):
            val = val.split("#")[1].split("\\r\\n")[0].split("&")

            # invalid state!
            if val[2].split("=")[1] != self.__state:
                self.__server_running = False
                return web.Response(status=401)

            self.__user_token = val[0].split("=")[1]
            if self.__user_token is None:
                # must provide code
                self.__server_running = False
                return web.Response(status=400)

            fn = path.join(path.dirname(__file__), "Success.html")
        else:
            self.__user_token = "  "

        fd = ""
        with open(fn, "r") as f:
            fd = f.read()

        return web.Response(text=fd, content_type="text/html")

    def authenticate(self):
        self.__start()
        # wait for the server to start up
        while not self.__server_running:
            sleep(0.01)

        self.__state = str(uuid.uuid4())
        url = (
            "https://id.twitch.tv/oauth2/authorize?"
            + "response_type=token"
            + "&client_id="
            + AppInfo.ClientId
            + "&redirect_uri="
            + AppInfo.RedirectUri
            + "&scope=chat:read chat:edit"
            + "&force_verify=true"
            + "&state="
            + self.__state
        )
        # open in browser

        webbrowser.open(url, new=2)
        while self.__user_token is None and self.__server_running:
            sleep(0.1)
        self.stop()
        self.__state = ""
        return self.__user_token


# -*- coding: utf-8 -*-
# author:           inspurer(月小水长)
# pc_type           lenovo
# create_time:      2019/12/1 23:07
# file_name:        history_stock_quotes_scrapy.py
# github            https://github.com/inspurer
# qq邮箱            2391527690@qq.com
# 微信公众号         月小水长(ID: inspurer)

import requests

from lxml import etree

from stock_analysis.utils import myDict

import pymongo

client = pymongo.MongoClient(host="localhost",port=27017)
db = client['stock']
table_basic = db['basic']

basis = table_basic.find()
print(basis.count())
stock_basis = myDict.AllowKeyRepeatDict()
for basic in basis:
    stock_basis.add(key=basic['name'],value=basic['code'])

stock_name = '中信证券'
stock_code = stock_basis.query(key=stock_name)[0]
table_quotes = db[stock_name+'_'+stock_code]

headers = {
    'Referer': 'http://quotes.money.163.com/trade/lsjysj_600030.html?year=2019&season=4',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'
}

base_url = 'http://quotes.money.163.com/trade/lsjysj_{code}.html'
# 2019 年
year = 2019
season = 1
# 四季度
for season in range(1,5):
    params = {
        'year':year,
        'season':season
    }
    response = requests.get(url=base_url.format(code=stock_code),headers=headers,params=params)

    html = etree.HTML(response.text)
    for tr in html.xpath('//table[@class="table_bg001 border_box limit_sale"]/tr'):
        tds = tr.xpath('.//td/text()')
        date = tds[0].replace('-','')
        open,high,low,close = tds[1],tds[2],tds[3],tds[4]
        change,change_pencentage = tds[5],tds[6]
        deal_amount,deal_money = tds[7].replace(',',''),tds[8].replace(',','') # 成交量，成交金额（万元）
        print(date,open,high,low,close,change,change_pencentage,deal_amount,deal_money)
        data = {
            'date': int(date),
            'open': open,
            'high': high,
            'low': low,
            'close': close,
            'change': change,
            'change_pencentage': change_pencentage,
            'deal_amount': deal_amount,
            'deal_money': deal_money
        }
        # 上海和深圳可能有两个同名的股票
        # 上海股市也可能有同名股票，但是code不一样
        table_quotes.update_one({'date': date}, {'$set': data}, True)


import os, sys
from os.path import dirname, abspath

if getattr(sys, 'frozen', False):
    filedir = os.path.dirname(sys.executable)
elif __file__:
    filedir = os.path.dirname(os.path.abspath(__file__))

if getattr(sys, 'frozen', False):
    basedir = os.path.dirname(os.path.dirname(sys.executable))
elif __file__:
    basedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(1, basedir)
import config
import pandas as pd
import general.df_preprocessing as dfpre
import humanfriendly
from datetime import datetime, timedelta
from hurry.filesize import size
SEP = "\t"

def apply(dataframe):
    dataframe = dfpre.apply(dataframe)

    event_num = dataframe.shape[0]
    account_num = len(dataframe['ACCOUNT'].unique())
    jobs_num = len(dataframe['JOBID'].unique())

    min_time = str(dataframe['CURRENT_TIMESTAMP'].min()).split('.')[0] if '.' in str(dataframe['CURRENT_TIMESTAMP'].min()) else str(dataframe['CURRENT_TIMESTAMP'].min())
    max_time = str(dataframe['CURRENT_TIMESTAMP'].max()).split('.')[0] if '.' in str(dataframe['CURRENT_TIMESTAMP'].max()) else str(dataframe['CURRENT_TIMESTAMP'].max())

    slurmy_accounts = list()
    slurmy_jobs = list()
    print("start finding slurmys", datetime.now())
    for index, row in dataframe.iterrows():
        if '(unfulfilled)' in str(row['DEPENDENCY']) and row['DEPENDENCY'] != '(null)':
            if row['ACCOUNT'] not in slurmy_accounts:
                slurmy_accounts.append(row['ACCOUNT'])

            if row['JOBID'] not in slurmy_jobs:
                slurmy_jobs.append(row['JOBID'])
        else:
            pass

    slurmy_accounts_num = len(slurmy_accounts)
    slurmy_jobs_num = len(slurmy_jobs)
    print("start finding cpus", datetime.now())
    cpu_sum = 0
    for cpu_val in dataframe['CPUS'].tolist():
        s_cpu_val = str(cpu_val)
        if len(s_cpu_val)>0 and s_cpu_val.isnumeric():
            cpu_sum = cpu_sum + int(s_cpu_val)
        else:
            pass

    average_cpu_usage = cpu_sum / len(dataframe['CPUS'].tolist())
    print("start finding RAMs", datetime.now())
    RAM_list = list()
    for RAM in dataframe['MIN_MEMORY'].to_list():
        if RAM != 'None' and any(char.isdigit() for char in RAM) and RAM not in dataframe['PARTITION'].unique():
            num_bytes = humanfriendly.parse_size(RAM)
            RAM_list.append(num_bytes)

    average_RAM_usage = sum(RAM_list) / len(RAM_list)

    res = {
        "startTime": str(min_time),
        "endTime": str(max_time),
        "eventsNumber": str(event_num),
        "jobNumber": str(jobs_num),
        "accountNumber": str(account_num),
        "SLURMaccountsPercentage": str(round(slurmy_accounts_num/account_num, 2)),
        "SLURMaccounts": str(slurmy_accounts),
        "SLURMaccountNumber": str(slurmy_accounts_num) + " out of " + str(account_num),
        "SLURMjobssPercentage": str(round(slurmy_jobs_num/jobs_num, 2)),
        "SLURMjobNumber": str(slurmy_jobs_num) + " out of " + str(jobs_num),
        "CPUusageAvg": str(round(average_cpu_usage, 2)),
        "RAMusageAvg": str(size(round(average_RAM_usage, 2)))
    }

    return res

def account_batches(df, start_time_obj, duration, batches_list):
    batches_dict = {}
    duration_obj = datetime.strptime(duration, '%H:%M:%S')
    next_time = start_time_obj + timedelta(hours=duration_obj.hour, minutes=duration_obj.minute, seconds=duration_obj.second)

    df['CURRENT_TIMESTAMP'] = pd.to_datetime(df.CURRENT_TIMESTAMP, format='%Y-%m-%d %H:%M:%S')
    new_df = df[(start_time_obj<df['CURRENT_TIMESTAMP']) & (df['CURRENT_TIMESTAMP']<next_time)]
    for account in new_df.ACCOUNT.unique():
        account_df = new_df[new_df['ACCOUNT']==account]
        batches_dict[account] = len(account_df.JOBID.unique())

    if len(batches_dict) > 0:
        batches_list.append(batches_dict)
    return next_time, batches_list

def num_of_intervals_an_account_did_batching(batches_list, threshold, num_of_intervals_account_did_batches):
    for batches in batches_list:
        for account in batches:
            if batches[account] >= threshold:
                if account in num_of_intervals_account_did_batches:
                    num_of_intervals_account_did_batches[account] += 1
                else:
                    num_of_intervals_account_did_batches[account] = 1

    return num_of_intervals_account_did_batches

def get_batches(df, duration, threshold):
    df = dfpre.apply(df)
    start_time = df['CURRENT_TIMESTAMP'].min()
    start_time_obj = datetime.strptime(start_time.split('.')[0], '%Y-%m-%d %H:%M:%S')
    end_time = df['CURRENT_TIMESTAMP'].max()
    end_time_obj = datetime.strptime(end_time.split('.')[0], '%Y-%m-%d %H:%M:%S')

    batches_list = []
    while start_time_obj < end_time_obj:
        next_time, new_batches_list = account_batches(df, start_time_obj, duration, batches_list)
        batches_list = new_batches_list
        start_time_obj = next_time

    num_of_intervals_account_did_batches = num_of_intervals_an_account_did_batching(batches_list, threshold, {})

    return num_of_intervals_account_did_batches

if __name__ == "__main__":
    dataframe = pd.read_csv(config.logger.local_log, sep=SEP, encoding="iso-8859-1", error_bad_lines=False)
    print(apply(dataframe))


# -*- coding:utf-8 -*-
import logging
import os
import sys
import time
import threading

import requests

# 任务执行间隔
delay = ''
# 目标 URL
url = ''
# 日志文件名
log = 'common.log'

# 单独线程请求 api
def request_api():
    try:
        logging.info(requests.get(url, timeout=5).text)
    except requests.exceptions.ConnectTimeout as timeout:
        logging.info('this request timeout.')
    except Exception as e:
        logging.info(e)

# 任务服务器
# -p 持续时间
# -t 任务执行间隔
# -s 开始时间
# -u 目标 url
# -d 日志文件名称
if __name__ == '__main__':
    # 循环读取参数
    for count in range(0, len(sys.argv)):
        arg = sys.argv[count]
        if arg == '-t':
            delay = sys.argv[count + 1]
        if arg == '-u':
            url = sys.argv[count + 1]
        if arg == '-d':
            log = sys.argv[count + 1]

    # 自动创建目录文件
    if not os.path.exists('./log'):
        os.makedirs('./log')
    # 日志配置
    logging.basicConfig(level=logging.DEBUG, filename=('./log/' + log + '.log'), filemode="w",
                        format="%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
    # 任务进行中
    while True:
        threading.Thread(target=request_api).start()
        time.sleep(float(delay))


from django.shortcuts import render
from django.views import generic
from django.core.paginator import PageNotAnInteger, Paginator, EmptyPage

from .models import Dytt8

class IndexView(generic.ListView):
    template_name = 'movie/index.html'
    context_object_name = 'movie_list'
    paginate_by = 20

    def get_queryset(self):
        return Dytt8.objects.exclude(link__isnull=True).order_by('-name')

def search(request):
    q = request.GET.get('q')
    error_msg = ''
    if not q:
        error_msg = '请输入关键词'
        return render(request, 'movie/index.html', {'error_msg': error_msg})
    movie_list = Dytt8.objects.filter(name__icontains=q)
    return render(request, 'movie/index.html', {'error_msg': error_msg,
                                                  'movie_list': movie_list})

def index(request):
    movie = Dytt8.objects.exclude(link__isnull=True)
    paginator = Paginator(movie, 20)
    page = request.GET.get('page')
    try:
        movie = paginator.page(page)
    except PageNotAnInteger:
        movie = paginator.page(1)
    except EmptyPage:
        movie = paginator.page(paginator.num_pages)
    return render(request, 'movie/index1.html', {'movie': movie})


from tensorflow.python.eager.context import num_gpus
import tensorflow_datasets as tfds
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from denseNet import DenseNet
from resblockmodel import MyResModel
import datetime
import argparse

def prepare_cifar(cifar10):
    # convert data from uint8 to float32
    cifar10 = cifar10.map(lambda img, target: (tf.cast(img, tf.float32), target))
    # sloppy input normalization, just bringing image values from range [0, 255] to [-1, 1]
    cifar10 = cifar10.map(lambda img, target: (tf.image.per_image_standardization(img), target))
    # create one-hot targets
    cifar10 = cifar10.map(lambda img, target: (img, tf.one_hot(target, depth=10)))
    # cache this progress in memory, as there is no need to redo it; it is deterministic after all
    cifar10 = cifar10.cache()
    # shuffle, batch, prefetch
    cifar10 = cifar10.shuffle(1000)
    cifar10 = cifar10.batch(64)
    cifar10 = cifar10.prefetch(32)
    # return preprocessed dataset
    return cifar10

@tf.function
def train_step(model, input, target, loss_function, optimizer):
    """
    This function executes a training step on the given network.
    Using gradient descend
    :param model: the network model
    :param input: the given input tensors
    :param target: the given target tensors
    :param loss_function: the given loss function
    :param optimizer: the given optimizer
    :return: the loss during this trainin step
    """
    # loss_object and optimizer_object are instances of respective tensorflow classes
    with tf.GradientTape() as tape:
        prediction = model(input, training=True)
        loss = loss_function(target, prediction)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    return loss

def test(model, test_data, loss_function):
    """
    # Tests over complete test data
    :param model: the network model
    :param test_data: the given input tensors
    :param loss_function: the given loss function
    :return: the test loss and test accuracy
    """
    test_accuracy_aggregator = []
    test_loss_aggregator = []
    for (input, target) in test_data:
        prediction = model(input, training=False)
        sample_test_loss = loss_function(target, prediction)
        sample_test_accuracy =  np.argmax(target, axis=1) == np.argmax(prediction, axis=1)
        sample_test_accuracy = np.mean(sample_test_accuracy)
        test_accuracy_aggregator.append(sample_test_accuracy)
        test_loss_aggregator.append(sample_test_loss.numpy())

    test_loss = tf.reduce_mean(test_loss_aggregator)
    test_accuracy = tf.reduce_mean(test_accuracy_aggregator)
    return test_loss, test_accuracy

if __name__ == "__main__":
    #parse arguments to determine which model should be used
    parser = argparse.ArgumentParser(description='ResNet or DenseNet')
    parser.add_argument('--model', type=str, help='ResNet of DenseNet', required=True)
    args = parser.parse_args()

    tf.keras.backend.clear_session()

    # loading the data set
    train_ds, test_ds = tfds.load('cifar10', split=['train', 'test'], as_supervised=True)
    train_ds = train_ds.apply(prepare_cifar)
    test_ds = test_ds.apply(prepare_cifar)
    #print(train_ds.take(1))
    # test preprocessing
    #for elem in train_ds:
    #    print(elem)
    #    break
    ### Hyperparameters
    num_epochs = 30
    learning_rate = tf.constant(0.001, dtype=tf.float32)
    # Initialize the loss: categorical cross entropy.
    cross_entropy_loss = tf.keras.losses.CategoricalCrossentropy()
    # Initialize the optimizer: SGD with default parameters.
    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
    # Initialize lists for later visualization.
    train_losses = []
    test_losses = []
    test_accuracies = []

    #create the model
    if args.model == 'ResNet':
        model = MyResModel()
    elif args.model == 'DenseNet':
        model = DenseNet()
    else:
        raise ValueError("Invalid model passed in as argument. Should be either ResNet or DenseNet!")

    # testing once before we begin
    test_loss, test_accuracy = test(model, test_ds, cross_entropy_loss)
    test_losses.append(test_loss)
    test_accuracies.append(test_accuracy)
    # check how model performs on train data once before we begin
    train_loss, _ = test(model, train_ds, cross_entropy_loss)
    train_losses.append(train_loss)
    #   We train for num_epochs epochs.
    for epoch in range(num_epochs):
        start_time = datetime.datetime.now()
        print(f'Epoch: {str(epoch)} starting with accuracy {test_accuracies[-1]}')
        # training (and checking in with training)
        epoch_loss_agg = []
        for input, target in train_ds:
            train_loss = train_step(model, input, target, cross_entropy_loss, optimizer)
            epoch_loss_agg.append(train_loss)
        # track training loss
        train_losses.append(tf.reduce_mean(epoch_loss_agg))
        # testing, so we can track accuracy and test loss
        test_loss, test_accuracy = test(model, test_ds, cross_entropy_loss)
        test_losses.append(test_loss)
        test_accuracies.append(test_accuracy)
        diff_time = datetime.datetime.now() - start_time
        print(f"Epoch {epoch} took {diff_time} to complete.")

    print(model.summary())
    # Visualize accuracy and loss for training and test data.
    plt.subplot(1, 1, 1)
    line1, = plt.plot(train_losses, '-x')
    line2, = plt.plot(test_losses, '-+')
    line3, = plt.plot(test_accuracies, '-o')
    plt.xlabel("Training steps")
    plt.ylabel("Loss/Accuracy")
    plt.legend((line1, line2, line3), ("training loss", "test loss", "test accuracy"))
    plt.show()


### write a script that resizes all images in a directory to 100 x 100 ###
import cv2
import glob

list_of_jpgs = glob.glob("batch_image_resizing/sample_images/*.jpg")

def trim_file_path(raw_path):
    return raw_path[35:]

trimmed_paths = list(map(trim_file_path, list_of_jpgs))

for index, image in enumerate(list_of_jpgs):
    print(index, image)
    cur_image = cv2.imread(image, 0)
    new_image = cv2.resize(cur_image, [100, 100])
    cv2.imwrite("batch_image_resizing/sample_images/resized_" + trimmed_paths[index], new_image)


import pygame as pg
import pytmx
from settings import WIDTH, HEIGHT

class TiledMap:
    def __init__(self, filename):
        self.tmxdata = pytmx.load_pygame(filename, pixelalpha=True)
        self.width = self.tmxdata.width * self.tmxdata.tilewidth
        self.height = self.tmxdata.height * self.tmxdata.tileheight

    def render(self, surface):
        ti = self.tmxdata.get_tile_image_by_gid
        for layer in self.tmxdata.visible_layers:
            if isinstance(layer, pytmx.TiledTileLayer):
                for x, y, gid, in layer:
                    tile = ti(gid)
                    if tile:
                        surface.blit(tile, (x * self.tmxdata.tilewidth, y * self.tmxdata.tileheight))

    def make_map(self):
        temp_surface = pg.Surface((self.width, self.height))
        self.render(temp_surface)
        return temp_surface

class Camera:
    def __init__(self, width, height):
        """
        Creates a camera object
        :param width: The width of the cameras view
        :param height: The height of the cameras view
        """
        self.camera = pg.Rect(0, 0, width, height)
        self.width = width
        self.height = height

    def apply(self, entity):
        """
        Applies the camera's focus on the target entity
        :param entity: The focus of the camera
        :return: The entity's rect moved according to the camera position
        """
        # Applies the camera offset to the entity
        return entity.rect.move(self.camera.topleft)

    def apply_to_point(self, point):
        return [point[0] + self.camera.x, point[1] + self.camera.y]

    def apply_rect(self, rect, copy=False):
        """
        Applies the camera's focus on the target rectangle
        :param rect: The focus of the camera
        :return: The moved rectangle according to the camera's position
        """
        # Applies the camera offset to
        # a rectangle
        if not copy:
            return rect.move(self.camera.topleft)
        else:
            x = rect.copy()
            return x.move(self.camera.topleft)

    def update(self, target):
        """
        Updates the location of the camera relative to the target
        :param target: The focus of the camera
        :return: None
        """
        # Get how much the target moved in the x & y
        # Objects that the pass by the target move in the opposite direction of the target
        x = -target.rect.centerx + (WIDTH // 2)
        y = -target.rect.centery + (HEIGHT // 2)

        # Limit the scrolling on all four sides of the screen
        x = min(0, x)  # left
        y = min(0, y)  # top
        x = max(-(self.width - WIDTH), x)  # right
        y = max(-(self.height - HEIGHT), y)  # bottom

        # Update the camera
        self.camera = pg.Rect(x, y, self.width, self.height)


"""Parses SAM given a template"""
import logging

from samcli.commands.local.cli_common.user_exceptions import InvalidSamTemplateException
from samcli.commands.local.lib.cfn_base_api_provider import CfnBaseApiProvider

LOG = logging.getLogger(__name__)

class CfnApiProvider(CfnBaseApiProvider):
    APIGATEWAY_RESTAPI = "AWS::ApiGateway::RestApi"
    APIGATEWAY_STAGE = "AWS::ApiGateway::Stage"
    TYPES = [
        APIGATEWAY_RESTAPI,
        APIGATEWAY_STAGE
    ]

    def extract_resources(self, resources, collector, cwd=None):
        """
        Extract the Route Object from a given resource and adds it to the RouteCollector.

        Parameters
        ----------
        resources: dict
            The dictionary containing the different resources within the template

        collector: samcli.commands.local.lib.route_collector.RouteCollector
            Instance of the API collector that where we will save the API information

        cwd : str
            Optional working directory with respect to which we will resolve relative path to Swagger file

        Return
        -------
        Returns a list of routes
        """
        for logical_id, resource in resources.items():
            resource_type = resource.get(CfnBaseApiProvider.RESOURCE_TYPE)
            if resource_type == CfnApiProvider.APIGATEWAY_RESTAPI:
                self._extract_cloud_formation_route(logical_id, resource, collector, cwd=cwd)

            if resource_type == CfnApiProvider.APIGATEWAY_STAGE:
                self._extract_cloud_formation_stage(resources, resource, collector)

    def _extract_cloud_formation_route(self, logical_id, api_resource, collector, cwd=None):
        """
        Extract APIs from AWS::ApiGateway::RestApi resource by reading and parsing Swagger documents. The result is
        added to the collector.

        Parameters
        ----------
        logical_id : str
            Logical ID of the resource

        api_resource : dict
            Resource definition, including its properties

        collector : ApiCollector
            Instance of the API collector that where we will save the API information
        """
        properties = api_resource.get("Properties", {})
        body = properties.get("Body")
        body_s3_location = properties.get("BodyS3Location")
        binary_media = properties.get("BinaryMediaTypes", [])

        if not body and not body_s3_location:
            # Swagger is not found anywhere.
            LOG.debug("Skipping resource '%s'. Swagger document not found in Body and BodyS3Location",
                      logical_id)
            return
        self.extract_swagger_route(logical_id, body, body_s3_location, binary_media, collector, cwd)

    @staticmethod
    def _extract_cloud_formation_stage(resources, stage_resource, collector):
        """
        Extract the stage from AWS::ApiGateway::Stage resource by reading and adds it to the collector.
        Parameters
       ----------
        resources: dict
            All Resource definition, including its properties

        stage_resource : dict
            Stage Resource definition, including its properties

        collector : ApiCollector
            Instance of the API collector that where we will save the API information
        """
        properties = stage_resource.get("Properties", {})
        stage_name = properties.get("StageName")
        stage_variables = properties.get("Variables")

        # Currently, we aren't resolving any Refs or other intrinsic properties that come with it
        # A separate pr will need to fully resolve intrinsics
        logical_id = properties.get("RestApiId")
        if not logical_id:
            raise InvalidSamTemplateException("The AWS::ApiGateway::Stage must have a RestApiId property")

        rest_api_resource_type = resources.get(logical_id, {}).get("Type")
        if rest_api_resource_type != CfnApiProvider.APIGATEWAY_RESTAPI:
            raise InvalidSamTemplateException(
                "The AWS::ApiGateway::Stage must have a valid RestApiId that points to RestApi resource {}".format(
                    logical_id))

        collector.stage_name = stage_name
        collector.stage_variables = stage_variables


import pandas as pd
import nltk
from nltk.tokenize import RegexpTokenizer

def calculate_words(element, result):
    results = element.split(' ')
    for key in results:
        if key in result:
            result[key] += 1
        else:
            result[key] = 1
    return result

def nltk_calculate_words(series: pd.Series, top = 100):
    string = series.str.lower().str.cat(sep=' ')
    tokenizer = RegexpTokenizer(r'\w+')
    words = tokenizer.tokenize(string)
    word_dist = nltk.FreqDist(words)
    return pd.DataFrame(word_dist.most_common(top), columns=['Word', 'Frequency'])

def nltk_word_frequencies(series: pd.Series, words=None, freq="15T"):
    if words is None:
        words = []

    def calc(df, column_name):
        string = df[column_name].str.lower().str.cat(sep=' ')
        tokenizer = RegexpTokenizer(r'\w+')
        words_distribution = nltk.FreqDist(tokenizer.tokenize(string))
        result = pd.Series()
        for word in words:
            result[word] = words_distribution[word]
        return result

    rv = series.groupby(pd.Grouper(freq=freq)).apply(calc, 'text')
    norm = series['id'].groupby(pd.Grouper(freq=freq)).count()
    for word in words:
        rv[word] = rv[word] / norm
        rv[word] = rv[word] / rv[word].max()
    return rv


import os
import logging

from collections import OrderedDict

from cmd import command
from cmd.cmd import Arg
from command_container import CommandContainer
from storage import NumbersStorage

class NumsBasicCommands(NumbersStorage, CommandContainer):
    @staticmethod
    @command('list',
             'выводит содержимое на экран через заданный разделеитель',
             Arg('separator', str, '\t'))
    def list(separator: str):
        print(f'{separator}'.join(map(str, NumbersStorage.storage)))

    @staticmethod
    @command('clear', descr='удаляет все из набора')
    def clear():
        NumsBasicCommands.storage.clear()

    @staticmethod
    @command('add', 'Добавляет значение в набор',
             Arg('value', int, required=True))
    def add(value: int):
        NumsBasicCommands.storage.append(value)

    @staticmethod
    @command('del', 'Удаляет из набора заданный элемент или элементы в '
                    'указанном диопазоне',
             Arg('start_index', int, required=True), Arg('stop_index', int))
    def del_(start: int, stop: int):
        if stop is None:
            del NumsBasicCommands.storage[start]
        else:
            del NumsBasicCommands.storage[slice(start, stop)]

    @staticmethod
    @command('find', 'выполняет поиск значения в наборе',
             Arg('value', int, required=True))
    def find(value: int):
        try:
            index = NumsBasicCommands.storage.index(value)
            print(f'Значение {value} найдено в позиции {index}')
        except ValueError:
            print(f'Значение {value} не найдено')

    @staticmethod
    @command('set', 'устанавливает значение в указанную позицию',
             Arg('index', int, required=True),
             Arg('value', int, required=True))
    def set(index: int, value: int):
        NumsBasicCommands.storage.insert(index, value)

    @staticmethod
    @command('get', 'читает значение в указанной позиции',
             Arg('index', int, required=True))
    def get(index: int):
        try:
            print(
                f'В позиции {index} находится значение '
                f'{NumsBasicCommands.storage[index]}'
            )
        except IndexError:
            print(f'По индексу {index} значение ещё не добавлено')

    @staticmethod
    @command('unique', 'удаляет дубликаты в наборе')
    def unique():
        NumsBasicCommands.storage = list(
            OrderedDict.fromkeys(NumsBasicCommands.storage)
        )

    @staticmethod
    @command('load', 'загружает набор из текстового файла',
             Arg('path', str, required=True), Arg('separator', str, '\t'))
    def load(path: str, separator: str):
        if os.path.exists(path):
            try:
                with open(path, encoding='utf8') as inp:
                    NumsBasicCommands.storage.extend(
                        map(int, inp.read().split(separator))
                    )
            except ValueError:
                print('Указан неправильный разделитель или файл содержит не '
                      'только цифры')
            except Exception as err:
                logging.getLogger('main').exception(err)
                print('При загрузке файла возникла ошибка')
        else:
            print('Такого файла не существует')

    @staticmethod
    @command('count', 'отображает количество чисел в наборе')
    def count():
        print(len(NumsBasicCommands.storage))


import logging
from typing import Any, Callable

from tabulate import tabulate
from telegram import Bot, ParseMode, ReplyKeyboardMarkup, Update
from telegram.error import NetworkError, TelegramError
from telegram.ext import CommandHandler, Updater

from freqtrade.rpc.__init__ import (rpc_status_table,
                                    rpc_trade_status,
                                    rpc_daily_profit,
                                    rpc_trade_statistics,
                                    rpc_balance,
                                    rpc_start,
                                    rpc_stop,
                                    rpc_forcesell,
                                    rpc_performance,
                                    rpc_count,
                                    )

from freqtrade import __version__

# Remove noisy log messages
logging.getLogger('requests.packages.urllib3').setLevel(logging.INFO)
logging.getLogger('telegram').setLevel(logging.INFO)
logger = logging.getLogger(__name__)

_UPDATER: Updater = None
_CONF = {}

def init(config: dict) -> None:
    """
    Initializes this module with the given config,
    registers all known command handlers
    and starts polling for message updates
    :param config: config to use
    :return: None
    """
    global _UPDATER

    _CONF.update(config)
    if not is_enabled():
        return

    _UPDATER = Updater(token=config['telegram']['token'], workers=0)

    # Register command handler and start telegram message polling
    handles = [
        CommandHandler('status', _status),
        CommandHandler('profit', _profit),
        CommandHandler('balance', _balance),
        CommandHandler('start', _start),
        CommandHandler('stop', _stop),
        CommandHandler('forcesell', _forcesell),
        CommandHandler('performance', _performance),
        CommandHandler('daily', _daily),
        CommandHandler('count', _count),
        CommandHandler('help', _help),
        CommandHandler('version', _version),
    ]
    for handle in handles:
        _UPDATER.dispatcher.add_handler(handle)
    _UPDATER.start_polling(
        clean=True,
        bootstrap_retries=-1,
        timeout=30,
        read_latency=60,
    )
    logger.info(
        'rpc.telegram is listening for following commands: %s',
        [h.command for h in handles]
    )

def cleanup() -> None:
    """
    Stops all running telegram threads.
    :return: None
    """
    if not is_enabled():
        return
    _UPDATER.stop()

def is_enabled() -> bool:
    """
    Returns True if the telegram module is activated, False otherwise
    """
    return bool(_CONF['telegram'].get('enabled', False))

def authorized_only(command_handler: Callable[[Bot, Update], None]) -> Callable[..., Any]:
    """
    Decorator to check if the message comes from the correct chat_id
    :param command_handler: Telegram CommandHandler
    :return: decorated function
    """
    def wrapper(*args, **kwargs):
        update = kwargs.get('update') or args[1]

        # Reject unauthorized messages
        chat_id = int(_CONF['telegram']['chat_id'])
        if int(update.message.chat_id) != chat_id:
            logger.info('Rejected unauthorized message from: %s', update.message.chat_id)
            return wrapper

        logger.info('Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id)
        try:
            return command_handler(*args, **kwargs)
        except BaseException:
            logger.exception('Exception occurred within Telegram module')
    return wrapper

@authorized_only
def _status(bot: Bot, update: Update) -> None:
    """
    Handler for /status.
    Returns the current TradeThread status
    :param bot: telegram bot
    :param update: message update
    :return: None
    """

    # Check if additional parameters are passed
    params = update.message.text.replace('/status', '').split(' ') \
        if update.message.text else []
    if 'table' in params:
        _status_table(bot, update)
        return

    # Fetch open trade
    (error, trades) = rpc_trade_status()
    if error:
        send_msg(trades, bot=bot)
    else:
        for trademsg in trades:
            send_msg(trademsg, bot=bot)

@authorized_only
def _status_table(bot: Bot, update: Update) -> None:
    """
    Handler for /status table.
    Returns the current TradeThread status in table format
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    # Fetch open trade
    (err, df_statuses) = rpc_status_table()
    if err:
        send_msg(df_statuses, bot=bot)
    else:
        message = tabulate(df_statuses, headers='keys', tablefmt='simple')
        message = "
{}
".format(message)

        send_msg(message, parse_mode=ParseMode.HTML)

@authorized_only
def _daily(bot: Bot, update: Update) -> None:
    """
    Handler for /daily <n>
    Returns a daily profit (in BTC) over the last n days.
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    try:
        timescale = int(update.message.text.replace('/daily', '').strip())
    except (TypeError, ValueError):
        timescale = 7
    (error, stats) = rpc_daily_profit(timescale,
                                      _CONF['stake_currency'],
                                      _CONF['fiat_display_currency'])
    if error:
        send_msg(stats, bot=bot)
    else:
        stats = tabulate(stats,
                         headers=[
                             'Day',
                             'Profit {}'.format(_CONF['stake_currency']),
                             'Profit {}'.format(_CONF['fiat_display_currency'])
                         ],
                         tablefmt='simple')
        message = 'Daily Profit over the last {} days:\n
{}
'.format(
                  timescale, stats)
        send_msg(message, bot=bot, parse_mode=ParseMode.HTML)

@authorized_only
def _profit(bot: Bot, update: Update) -> None:
    """
    Handler for /profit.
    Returns a cumulative profit statistics.
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    (error, stats) = rpc_trade_statistics(_CONF['stake_currency'],
                                          _CONF['fiat_display_currency'])
    if error:
        send_msg(stats, bot=bot)
        return

    # Message to display
    markdown_msg = """
*ROI:* Close trades
  ∙ `{profit_closed_coin:.8f} {coin} ({profit_closed_percent:.2f}%)`
  ∙ `{profit_closed_fiat:.3f} {fiat}`
*ROI:* All trades
  ∙ `{profit_all_coin:.8f} {coin} ({profit_all_percent:.2f}%)`
  ∙ `{profit_all_fiat:.3f} {fiat}`

*Total Trade Count:* `{trade_count}`
*First Trade opened:* `{first_trade_date}`
*Latest Trade opened:* `{latest_trade_date}`
*Avg. Duration:* `{avg_duration}`
*Best Performing:* `{best_pair}: {best_rate:.2f}%`
    """.format(
        coin=_CONF['stake_currency'],
        fiat=_CONF['fiat_display_currency'],
        profit_closed_coin=stats['profit_closed_coin'],
        profit_closed_percent=stats['profit_closed_percent'],
        profit_closed_fiat=stats['profit_closed_fiat'],
        profit_all_coin=stats['profit_all_coin'],
        profit_all_percent=stats['profit_all_percent'],
        profit_all_fiat=stats['profit_all_fiat'],
        trade_count=stats['trade_count'],
        first_trade_date=stats['first_trade_date'],
        latest_trade_date=stats['latest_trade_date'],
        avg_duration=stats['avg_duration'],
        best_pair=stats['best_pair'],
        best_rate=stats['best_rate']
    )
    send_msg(markdown_msg, bot=bot)

@authorized_only
def _balance(bot: Bot, update: Update) -> None:
    """
    Handler for /balance
    """
    (error, result) = rpc_balance(_CONF['fiat_display_currency'])
    if error:
        send_msg('`All balances are zero.`')
        return

    (currencys, total, symbol, value) = result
    output = ''
    for currency in currencys:
        output += """*Currency*: {currency}
*Available*: {available}
*Balance*: {balance}
*Pending*: {pending}
*Est. BTC*: {est_btc: .8f}
""".format(**currency)

    output += """*Estimated Value*:
*BTC*: {0: .8f}
*{1}*: {2: .2f}
""".format(total, symbol, value)
    send_msg(output)

@authorized_only
def _start(bot: Bot, update: Update) -> None:
    """
    Handler for /start.
    Starts TradeThread
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    (error, msg) = rpc_start()
    if error:
        send_msg(msg, bot=bot)

@authorized_only
def _stop(bot: Bot, update: Update) -> None:
    """
    Handler for /stop.
    Stops TradeThread
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    (error, msg) = rpc_stop()
    send_msg(msg, bot=bot)

# FIX: no test for this!!!!
@authorized_only
def _forcesell(bot: Bot, update: Update) -> None:
    """
    Handler for /forcesell <id>.
    Sells the given trade at current price
    :param bot: telegram bot
    :param update: message update
    :return: None
    """

    trade_id = update.message.text.replace('/forcesell', '').strip()
    (error, message) = rpc_forcesell(trade_id)
    if error:
        send_msg(message, bot=bot)
        return

@authorized_only
def _performance(bot: Bot, update: Update) -> None:
    """
    Handler for /performance.
    Shows a performance statistic from finished trades
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    (error, trades) = rpc_performance()
    if error:
        send_msg(trades, bot=bot)
        return

    stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format(
        index=i + 1,
        pair=trade['pair'],
        profit=trade['profit'],
        count=trade['count']
    ) for i, trade in enumerate(trades))
    message = 'Performance:\n{}'.format(stats)
    send_msg(message, parse_mode=ParseMode.HTML)

@authorized_only
def _count(bot: Bot, update: Update) -> None:
    """
    Handler for /count.
    Returns the number of trades running
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    (error, trades) = rpc_count()
    if error:
        send_msg(trades, bot=bot)
        return

    message = tabulate({
        'current': [len(trades)],
        'max': [_CONF['max_open_trades']]
    }, headers=['current', 'max'], tablefmt='simple')
    message = "
{}
".format(message)
    logger.debug(message)
    send_msg(message, parse_mode=ParseMode.HTML)

@authorized_only
def _help(bot: Bot, update: Update) -> None:
    """
    Handler for /help.
    Show commands of the bot
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    message = """
*/start:* `Starts the trader`
*/stop:* `Stops the trader`
*/status [table]:* `Lists all open trades`
            *table :* `will display trades in a table`
*/profit:* `Lists cumulative profit from all finished trades`
*/forcesell <trade_id>|all:* `Instantly sells the given trade or all trades, regardless of profit`
*/performance:* `Show performance of each finished trade grouped by pair`
*/daily <n>:* `Shows profit or loss per day, over the last n days`
*/count:* `Show number of trades running compared to allowed number of trades`
*/balance:* `Show account balance per currency`
*/help:* `This help message`
*/version:* `Show version`
    """
    send_msg(message, bot=bot)

@authorized_only
def _version(bot: Bot, update: Update) -> None:
    """
    Handler for /version.
    Show version information
    :param bot: telegram bot
    :param update: message update
    :return: None
    """
    send_msg('*Version:* `{}`'.format(__version__), bot=bot)

def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None:
    """
    Send given markdown message
    :param msg: message
    :param bot: alternative bot
    :param parse_mode: telegram parse mode
    :return: None
    """
    if not is_enabled():
        return

    bot = bot or _UPDATER.bot

    keyboard = [['/daily', '/profit', '/balance'],
                ['/status', '/status table', '/performance'],
                ['/count', '/start', '/stop', '/help']]

    reply_markup = ReplyKeyboardMarkup(keyboard)

    try:
        try:
            bot.send_message(
                _CONF['telegram']['chat_id'], msg,
                parse_mode=parse_mode, reply_markup=reply_markup
            )
        except NetworkError as network_err:
            # Sometimes the telegram server resets the current connection,
            # if this is the case we send the message again.
            logger.warning(
                'Telegram NetworkError: %s! Trying one more time.',
                network_err.message
            )
            bot.send_message(
                _CONF['telegram']['chat_id'], msg,
                parse_mode=parse_mode, reply_markup=reply_markup
            )
    except TelegramError as telegram_err:
        logger.warning('TelegramError: %s! Giving up on that message.', telegram_err.message)


import face_recognition
from tkinter import *
from PIL import Image, ImageTk

# config variables
IMAGE_LOCATION_KNOWN = "images/almaz_office.jpg"
# IMAGE_LOCATION_UNKNOWN = "images/almaz_bike.jpg"
IMAGE_LOCATION_UNKNOWN = "images/almazer.jpg"
IMAGE_SCALE = 200

known_image = face_recognition.load_image_file(IMAGE_LOCATION_KNOWN)
unknown_image = face_recognition.load_image_file(IMAGE_LOCATION_UNKNOWN)

known_encoding = face_recognition.face_encodings(known_image)
unknown_encoding = face_recognition.face_encodings(unknown_image)

face_locations_known = face_recognition.face_locations(
    face_recognition.load_image_file(IMAGE_LOCATION_KNOWN)
)
face_locations_unknown = face_recognition.face_locations(
    face_recognition.load_image_file(IMAGE_LOCATION_UNKNOWN)
)

results = face_recognition.compare_faces([known_encoding[0]], unknown_encoding[0], 0.5)

class Window(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.master = master
        self.pack(fill=BOTH, expand=1)

        # Tkinter loads same image on its own
        image_known = Image.open(IMAGE_LOCATION_KNOWN)
        image_unknown = Image.open(IMAGE_LOCATION_UNKNOWN)

        top1, right1, bottom1, left1 = face_locations_known[0]
        top2, right2, bottom2, left2 = face_locations_unknown[0]

        cropped_image_known = image_known.crop(
            [left1 - 20, top1 - 20, right1 + 20, bottom1 + 20]
        )
        cropped_image_unknown = image_unknown.crop(
            [left2 - 20, top2 - 20, right2 + 20, bottom2 + 20]
        )

        cropped_image_known = cropped_image_known.resize(
            (IMAGE_SCALE, IMAGE_SCALE), Image.ANTIALIAS
        )
        cropped_image_unknown = cropped_image_unknown.resize(
            (IMAGE_SCALE, IMAGE_SCALE), Image.ANTIALIAS
        )

        render_known = ImageTk.PhotoImage(cropped_image_known)
        render_unknown = ImageTk.PhotoImage(cropped_image_unknown)

        label_known = Label(self, image=render_known)
        label_known.image = render_known

        label_unknown = Label(self, image=render_unknown)
        label_unknown.image = render_unknown

        label_known.place(x=50, y=50)
        label_unknown.place(x=300, y=50)

        text = Label(
            width=30,
            height=2,
            text="SAME PERSON" if results[0] else "NOT SAME PERSON",
            font="Arial 25",
            padx=0,
            pady=0,
        )
        text.pack()

# Rendering window with size based on grid
root = Tk()
app = Window(root)
root.wm_title("Check Person")
root.geometry("550x350")
root.mainloop()


#!/usr/bin/env python3
#-*- coding: utf-8 -*-

"""
This module provides class and methods for dealing with RDKit Conformer.
"""

from typing import Sequence, Union

import numpy as np
from rdkit.Chem import rdMolTransforms as rdMT
from rdkit.Chem.rdchem import Conformer

from rdmc.utils import find_internal_torsions, set_rdconf_coordinates

class RDKitConf(object):
    """
    A wrapper for rdchem.Conformer.

    The method nomenclature follows the Camel style to be consistent with RDKit.
    """

    def __init__(self, conf):
        self._conf = conf
        self._owning_mol = conf.GetOwningMol()
        for attr in dir(conf):
            if not attr.startswith('_') and not hasattr(self, attr):
                setattr(self, attr, getattr(self._conf, attr,))

    @classmethod
    def FromConformer(cls,
                      conf: Conformer,
                      ) -> 'RDKitConf':
        """
        Convert a RDKit Chem.rdchem.Conformer to a RDKitConf. This allows a more
        capable and flexible Conformer class.

        Args:
            conf (Chem.rdchem.Conformer): A RDKit Conformer instance to be converted.

        Returns:
            RDKitConf: The conformer corresponding to the RDKit Conformer in RDKitConf
        """
        return cls(conf)

    @classmethod
    def FromMol(cls,
                mol: Union['RWMol', 'Mol'],
                id: int = 0,
                ) -> 'RDkitConf':
        """
        Get a RDKitConf instance from a Chem.rdchem.Mol/RWMol instance.

        Args:
            mol (Union[RWMol, Mol]): a Molecule in RDKit Default format.
            id (int): The id of the conformer to be extracted from the molecule.

        Returns:
            RDKitConf: A Conformer in RDKitConf of the given molecule
        """
        return cls(mol.GetConformer(id))

    @classmethod
    def FromRDKitMol(cls,
                     rdkitmol: 'RDKitMol',
                     id: int = 0,
                     ) -> 'RDkitConf':
        """
        Get a RDKitConf instance from a RDKitMol instance. The owning molecule
        of the generated conformer is RDKitMol instead of Chem.rdchem.Mol.

        Args:
            rdkitmol (RDKitMol): a Molecule in RDKitMol.
            id (int): The id of the conformer to be extracted from the molecule.

        Returns:
            RDKitConf: A Conformer in RDKitConf of the given molecule
        """
        return rdkitmol.GetConformer(id)

    def GetBondLength(self,
                      atomIds: Sequence[int],
                     ) -> float:
        """
        Get the bond length between atoms in Angstrom.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.

        Returns:
            float: Bond length in Angstrom.
        """
        assert len(atomIds) == 2, ValueError(f'Invalid atomIds. It should be a sequence with a length of 2. Got {atomIds}')
        return rdMT.GetBondLength(self._conf, *atomIds)

    def GetAngleDeg(self,
                    atomIds: Sequence[int],
                    ) -> float:
        """
        Get the angle between atoms in degrees.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.

        Returns:
            float: Angle value in degrees.
        """
        assert len(atomIds) == 3, ValueError(f'Invalid atomIds. It should be a sequence with a length of 3. Got {atomIds}.')
        return rdMT.GetAngleDeg(self._conf, *atomIds)

    def GetAngleRad(self,
                    atomIds: Sequence[int],
                    ) -> float:
        """
        Get the angle between atoms in rads.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.

        Returns:
            float: Angle value in rads.
        """
        assert len(atomIds) == 3, ValueError(f'Invalid atomIds. It should be a sequence with a length of 3. Got {atomIds}')
        return rdMT.GetAngleRad(self._conf, *atomIds)

    def GetAllTorsionsDeg(self) -> list:
        """
        Get the dihedral angles of all torsional modes (rotors) of the Conformer. The sequence of the
        values are corresponding to the torsions of the molecule (``GetTorsionalModes``).

        Returns:
            list: A list of dihedral angles of all torsional modes.
        """
        return [self.GetTorsionDeg(tor) for tor in self.GetTorsionalModes()]

    def GetOwningMol(self):
        """
        Get the owning molecule of the conformer.

        Returns:
            Union[Mol, RWMol, RDKitMol]: The owning molecule
        """
        return self._owning_mol

    def GetTorsionDeg(self,
                      torsion: list,
                      ) -> float:
        """
        Get the dihedral angle of the torsion in degrees. The torsion can be defined
        by any atoms in the molecule (not necessarily bonded atoms.)

        Args:
            torsion (list): A list of four atom indexes.

        Returns:
            float: The dihedral angle of the torsion.
        """
        return rdMT.GetDihedralDeg(self._conf, *torsion)

    def GetTorsionalModes(self,
                          indexed1: bool = False):
        """
        Get all of the torsional modes (rotors) of the Conformer. This information
        is obtained from its owning molecule.

        Args:
            indexed1: The atom index in RDKit starts from 0. If you want to have
                       indexed 1 atom indexes, please set this argument to ``True``.

        Returns:
            Optinal[list]: A list of four-atom-indice to indicating the torsional modes.
        """
        try:
            return self._torsions if not not indexed1 \
                else [[ind + 1 for ind in tor] for tor in self._torions]
        except AttributeError:
            self._torsions = find_internal_torsions(self._owning_mol)
            return self._torsions

    def HasOwningMol(self):
        """
        Whether the conformer has a owning molecule.

        Returns:
            bool: ``True`` if the conformer has a owning molecule.
        """
        if self._owning_mol:
            return True
        return False

    def SetOwningMol(self,
                     owningMol: Union['RDKitMol',
                                      'Mol',
                                      'RWMol']):
        """
        Set the owning molecule of the conformer. It can be either RDKitMol
        or Chem.rdchem.Mol.

        Args:
            owningMol: Union[RDKitMol, Chem.rdchem.Mol] The owning molecule of the conformer.

        Raises:
            ValueError: Not a valid ``owning_mol`` input, when giving something else.
        """
        if not hasattr(owningMol, 'GetConformer'):
            raise ValueError('Provided an invalid molecule object.')
        self._owning_mol = owningMol

    def SetPositions(self,
                     coords: Union[tuple, list]):
        """
        Set the Positions of atoms of the conformer.

        Args:
            coords: a list of tuple of atom coordinates.
        """
        set_rdconf_coordinates(self._conf, coords)

    def SetBondLength(self,
                      atomIds: Sequence[int],
                      value: Union[int, float],
                     ) -> float:
        """
        Set the bond length between atoms in Angstrom.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.
            value (int or float, optional): Bond length in Angstrom.
        """
        assert len(atomIds) == 2, ValueError(f'Invalid atomIds. It should be a sequence with a length of 2. Got {atomIds}')
        return rdMT.SetBondLength(self._conf, *atomIds, value)

    def SetAngleDeg(self,
                    atomIds: Sequence[int],
                    value: Union[int, float],
                    ) -> float:
        """
        Set the angle between atoms in degrees.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.
            value (int or float, optional): Bond angle in degrees.
        """
        assert len(atomIds) == 3, ValueError(f'Invalid atomIds. It should be a sequence with a length of 3. Got {atomIds}.')
        return rdMT.SetAngleDeg(self._conf, *atomIds, value)

    def SetAngleRad(self,
                    atomIds: Sequence[int],
                    value: Union[int, float],
                    ) -> float:
        """
        Set the angle between atoms in rads.

        Args:
            atomIds (Sequence): A 3-element sequence object containing atom indexes.
            value (int or float, optional): Bond angle in rads.
        """
        assert len(atomIds) == 3, ValueError(f'Invalid atomIds. It should be a sequence with a length of 3. Got {atomIds}')
        return rdMT.SetAngleRad(self._conf, *atomIds, value)

    def SetTorsionDeg(self,
                      torsion: list,
                      degree: Union[float, int]):
        """
        Set the dihedral angle of the torsion in degrees. The torsion can only be defined
        by a chain of bonded atoms.

        Args:
            torsion (list): A list of four atom indexes.
            degree (float, int): The dihedral angle of the torsion.
        """
        rdMT.SetDihedralDeg(self._conf, *torsion, degree)

    def SetAllTorsionsDeg(self, angles: list):
        """
        Set the dihedral angles of all torsional modes (rotors) of the Conformer. The sequence of the
        values are corresponding to the torsions of the molecule (``GetTorsionalModes``).

        Args:
            angles (list): A list of dihedral angles of all torsional modes.
        """
        if len(angles) != len(self.GetTorsionalModes()):
            raise ValueError(
                'The length of angles is not equal to the length of torsional modes')
        for angle, tor in zip(angles, self.GetTorsionalModes()):
            self.SetTorsionDeg(tor, angle)

    def SetTorsionalModes(self,
                          torsions: Union[list, tuple]):
        """
        Set the torsional modes (rotors) of the Conformer. This is useful when the
        default torsion is not correct.

        Args:
            torsions (Union[list, tuple]): A list of four-atom-lists indicating the torsional modes.

        Raises:
            ValueError: The torsional mode used is not valid.
        """
        if isinstance(torsions, (list, tuple)):
            self._torsions = torsions
        else:
            raise ValueError('Invalid torsional mode input.')

    def ToConformer(self) -> 'Conformer':
        """
        Get its backend RDKit Conformer object.

        Returns:
            Conformer: The backend conformer
        """
        return self._conf


from django.http import HttpResponse
from django.shortcuts import render
from tutorials.books.models import Book, BookLog
from tutorials.books.tasks import create_log

def index(request):
    books = Book.objects.all()
    return render(request, 'books/index.html', { 'books': books })

def book_log(request):
    logs = BookLog.objects.all()
    return render(request, 'books/book_log.html', { 'logs': logs })

def mark_read(request, book_id=None):
    if book_id:
        book = Book.objects.get(id=book_id)
        create_log.delay(request.user, book)
        return HttpResponse('Sent off for Processing')
    else:
        return HttpResponse('No ID match')


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 16 01:41:20 2022

@author: Greta
"""

# Importing needed libraries
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from googletrans import Translator
translator = Translator()
# NLTK tools for text processing
import re, nltk
from nltk import word_tokenize
from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('punkt')
from nltk import pos_tag
nltk.download('wordnet')
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
from nltk.stem import PorterStemmer
ps = PorterStemmer()
from textblob import TextBlob
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

# Charger les données
df_pt = pd.read_csv(r'C:\Users\Greta\Desktop\Projet Brazil\olist_order_reviews_dataset.csv',sep=',',encoding='utf-8')

#Séparer reviews et titre, supprimer titres
review_data_title = df_pt['review_comment_title']
review_data = df_pt.drop(['review_comment_title'],axis=1)

# Drop valeurs nulles
reviews = review_data.dropna(subset=['review_comment_message'])

# Reset index
review_data = review_data.reset_index(drop=True)

# Traitement des reviews sous forme de liste
criticas = list(reviews['review_comment_message'].values)

# Regex pour enlever les retours à la ligne
criticas_temp = []
criticas_pos_quebra = []
for c in criticas:
    c = re.sub(r'\n', ' ', c)
    criticas_temp.append(c)
for c in criticas_temp:
    c = re.sub(r'\r', ' ', c)
    criticas_pos_quebra.append(c)

# Regex pour enlever les liens url
criticas_pos_hyperlink = []
for c in criticas_pos_quebra:
    urls = re.findall('(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?', c)
    if len(urls) == 0:
        pass
    else:
        for url in urls:
            for link in url:
                c = c.replace(link, '')
        c = c.replace(':', '')
        c = c.replace('/', '')
    criticas_pos_hyperlink.append(c)

#Regex pour enlever les nombres
criticas_pos_num = []
for c in criticas_pos_hyperlink:
    c = re.sub(r'\d+(?:\.\d*(?:[eE]\d+))?', 'numero', c)
    criticas_pos_num.append(c)

# Regex pour tout en alphanumerique
criticas_sem_alphanumerico = []
for c in criticas_pos_num:
    c = re.sub(r'R\$', ' ', c)
    c = re.sub(r'\W', ' ', c)
    criticas_sem_alphanumerico.append(c)

# Regex pour enlever les espaces en trop
criticas_sem_espacos_adicionais = []
for c in criticas_sem_alphanumerico:
    c = re.sub(r'\s+', ' ', c)
    criticas_sem_espacos_adicionais.append(c)

# Sauvegarde de la df traitée après regex
processed_reviews = reviews.copy()
processed_reviews['review_after_regex'] = criticas_sem_espacos_adicionais
processed_reviews = processed_reviews.iloc[:, np.r_[4, -1, 2]]
processed_reviews.head()

# Transforme les reviews en enlevant les stopwords, tout mettre en lower case
comments = []
stopwords = nltk.corpus.stopwords.words('portuguese')

#Retrait des stopword et tout en lower case
for words in processed_reviews['review_after_regex']:
    only_letters = re.sub("[^a-zA-Z]", " ",words)
    tokens = nltk.word_tokenize(only_letters) #tokenize the sentences
    lower_case = [l.lower() for l in tokens] #convert all letters to lower case
    filtered_result = list(filter(lambda l: l not in stopwords, lower_case)) #Remove stopwords from the comments
    comments.append(' '.join(filtered_result))

# Frequence des trigrams
co = CountVectorizer(ngram_range=(3,3))
counts = co.fit_transform(comments)
important_trigrams = pd.DataFrame(counts.sum(axis=0),columns=co.get_feature_names()).T.sort_values(0,ascending=False).head(200)

# Rename colonnes
important_trigrams=important_trigrams.reset_index()
important_trigrams.rename(columns={'index':'trigrams',0:'frequency'},inplace=True)

#Prochaine etape Traduction des trigrams en anglais
important_trigrams['trigrams_traduit'] =important_trigrams['trigrams'].map(lambda x: translator.translate(x, src="pt", dest="en").text)

#  Definition des noms, verbes, adverbes
pos_dict = {'J':wordnet.ADJ, 'V':wordnet.VERB, 'N':wordnet.NOUN, 'R':wordnet.ADV}
def token_stop_pos(text):
    tags = pos_tag(word_tokenize(text))
    newlist = []
    for word, tag in tags:
        if word.lower() not in stopwords:
            newlist.append(tuple([word, pos_dict.get(tag[0])]))
    return newlist

important_trigrams['POS tagged'] = important_trigrams['trigrams_traduit'].apply(token_stop_pos)
important_trigrams.head()

# Definition de la racine du mot
def lemmatize(pos_data):
    lemma_rew = " "
    for word, pos in pos_data:
        if not pos:
            lemma = word
            lemma_rew = lemma_rew + " " + lemma
        else:
                lemma = wordnet_lemmatizer.lemmatize(word, pos=pos)
                lemma_rew = lemma_rew + " " + lemma
    return lemma_rew

important_trigrams['Lemma'] = important_trigrams['POS tagged'].apply(lemmatize)
important_trigrams.head()

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

# function to calculate vader sentiment
def vadersentimentanalysis(review):
    vs = analyzer.polarity_scores(review)
    return vs['compound']

important_trigrams['Vader Sentiment'] = important_trigrams['Lemma'].apply(vadersentimentanalysis)

#Fonction pour analyser les reviews : Negatif, neutre et positif
def vader_analysis(compound):
    if compound >= 0.5:
        return 'Positive'
    elif compound <= -0.3 :
        return 'Negative'
    else:
        return 'Neutral'
important_trigrams['Vader Analysis'] = important_trigrams['Vader Sentiment'].apply(vader_analysis)
important_trigrams.head()

file_name= "trigrams_200"
important_trigrams.to_csv(file_name,index=False, encoding="utf-8", sep=";")

vader_counts = important_trigrams['Vader Analysis'].value_counts()
print(vader_counts)

import matplotlib.pyplot as plt
plt.pie(vader_counts.values, labels = vader_counts.index, autopct='%1.1f%%', shadow=False)


#!/usr/bin/python3

import argparse
from typing import List, Tuple

from corrade import containers
from magnum import *
from magnum import math, trade

parser = argparse.ArgumentParser()
parser.add_argument('input')
parser.add_argument('output')
parser.add_argument('--offset', type=int, default=0)
parser.add_argument('--count', type=int, default=4)
args = parser.parse_args()

importer_manager = trade.ImporterManager()
importer_manager.metadata('DdsImporter').configuration['assumeYUpZBackward'] = True
importer_manager.metadata('KtxImporter').configuration['assumeOrientation'] = 'ruo'
importer = importer_manager.load_and_instantiate('AnyImageImporter')
importer.open_file(args.input)

image = importer.image2d(0)

assert image.is_compressed, "expected a compressed image"

print(f"format: {image.compressed_format}")

block_size = image.compressed_format.block_size
assert image.size % block_size.xy == Vector2i(0), "expected whole blocks"

block_data_size = image.compressed_format.block_data_size
# TODO use image.blocks once it exists
blocks = containers.StridedArrayView1D(image.data).expanded(0, (
    image.size.y//block_size.y,
    image.size.x//block_size.x,
    block_data_size
))

if image.compressed_format.name.startswith('BC'):
    decoder_name = 'BcDecImageConverter'
elif image.compressed_format.name.startswith('ETC') or \
     image.compressed_format.name.startswith('EAC'):
    decoder_name = 'EtcDecImageConverter'
else: assert False, f"unsupported format {image.compressed_format}"

converter_manager = trade.ImageConverterManager()
decoder = converter_manager.load_and_instantiate(decoder_name)
decoded = decoder.convert(image)
assert decoded.size == image.size
# TODO some better way to get this directly from pixels? i.e., want to access
#   the bytes and not the high-level type
# assert decoded.pixels.is_contiguous
decoded_pixel_size = decoded.format.size
decoded_pixel_data = containers.StridedArrayView1D(decoded.data).expanded(0, (
    image.size.y,
    image.size.x,
    decoded_pixel_size
))

block_count = decoded.size/block_size.xy

pixels = decoded.pixels

if decoded.format.channel_count == 1:
    default_min = math.inf
    default_max = -math.inf
    sum_ = lambda i: i
elif decoded.format.channel_count == 2:
    default_min = Vector2(math.inf)
    default_max = Vector2(-math.inf)
    sum_ = lambda i: i.sum()
elif decoded.format.channel_count == 3:
    default_min = Vector3(math.inf)
    default_max = Vector3(-math.inf)
    sum_ = lambda i: i.sum()
elif decoded.format.channel_count == 4:
    default_min = Vector4(math.inf)
    default_max = Vector4(-math.inf)
    sum_ = lambda i: i.sum()

block_ranges: List[Tuple[int, int, float]] = []

# This is extremely slow. Pybind11 ugh what are you doing?!
for y in range(block_count.y):
    oy = y*block_size.y
    for x in range(block_count.x):
        ox = x*block_size.x
        min_ = default_min
        max_ = default_max
        for by in range(block_size.y):
            for bx in range(block_size.x):
                min_ = math.min(pixels[oy + by, ox + bx], min_)
                max_ = math.max(pixels[oy + by, ox + bx], max_)

        block_ranges += [(x, y, sum_(max_ - min_))]

# Get most interesting blocks
most_interesting_blocks = list(reversed(sorted(block_ranges, key=lambda block: block[2])))[args.offset:(args.offset + args.count)]

print("    char blocks[]{")
for x, y, range_ in most_interesting_blocks:
    print("        /* [{}, {}], {:1.3f} */\n        {},".format(
        x, y, range_,
        ', '.join(["'\\x{:02x}'".format(ord(blocks[y, x, i])) for i in range(block_data_size)])
    ))
print("    };")

# Combine the decoded pixels from the most interesting blocks
data = bytearray(b'\0'*(args.count*block_size.x*block_size.y*decoded_pixel_size))
offset = 0
for x, y, range_ in most_interesting_blocks:
    oy = y*block_size.y
    ox = x*block_size.x
    for by in range(block_size.y):
        for bx in range(block_size.x):
            for i in range(decoded_pixel_size):
                data[offset] = ord(decoded_pixel_data[oy + by, ox + bx, i])
                offset += 1
assert offset == len(data)

# Write them to a file
converter = converter_manager.load_and_instantiate('AnyImageConverter')
converter.convert_to_file(ImageView2D(
    decoded.format,
    (block_size.x, block_size.y*args.count),
    data), args.output)


from Bio import SeqIO
import pandas as pd
import os

def list_files(path):
    path_to_files = []
    for root, dir, files in os.walk(path):
        for f in files:
            if 'japan' in f:
                path_to_files.append(os.path.join(root, f))
    return path_to_files

def get_basename(path_to_file):
    filepath = os.path.splitext(path_to_file)
    base = filepath[0].split('/')[-1]
    return base

df = pd.read_csv('/data/san/data1/users/linda/crAss_DB/circular_genomes/japan4D/genbank_to_contig_map.csv',sep='\t',header=None)
df.columns = ['ncbi_accn','contig']
print(df.head())

japan_opt_translations = list_files('/data/san/data1/users/linda/crAss_DB/optimal_translations_a_i_jap_mon')
for fiLe in japan_opt_translations:
    fiLeName = get_basename(fiLe)
    corrected = f'/data/san/data1/users/linda/crAss_DB/circular_genomes/japan4D/corrected_transl_headers/{fiLeName}.faa'
    with open(fiLe) as original, open(corrected, 'w') as corrected:
        records = SeqIO.parse(fiLe, 'fasta')
        for record in records:
            if record.id:
                try:
                    print('\n')
                    print(record.id)
                    protein_no = record.id.rsplit('_',1)[1]
                    print(protein_no)
                    seq_to_search=(record.id.replace('_k141',''))
                    search_string=seq_to_search.replace('japan4D_','')
                    search_string=search_string.rsplit('_',1)[0]
                    print(search_string)
                    accn = (df.loc[df.contig.str.contains(search_string)]['ncbi_accn'].values[0])
                    print(accn)
                    record.id=f'{accn}_japan4D_{search_string}_{protein_no}'
                    record.description=record.id
                    SeqIO.write(record, corrected, 'fasta')
                except Exception as e:
                    print('printing ERROR')
                    print(f"{e}")
                    record.id=f'{accn}_japan4D_{search_string}'
                    print(record.id)


from datetime import datetime
from dateutil.parser import parse

def compareTimeDifference(first_timeRaw, timePeriodSeconds = 0, timePeriodDays = 0):
	#first_timeRaw - first time
	#timePeriod - time period witch need to be passed in order to return True
	#Function is true if time has past margin
	first_time = first_timeRaw
	second_time = datetime.now()
	first_time = datetime.strptime(str(first_time), '%Y-%m-%d %H:%M:%S.%f')#convert string to datetime object
	difference = second_time - first_time
	#print(difference.seconds, '    ', difference.days)
	if timePeriodDays != 0:
		if difference.seconds > timePeriodSeconds and difference.days >= timePeriodDays:
			return True
	else:
		if difference.seconds > timePeriodSeconds:
			return True
	return False

def unique(it):#filter unique data - entry will be registered only once all other same values will be discarded
#filter out only unique data, example: [1,1,4,3,6,3] --> [1,4,3,6]
	s = []
	for el in it:
		if el not in s:
			s.append(el)
	return s

def filterIfContain(dataList,keyWord):
	#print('--------keyWord--------->', keyWord)
	filteredTableNames = [table for table in dataList if not keyWord in table]
	print(filteredTableNames)
	return filteredTableNames

def is_date(string, fuzzy=False):
	#this is not reliable for date detection
    """
    Return whether the string can be interpreted as a date.

    :param string: str, string to check for date
    :param fuzzy: bool, ignore unknown tokens in string if True
    """
    try:
        parse(string, fuzzy=fuzzy)
        return True

    except ValueError:
        return False

def validateDate(date_text):
	#detect date based on provided format
	try:
		#datetime.strptime(date_text, '%Y-%m-%d')
		#28/6/2021
		#datetime.strptime(date_text, '%d/%m/%Y')
		datetime.strptime(date_text, '%m/%d/%Y')
		return True
	except ValueError:
		return False

def changeDateFormat(itemList):
	dateItemsIndexes = []
	for item in range(len(itemList[0])):
		if validateDate(itemList[0][item]) == True: dateItemsIndexes.append(item)
	newList = itemList

	for currlist in range(len(itemList)):
		for index in dateItemsIndexes:
			try:
				#print("------------------itemList[currlist][index]----------------->",itemList[currlist][index])
				item = datetime.strptime(itemList[currlist][index], '%m/%d/%Y')
				#item = datetime.strptime(itemList[currlist][index], '%d/%m/%Y')
				newList[currlist][index] = item.strftime('%Y-%m-%d')
			except:
				print('-------------------------->>>>-------------does not match format--<<<<------------------------------')
				print('item at index: ', itemList[currlist][index])
				print('index: ', index)
				print('liste: ', itemList[currlist])
	return newList

def isInt(s):
	try:
		int(s)
		return True
	except ValueError:
		return False

def isFloat(s):
	try:
		float(s)
		return True
	except ValueError:
		return False


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Tension webapp.

Usage:
  tension_webapp [--port=<port>] [--localhost]
  tension_webapp -h | --help
  tension_webapp --version

Options:
  --localhost   Only allow connections from this computer.
  -h --help     Show this screen.
  --version     Show version.

"""
from docopt import docopt
from tension.webapp import tension_app

def main():
    args = docopt(__doc__, version="Tension webapp 1.0")
    port = int(args.get("--port", False) or 5000)
    host = "localhost" if args.get("--localhost", False) else "0.0.0.0"

    print("tension webapp started on {}:{}".format(host, port))

    tension_app.run(host=host, port=port, debug=True)

if __name__ == "__main__":
    main()


import sys
from io import StringIO

test1 = '''2
1, 2, 3
4, 5, 6'''
test2 = '''3
10, 2, 21, 4
5, 20, 41, 9
6, 2, 4, 99'''

sys.stdin = StringIO(test2)

rows = int(input())
matrix = []
for _ in range(rows):
    line = [int(x) for x in input().split(', ')]
    matrix.extend(line)
print(matrix)

Python Advanced/Multidimensional Lists - Lab/flattening_matrix.py


import logging
import re
from django.db import models

logger = logging.getLogger('app')

class Videos(models.Model):
    CATEGORY = [
        ('tutorials', 'Lunar Magic Tutorials'),
        ('speedruns', 'Robfather Speedruns'),
    ]
    HELP_TEXT = {
        'category': 'Which section the video will appear in.',
        'title': 'Title of the video.',
        'description': 'Description text about the video.',
        'video_id': 'Video URL or ID from the YouTube share button.',
    }

    category = models.CharField(max_length=255, choices=CATEGORY, default='tutorials', help_text=HELP_TEXT['category'])
    title = models.CharField(max_length=255, help_text=HELP_TEXT['title'])
    description = models.TextField(help_text=HELP_TEXT['description'])
    video_id = models.CharField(max_length=255, help_text=HELP_TEXT['video_id'])
    updated_at = models.DateTimeField(auto_now=True)
    created_at = models.DateTimeField(auto_now_add=True)

    def save(self, *args, **kwargs):
        m = re.search('[a-zA-Z0-9\-\_]{8,20}', self.video_id)
        logger.info(m)
        self.video_id = m.group(0)
        logger.info(self.video_id)
        super(Videos, self).save()

    class Meta:
        verbose_name = 'Videos'
        verbose_name_plural = 'Videos'


import RPi.GPIO as GPIO
import smbus
import signal
import sys
import time
import threading
from enum import Enum

class TachometerState(Enum):
    SPLASH_SCREEN = 0
    MENU_OPTIONS = 1
    MEASURE_FREQUENCY = 2
    INPUT_FREQUENCY = 3

def event_callback(channel):
    global lt, count, time_difference
    count = (count + 1) % 50
    t = time.time_ns()

    if count == 0:
        flash()
    time_difference = t - lt
    lt = t

def flash():
    lock.acquire()
    GPIO.output(STROBE_TRIGGER, GPIO.LOW)
    GPIO.output(STROBE_TRIGGER, GPIO.HIGH)
    lock.release()

def read():
    print(bus.read_byte_data(0x30, 0))

def locking_read_i2c_THREAD(address, byte):
    global I2C_READ_BYTE_GLOBAL

    lock.acquire()
    I2C_READ_BYTE_GLOBAL = bus.read_byte_data(address, byte)
    lock.release()

def read_i2c(address, byte):
    t = threading.Thread(target=locking_read_i2c_THREAD, args=(address, byte), daemon=True)
    t.start()
    t.join()

    return I2C_READ_BYTE_GLOBAL

def check_input():
    # key = read_i2c(keyboard_address, 0)
    t = threading.Thread(target=locking_read_i2c_THREAD, args=(keyboard_address, 0), daemon=True)
    t.start()
    t.join()

    key = I2C_READ_BYTE_GLOBAL

    c = None
    if key != 0:
        c = chr(key)

    return c

def debounced_check():
    global last_char

    k = check_input()
    if k == last_char:
        last_char = k
        return None

    last_char = k
    return k

def wait_for_input():
    global last_char
    k = check_input()

    while k == None or k == last_char:
        last_char = k
        k = check_input()

    last_char = k

    return k

def signal_handler(signal, frame):
    GPIO.cleanup()
    sys.exit(0)

I2C_READ_BYTE_GLOBAL = 0
last_char = None

state = TachometerState.SPLASH_SCREEN
PHOTOGATE_GPIO = 22
STROBE_TRIGGER = 6
keyboard_address = 0x30
bus = smbus.SMBus(1)

lock = threading.Lock() # The talking stick.

lt = time.time_ns()
count = 0
time_difference = sys.maxsize

GPIO.setmode(GPIO.BCM)
GPIO.setup(PHOTOGATE_GPIO, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
GPIO.setup(STROBE_TRIGGER, GPIO.OUT)

GPIO.add_event_detect(PHOTOGATE_GPIO, GPIO.FALLING, callback=event_callback)

signal.signal(signal.SIGINT, signal_handler)

try:
    while True:

        if state == TachometerState.SPLASH_SCREEN:
            print("Loading")
            time.sleep(1)
            state = TachometerState.MENU_OPTIONS
        elif state == TachometerState.MENU_OPTIONS:
            print("Menu options")

            k = debounced_check()
            print(I2C_READ_BYTE_GLOBAL)
            if k == '1':
                state = TachometerState.MEASURE_FREQUENCY
            elif k == '2':
                state = TachometerState.INPUT_FREQUENCY

        elif state == TachometerState.MEASURE_FREQUENCY:
            print("Measure frequency")
            time.sleep(1)
        elif state == TachometerState.INPUT_FREQUENCY:
            print("Input frequency")
            time.sleep(1)

        time.sleep(0.2)
except Exception as e:
    print(str(e))
    GPIO.cleanup()
    sys.exit(0)