import sqlalchemy as sa
from flask_apispec import doc
import re

from webservices import args
from webservices import docs
from webservices import utils
from webservices import filters
from webservices import schemas
from webservices.common import models
from webservices.common import views
from webservices.common.views import ItemizedResource
from webservices import exceptions

"""
two years restriction removed from schedule_a. For details, refer:
https://github.com/fecgov/openFEC/issues/3595
"""

# Used for '/schedules/schedule_a/'
# /schedules/schedule_a/<string:sub_id>/
# under tag:'receipts'
# Ex:
# http://127.0.0.1:5000/v1/schedules/schedule_a/?two_year_transaction_period=2020
# http://127.0.0.1:5000/v1/schedules/schedule_a/2071120191659332613/?two_year_transaction_period=2020
@doc(
    tags=['receipts'], description=docs.SCHEDULE_A,
)
class ScheduleAView(ItemizedResource):

    model = models.ScheduleA
    schema = schemas.ScheduleASchema
    page_schema = schemas.ScheduleAPageSchema

    @property
    def year_column(self):
        return self.model.two_year_transaction_period

    @property
    def index_column(self):
        return self.model.sub_id

    @property
    def amount_column(self):
        return self.model.contribution_receipt_amount

    filter_multi_fields = [
        ('image_number', models.ScheduleA.image_number),
        ('committee_id', models.ScheduleA.committee_id),
        ('contributor_id', models.ScheduleA.contributor_id),
        ('contributor_city', models.ScheduleA.contributor_city),
        ('contributor_state', models.ScheduleA.contributor_state),
        ('recipient_committee_type', models.ScheduleA.recipient_committee_type),
        ('recipient_committee_org_type', models.ScheduleA.recipient_committee_org_type),
        (
            'recipient_committee_designation',
            models.ScheduleA.recipient_committee_designation,
        ),
        ('two_year_transaction_period', models.ScheduleA.two_year_transaction_period),
    ]
    filter_match_fields = [
        ('is_individual', models.ScheduleA.is_individual),
    ]
    filter_range_fields = [
        (('min_date', 'max_date'), models.ScheduleA.contribution_receipt_date),
        (('min_amount', 'max_amount'), models.ScheduleA.contribution_receipt_amount),
        (('min_image_number', 'max_image_number'), models.ScheduleA.image_number),
        (('min_load_date', 'max_load_date'), models.ScheduleA.load_date),
    ]
    filter_fulltext_fields = [
        ('contributor_name', models.ScheduleA.contributor_name_text),
        ('contributor_employer', models.ScheduleA.contributor_employer_text),
        ('contributor_occupation', models.ScheduleA.contributor_occupation_text),
    ]
    filter_multi_start_with_fields = [
        ('contributor_zip', models.ScheduleA.contributor_zip),
    ]
    query_options = [
        sa.orm.joinedload(models.ScheduleA.committee),
        sa.orm.joinedload(models.ScheduleA.contributor),
    ]
    sort_options = [
        'contribution_receipt_date',
        'contribution_receipt_amount',
    ]
    filters_with_max_count = [
        'committee_id',
        'contributor_name',
        'contributor_zip',
        'contributor_city',
        'contributor_employer',
        'contributor_occupation',
    ]
    secondary_index_options = [
        'committee_id',
        'contributor_id',
        'contributor_name',
        'contributor_city',
        'contributor_zip',
        'contributor_employer',
        'contributor_occupation',
        'image_number',
    ]
    use_pk_for_count = True

    @property
    def args(self):
        return utils.extend(
            args.itemized,
            args.schedule_a,
            args.make_seek_args(),
            args.make_sort_args(
                default='-contribution_receipt_date',
                validator=args.OptionValidator(self.sort_options),
                show_nulls_last_arg=False,
            ),
        )

    def build_query(self, **kwargs):
        query = super().build_query(**kwargs)
        query = filters.filter_contributor_type(query, self.model.entity_type, kwargs)
        zip_list = []
        if kwargs.get('contributor_zip'):
            for value in kwargs['contributor_zip']:
                if re.search('[^a-zA-Z0-9-\s]', value):  # noqa
                    raise exceptions.ApiError(
                        'Invalid zip code. It can not have special character',
                        status_code=422,
                    )
                else:
                    zip_list.append(value[:5])
            contributor_zip_list = {'contributor_zip': zip_list}
            query = filters.filter_multi_start_with(
                query, contributor_zip_list, self.filter_multi_start_with_fields
            )
        if kwargs.get('sub_id'):
            query = query.filter_by(sub_id=int(kwargs.get('sub_id')))
        if kwargs.get('line_number'):
            # line_number is a composite value of 'filing_form-line_number'
            if len(kwargs.get('line_number').split('-')) == 2:
                form, line_no = kwargs.get('line_number').split('-')
                query = query.filter_by(filing_form=form.upper())
                query = query.filter_by(line_number=line_no)
            else:
                raise exceptions.ApiError(
                    exceptions.LINE_NUMBER_ERROR, status_code=400,
                )
        return query

# Used for '/schedules/schedule_a/efile/'
# under tag:'receipts'
# Ex: http://127.0.0.1:5000/v1/schedules/schedule_a/efile/
@doc(
    tags=['receipts'], description=docs.EFILING_TAG,
)
class ScheduleAEfileView(views.ApiResource):
    model = models.ScheduleAEfile
    schema = schemas.ItemizedScheduleAfilingsSchema
    page_schema = schemas.ScheduleAEfilePageSchema

    filter_multi_fields = [
        ('image_number', models.ScheduleAEfile.image_number),
        ('committee_id', models.ScheduleAEfile.committee_id),
        ('contributor_city', models.ScheduleAEfile.contributor_city),
        ('contributor_state', models.ScheduleAEfile.contributor_state),
    ]

    filter_range_fields = [
        (('min_date', 'max_date'), models.ScheduleAEfile.contribution_receipt_date),
        (
            ('min_amount', 'max_amount'),
            models.ScheduleAEfile.contribution_receipt_amount,
        ),
        (('min_image_number', 'max_image_number'), models.ScheduleAEfile.image_number),
    ]

    filter_fulltext_fields = [
        ('contributor_name', models.ScheduleAEfile.contributor_name_text),
        ('contributor_employer', models.ScheduleAEfile.contributor_employer_text),
        ('contributor_occupation', models.ScheduleAEfile.contributor_occupation_text),
    ]

    @property
    def args(self):
        return utils.extend(
            args.paging,
            args.schedule_a_e_file,
            args.itemized,
            args.make_sort_args(
                default='-contribution_receipt_date',
                validator=args.OptionValidator(
                    [
                        'contribution_receipt_date',
                        'contribution_receipt_amount',
                    ]
                ),
            ),
        )


import base64
import datetime
import uuid
from typing import Optional, Tuple
import jwt
import requests
from cryptography import x509
from cryptography.hazmat.primitives import hashes

# list of constants
BASE_URL_3M_GPCS: str = 'http://gpcs.3m.com' # should use http
OATH_URL_3M_GPCS: str = 'https://gpcs.3m.com/GpcsSts/oauth/token' # should use secured http
BASE_PAYLOAD_FIELD_3M_GPCS: Tuple[str, str, str, str] = ('jti', 'iat', 'exp', 'aud')
BASE_HEADER_FIELD_3M_GPCS: Tuple[str, str] = ('x5t', 'kid')
OATH_FIELD_3M_GPCS: Tuple[str, str] = ('grant_type', 'assertion')
OATH_GRANT_TYPE_3M_GPCS: str = 'jwt-bearer'
HTTP_RES_SUCC_CODE: int = 200
OATH_EXCEPTION_MSG: str = 'STS authentication error'
OATH_RES_FIELD_3M_GPCS: str = 'access_token'
ENCRYPT_ALGO_3M_GPCS: str = 'RS256'
MAX_AUTH_DUR_MIN: int = 10
AUTH_PUB_SUF: str = '_cert.pem'
AUTH_PRI_SUF: str = '_key.pem'
AUTH_CODESET: str = 'utf-8'

def generate_gpcs_auth_token(
    public_key: bytes,
    private_key: bytes,
    duration_minutes: Optional[int] = MAX_AUTH_DUR_MIN
) -> str:
    """This method returns an authorization token from 3M GPCS, given the public and private key pair.

    Input Arguments: public_key, byte stream from a certificate file that corresponds to a key file, with default encryption algorithm RS256
                     private_key, byte stream from a key file that corresponds to a certificate file, with default encryption algorithm RS256
                     [optional] duration_minutes, default to MAX_AUTH_DUR_MIN minutes, integer specifying how long the authorization key is valid
    Returns: authorization token, string stream representing an authorization token
    """
    cert = x509.load_pem_x509_certificate(public_key)
    fingerprint = cert.fingerprint(hashes.SHA1())
    kid = fingerprint.hex()
    x5t = base64.urlsafe_b64encode(fingerprint).decode(AUTH_CODESET)

    now_time = datetime.datetime.utcnow()
    payload: dict = dict(zip(
        BASE_PAYLOAD_FIELD_3M_GPCS,
        (str(uuid.uuid4()),
            now_time,
            now_time + datetime.timedelta(minutes=duration_minutes),
            BASE_URL_3M_GPCS
        )
    ))
    headers: dict = dict(zip(
        BASE_HEADER_FIELD_3M_GPCS,
        (x5t, kid)
    ))
    auth_token: str = jwt.encode(
        payload=payload,
        key=private_key,
        algorithm=ENCRYPT_ALGO_3M_GPCS,
        headers=headers
    )

    return auth_token

def generate_gpcs_auth_token_from_cert_files(
    cert_basedir: str,
    cert_basename: str,
    duration_minutes: Optional[int] = MAX_AUTH_DUR_MIN
) -> str:
    """This method returns an authorization token from 3M GPCS to use locally, given path and file name prefix of the public and private key pair.

    Input Arguments: cert_basedir, string specifying path of the files of certificates
                     cert_basename, string specifying file name prefix of certificates
                     [optional] duration_minutes, default to MAX_AUTH_DUR_MIN minutes, integer specifying how long the authorization key is valid
    Returns: authorization token, string stream representing an authorization token
    """
    cert_prefix: str = cert_basedir + '/' + cert_basename
    private_key: bytes = bytes()
    public_key: bytes = bytes()
    with open(cert_prefix + AUTH_PRI_SUF, mode='rb') as private_key_file:
        private_key = private_key_file.read()
    with open(cert_prefix + AUTH_PUB_SUF, mode='rb') as public_key_file:
        public_key = public_key_file.read()

    auth_token: str = generate_gpcs_auth_token(
        public_key,
        private_key,
        duration_minutes
    )
    return auth_token

def get_gpcs_oath_access_token(auth_token: str) -> str:
    """This method returns an OATH token from 3M GPCS, given the authorization token.

    Input Arguments: auth_token, string stream from an authorization token
    Returns: OATH token, string stream representing an OATH token
    """
    response = requests.post(
        OATH_URL_3M_GPCS,
        json = dict(zip(
            OATH_FIELD_3M_GPCS,
            (OATH_GRANT_TYPE_3M_GPCS, auth_token)
        ))
    )
    if response.status_code != HTTP_RES_SUCC_CODE:
        raise Exception(OATH_EXCEPTION_MSG, response.status_code)
    access_token = response.json()[OATH_RES_FIELD_3M_GPCS]
    return access_token


import pandas as pd
from datetime import datetime

now = datetime.now() # current date and time
today = now.strftime('%Y-%m-%d')

# show complete records by changing rules
# pd.set_option('display.max_rows', None)
# pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)

# Scrape tables from the webpage
dfs = pd.read_html('https://projects.fivethirtyeight.com/soccer-predictions/premier-league/')

# Load the dataset into the dataframe
df = dfs[-1]

list_538 = []

team_dict = {'Arsenal': 'Arsenal',
             'Aston Villa': 'Aston Villa',
             'Brighton': 'Brighton',
             'Burnley': 'Burnley',
             'Chelsea': 'Chelsea',
             'Crystal Palace': 'Crystal Palace',
             'Everton': 'Everton',
             'Fulham': 'Fulham',
             'Leeds United': 'Leeds',
             'Leicester': 'Leicester',
             'Liverpool': 'Liverpool',
             'Newcastle': 'Newcastle',
             'Man. City': 'Man City',
             'Man. United': 'Man Utd',
             'Sheffield Utd': 'Sheffield Utd',
             'Southampton': 'Southampton',
             'Tottenham': 'Spurs',
             'West Brom': 'West Brom',
             'West Ham': 'West Ham',
             'Wolves': 'Wolves'}

for index, row in df.iterrows():
    list_538.append((row[('Unnamed: 0_level_0', 'team')], row[('avg. simulated season', 'proj. pts.pts.')]))
    pass

new_list = []

print('538 predictions on ', today, '\n')

for team, pts in list_538:
    team = team[0:-6]
    print(team_dict[team], pts)
    new_list.append((team_dict[team], pts))

new_list.sort()

""" Upto here. Need to start thinking about putting the 538 data and my scraped data into an append csv."""

df_output = pd.DataFrame(new_list, columns=['team', today])

df_output = df_output.drop(['team'], axis=1)

df_output = df_output.transpose()

df_output.to_csv('538.csv', mode='a', header=False)

print('\n 538 Scrape has been written to csv')


#Importing
import matplotlib.pyplot as plt

life_exp = [45,46,45,23,45,67,86,54,75,86,54,87,46,23,45,87,99,65,76,57,56,33]
# Build histogram with 5 bins
plt.hist(life_exp,bins=5)

# Show and clean up plot
plt.show()
plt.clf()

# Build histogram with 20 bins
plt.hist(life_exp,bins=20)

# Show and clean up again
plt.show()
plt.clf()

1 Graphical Analysis/1 Making_Multiple_Graphs_Together.py


import asyncio
import logging
from collections import defaultdict
from concurrent.futures import ProcessPoolExecutor

import io
from aiohttp import web

from ept.boundingboxes import BoundingBox2D
from ept.eptresource import EPTResource
from ept.queryparams import QueryParams, sync_read_laz_files, sync_filter_las_points

logger = logging.getLogger(__name__)

RESOURCES = defaultdict(EPTResource)
POOL = ProcessPoolExecutor(8)

async def process(lazes_bytes, query):
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(POOL, _process, lazes_bytes, query)

def _process(lazes_bytes, query):
    las = sync_read_laz_files(lazes_bytes)
    sync_filter_las_points(las, query)
    return _las_to_bytes(las)

def _las_to_bytes(las):
    with io.BytesIO() as buffer:
        las.write(buffer, do_compress=True)
        return buffer.getvalue()

async def las_to_bytes(las):
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(None, _las_to_bytes, las)

async def get_info(request):
    name = request.match_info["resource_name"]
    address = "https://na-c.entwine.io/{}".format(name)
    ept = EPTResource(address)
    return web.json_response(await ept.info)

async def read(request):
    name = request.match_info["resource_name"]
    address = "https://na-c.entwine.io/{}".format(name)
    xmin, ymin = request.match_info['xmin'], request.match_info['ymin']
    xmax, ymax = request.match_info['xmax'], request.match_info['ymax']

    text = " ".join(a for a in (name, xmin, ymin, xmax, ymax))
    logger.info("The Query: {}".format(text))
    ept = RESOURCES[address]

    query_bounds = BoundingBox2D(int(xmin), int(ymin), int(xmax), int(ymax))
    params = QueryParams(query_bounds)

    logger.info("Downloading")
    tiles_bytes = await ept.query_tile_bytes(params)
    logger.info("Processing")
    las_bytes = await process(tiles_bytes, params)

    logger.info("Sending {} bytes".format(len(las_bytes)))
    return web.Response(body=las_bytes)

async def prepare():
    ept = EPTResource("https://na-c.entwine.io/dk")
    await ept.hierarchy
    RESOURCES["https://na-c.entwine.io/dk"] = ept

if __name__ == '__main__':
    import logging

    logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    loop = asyncio.get_event_loop()
    future = asyncio.ensure_future(prepare())
    loop.run_until_complete(future)

    app = web.Application()
    app.add_routes(
        [
            web.get("/info/{resource_name}", get_info),
            web.get("/read/{resource_name}/[{xmin},{ymin},{xmax},{ymax}]", read),
        ]
    )

    web.run_app(app)


# coding:utf-8
from crossknight.sqink.domain import isUuid
from crossknight.sqink.domain import NoteStatus
from crossknight.sqink.markdown import renderHtml
from crossknight.sqink.plist import readNote
from crossknight.sqink.plist import writeNote
from crossknight.sqink.provider import LocalNoteProvider
from datetime import datetime
from datetime import timezone
from os import listdir
from os import remove
from os import utime
from os.path import exists
from os.path import getmtime
from os.path import isfile
from os.path import normpath
import re

def getModifiedOn(path):
    return datetime.utcfromtimestamp(int(getmtime(path)))

def getTimestamp(lastModified):
    return lastModified.replace(tzinfo=timezone.utc).timestamp()

def inflateNote(notePath, photoPath=None, withoutText=False):
    with open(notePath, "rb") as file:
        note = readNote(file, getModifiedOn(notePath), withoutText)
    if photoPath and isfile(photoPath):
        with open(photoPath, "rb") as file:
            note.photo = file.read()
    return renderHtml(note)

def flattenNote(note, notePath, photoPath):
    with open(notePath, "wb") as file:
        writeNote(note, file)
    timestamp = getTimestamp(note.lastModified)
    utime(notePath, (timestamp, timestamp))
    savePhoto(note, photoPath)

def savePhoto(note, path):
    if note.photo is not None:
        with open(path, "wb") as file:
            file.write(note.photo)
        timestamp = getTimestamp(note.lastModified)
        utime(path, (timestamp, timestamp))
    elif isfile(path):
        remove(path)

class FilesystemNoteProvider(LocalNoteProvider):

    def __init__(self, narratePath):
        self.__notesPath = normpath(narratePath + "/entries")
        self.__removedNotesPath = normpath(narratePath + "/entries/deleted")
        self.__photosPath = normpath(narratePath + "photos")

    def list(self):
        names = listdir(self.__notesPath)
        return [inflateNote(self.__notePath(name), withoutText=True) for name in names if isUuid(name)]

    def search(self, alphanum):
        regexs = list(map(lambda t: re.compile(re.escape(t)), filter(lambda t: len(t) > 0, alphanum.split(" "))))
        uuids = []
        for name in listdir(self.__notesPath):
            if isUuid(name):
                note = inflateNote(self.__notePath(name))
                if all(map(lambda r: r.search(note.title) or r.search(note.tags) or r.search(note.text), regexs)):
                    uuids.append(note.uuid)
        return uuids

    def sync(self):
        notes = {}
        for name in listdir(self.__notesPath):
            if isUuid(name):
                hasPhoto = isfile(self.__photoPath(name))
                notes[name] = NoteStatus(name, getModifiedOn(self.__notePath(name)), hasPhoto=hasPhoto)
        for name in listdir(self.__removedNotesPath):
            if isUuid(name):
                removedNote = NoteStatus(name, getModifiedOn(self.__removedNotePath(name)), True)
                # Clean inconsistent files
                if name in notes:
                    if removedNote.lastModified >= notes[name].lastModified:
                        remove(self.__notePath(name))
                        if notes[name].hasPhoto:
                            remove(self.__photoPath(name))
                    else:
                        remove(self.__removedNotePath(name))
                        continue
                notes[name] = removedNote
        return notes

    def get(self, uuid):
        return inflateNote(self.__notePath(uuid), self.__photoPath(uuid))

    def add(self, note):
        uuid = note.uuid
        notePath = self.__notePath(uuid)
        if exists(notePath):
            raise RuntimeError("Note[uuid=%s] already exists" % uuid)
        flattenNote(note, notePath, self.__photoPath(uuid))
        removedNotePath = self.__removedNotePath(uuid)
        if exists(removedNotePath):
            remove(removedNotePath)

    def update(self, note):
        uuid = note.uuid
        notePath = self.__notePath(uuid)
        if not exists(notePath):
            raise RuntimeError("Note[uuid=%s] does not exist" % uuid)
        flattenNote(note, notePath, self.__photoPath(uuid))

    def remove(self, note):
        uuid = note.uuid
        notePath = self.__notePath(uuid)
        if exists(notePath):
            remove(notePath)
        photoPath = self.__photoPath(uuid)
        if exists(photoPath):
            remove(photoPath)

        removedNotePath = self.__removedNotePath(uuid)
        open(removedNotePath, "wb").close()
        timestamp = getTimestamp(note.lastModified)
        utime(removedNotePath, (timestamp, timestamp))

    def __notePath(self, uuid):
        return normpath(self.__notesPath + "/" + uuid)

    def __removedNotePath(self, uuid):
        return normpath(self.__removedNotesPath + "/" + uuid)

    def __photoPath(self, uuid):
        return normpath(self.__photosPath + "/" + uuid + ".jpg")


from kivy.app import App
from kivy.uix.gridlayout import GridLayout
from kivy.uix.label import Label
from kivy.uix.textinput import TextInput
from kivy.uix.button import Button
from kivy.uix.image import Image
class Formularz(GridLayout):
    def __init__(self, **kwargs):
        super(Formularz, self).__init__(**kwargs)

        self.cols = 2
        self.label = self.add_widget(Label(text="To jest etykieta ", font_size = "30sp"))
        self.name = TextInput(multiline=False, font_size="30sp")
        self.name.background_color = [1, 1, 0, 1]
        self.add_widget(self.name)
        self.img =Image(source="rys.png")

        self.add_widget(self.img)
        self.submit = Button(text = "Wyślij", font_size = "32sp")
        self.submit.background_color = [1, 0, 0, 1]
        self.add_widget(self.submit)
class MobileApp(App):
   def build(self):
      return Formularz()

if __name__ == '__main__':
   MobileApp().run()


from __future__ import print_function
import datetime
from flask import Flask, request, jsonify
import pickle
from flask_cors import CORS
import os.path
import json
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/calendar']

app = Flask(__name__)
CORS(app)
cors = CORS(app, resources={r'/*': {'origins': '*'}})
@app.route('/event', methods=["GET", "POST"])
def main():
    """Shows basic usage of the Google Calendar API.
    Prints the start and name of the next 10 events on the user's calendar.
    """
    creds = None
    content = json.loads(request.data)
    if os.path.exists('token.pickle'):
        with open('token.pickle', 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'credentials.json', SCOPES)
            creds = flow.run_local_server()
        # Save the credentials for the next run
        with open('token.pickle', 'wb') as token:
            pickle.dump(creds, token)

    service = build('calendar', 'v3', credentials=creds)
    if(service):

            event = {
            'summary': content['summary'],
            'location': content['location'],
            'start': {
                'dateTime': content['stime']+":00",
                'timeZone': 'Asia/Kolkata',
                },
            'end': {
                'dateTime': content['etime'] + ":00",
                'timeZone': 'Asia/Kolkata',
                },
            'attendees': [
                {'email':content['firstemail']},
                {'email': content['secondemail']},
                {'email': content['thirdemail']}
                  ]
        }
            event = service.events().insert(calendarId='primary', body=event).execute()
            print ('Event created: %s' % (event.get('htmlLink')))
            return ("Event created")

    else:
        return("Event not created pls try after some time")

if __name__ == '__main__':
    app.run(threaded=True, debug=True)


import json
import os
import re
import time
from datetime import datetime

import requests
from fake_useragent import UserAgent
from lxml import etree

import Backend.storage.mongodb as mg
import config
import public_fun
from definitions import ROOT_DIR
from utils import bwlist
from utils.errors import updateError

def get_pdf_urls(search_keyword, page):
    """
    获取搜索结果中的文章url
    :param search_keyword: 搜素关键词
    :param page: 页数
    :return: [list of 文章 urls],
    """
    index_url2 = 'https://www.huxiu.com/article/{}.html'
    url = 'https://search-api.huxiu.com/api/article?platform=www&s={}&sort=&page={}&pagesize=20'.format(search_keyword,
                                                                                page)
    print(str(UserAgent().random))
    header = {'user-agent': str(UserAgent().random)}
    res = requests.post(url=url, headers=header)

    try:
        data = json.loads(res.text)
        result = []
        pages = data['data']['total_pages']
        for d in data['data']['datalist']:
            timeArray = time.localtime(d['dateline'])
            date = time.strftime("%Y-%m-%d", timeArray)
            if public_fun.calc_date(date) and not d['is_video_article']:  # 时间过滤且不说视文章
                result.append(index_url2.format(d['aid']))
        print('--------Found %d articles from huxiu--------' % len(result))
        print(result, pages)
        return result, pages
    except Exception as e:
        updateError("ResponseError: Empty response when downloading from huxiu. Status code: " + str(res.status_code))
        return [], 0

def download_articles(url, search_keyword, min_word_count, get_pdf, summary):
    """
    1. 网站来源 “source”（白鲸出海“bjch”/未来智库“wlzk”……）
    2. 文章ID “doc_id”（在所爬取的网页的id）
    3. 发表日期 “date”（格式：20200129）
    4. 下载url “download_url”
    5. 作者/机构 “org_name”（如 xx券商/xx证券)
    6. 页数/字数 “page_num”/ “word_count”，如果有pdf文件就用页数，没有就用字数
    7. 资料种类 “doc_type”（研报： “EXTERNAL_REPORT”/咨询： “NEWS”）
    8. 文章标题 “title” ("中芯国际：首次公开发行股票并在科创板上市招股说明书")
    :param summary: the summary file
    :param get_pdf: 是否要创建html文件
    :param min_word_count: 最少字数限制
    :param search_keyword: 搜索关键词
    :param url:url2 文章内容链接
    :return:
    """
    json_result = {'source': 'huxiu',
                   'doc_id': '',
                   'date': '',
                   'download_url': '',
                   'org_name': '',
                   'doc_type': 'NEWS',
                   'title': '',
                   'oss_path': '',
                   'has_pdf': 'html',
                   'content': '',
                   'filtered': 0,
                   'search_keyword': search_keyword}
    path = os.path.join(ROOT_DIR, 'cache', search_keyword, 'html', 'huxiu')
    header = {'user-agent': str(UserAgent().random)}
    res = requests.get(url=url, headers=header)
    html = etree.HTML(res.text)
    content_text = public_fun.filter_space_json(html.xpath('//*[@class="article__bottom-content__right fl"]//text()'))

    if len(content_text) > min_word_count:
        doc_id = re.findall('\d+', string=url)[0]  # 文章ID

        # Blacklist
        blacklist = bwlist.BWList(search_keyword, 'black')
        blacklist_exist = blacklist.bwlist_exist()

        if blacklist_exist and blacklist.in_bwlist(doc_id, 'huxiu'):
            return

        # whitelist by database
        id_match_res = mg.show_datas('articles', query={'doc_id': str(doc_id),
                                                        'search_keyword': search_keyword})  # new whitelist --vincent
        if id_match_res:
            print('article #' + str(doc_id) + ' is already in database. Skipped.')
            return

        try:
            description = html.xpath('//*[@id="article_read"]')[0]  # 文章内容
            html_list = [description]
            html_result = public_fun.filter_space_html(html_list)

            json_result['doc_id'] = doc_id
            json_result['date'] = html.xpath('//*[@class="article__time"]/text()')[0].split()[0]
            json_result['download_url'] = url
            json_result['org_name'] = html.xpath('//*[@class="author-info__username"]/text()')[0]
            json_result['title'] = html.xpath('//*[@class="article__title"]/text()')[0]
            json_result['content'] = content_text

            # Downloading
            print('Downloading article %s' % doc_id)
            public_fun.write_down_json(path=path, filename=doc_id + '.json', text=json_result)

            if get_pdf:
                public_fun.write_down_html(path=path, filename=doc_id + '.html', text=html_result)

            summary['data'].append(json_result.copy())
        except Exception as e:
            updateError(
                'Download Error: Error occurred when downloading article from huxiu. \n' + str(e.__traceback__.tb_lineno) + ": " + str(e))
    else:
        print('文章字数不足' + str(min_word_count), url)

def run(search_keyword, min_word_count, get_pdf):
    """
    铅笔道爬虫入口函数
    :param min_word_count: 最少字数限制
    :param search_keyword:搜索关键词
    :return: None
    """
    print('--------Begin searching articles from huxiu--------')
    os.chdir(ROOT_DIR)

    page = 1
    url2_list = []
    summary = {}

    summary.update({'source': '36kr'})
    summary.update({'has_pdf': 'html'})
    summary.update({'search_keyword': search_keyword})
    summary.update({'search_time': str(datetime.now().date())})
    summary.update({'data': []})

    while page:
        one_page_url, pages = get_pdf_urls(search_keyword=search_keyword, page=page)
        print(str(page) + '/' + str(pages))
        if pages == 0:
            continue
        url2_list += one_page_url
        if len(url2_list) < 1000 and page < int(pages):
            page += 1
        else:
            print(len(url2_list))
            page = 0

    url2_list = url2_list[:3] if config.DEBUG else url2_list
    for url2 in url2_list:
        download_articles(url=url2, search_keyword=search_keyword, min_word_count=min_word_count, get_pdf=get_pdf,
                          summary=summary)

    with open(os.path.join('cache', search_keyword, 'html', 'huxiu', 'summary.json'), 'w', encoding='utf-8') as f:
        json.dump(summary, f, ensure_ascii=False, indent=4)

    print('--------Finished downloading articles from huxiu--------')

if __name__ == '__main__':
    s_w = '恒大'
    run(search_keyword=s_w, min_word_count=0, get_pdf=True)


"""
Lookout

This is the main control script for the Lookout garden monitor, responsible for executing the overall program flow (as
defined in the UML activity diagram), coordinating and timing the vision and gpio, and auxiliary functions such as
shutting down the pi when the power button is pressed.

Author: Finin Quincey
"""

import os                      # Operating system commands
import sys                     # Python system commands
import time                    # Timing functions
from datetime import datetime  # Real-world date and time
import logging as log          # Log messages and log file output
from enum import Enum          # Enumeration types
import cv2                     # OpenCV for image processing
import threading               # Concurrency
import signal                  # System exit signals

### Constants ###

FORCE_SHUTDOWN_TIME = 5 # If the power button is held for longer than this many seconds, it forces a shutdown

# N.B. Because we're using a callback approach here, we can't just have a shutdown method, we also need a way of
# preventing any callbacks from executing. The only way of doing this is to have the callback exit early when the state
# changes to shutting down. (Also, a state pattern would probably be overkill here, this is Python not Java!)
class State(Enum):
    """Enum representing the different states the program can be in"""
    INACTIVE      = "Inactive"      # Waiting to be triggered via the PIR sensor
    ACTIVE        = "Active"        # Currently recording footage
    SHUTTING_DOWN = "Shutting down" # Waiting to shut down the pi (saving and exiting)

### Initial Setup ###

# Global variables
state = State.INACTIVE
idle_timer = 0
buzzer_timer = 0
btn_hold_timer = 0

# Initialise other modules
import settings
import gpio_manager as gpio
gpio.set_power_led_state(True)
gpio.enable_pir_sensor(True)
import camera_manager as camera
import object_detector

if settings.DEV_MODE:
    camera.display_current() # Show blank screen
    cv2.waitKey(2) # For some reason it needs 2ms

### Functions ###

def on_pir_activated():
    """
    Called from the GPIO manager when the PIR sensor activates.
    """
    log.info("PIR sensor activated")
    global state
    state = State.ACTIVE

def on_power_btn_pressed(released):
    """
    Called from the GPIO manager when the power button is pressed or released.
    """
    global btn_hold_timer

    if released:

        if btn_hold_timer != 0 and time.perf_counter() - btn_hold_timer > FORCE_SHUTDOWN_TIME:

            log.warning("Forcing shutdown! Capture data may be lost!")

            if settings.DEV_MODE:
                os.kill(os.getpid(), signal.SIGINT) # sys.exit() won't work here
            else:
                os.system("sudo shutdown -h now")

        else:

            log.info("Shutting down...")

            global state
            state = State.SHUTTING_DOWN

            gpio.set_power_led_state(True)

    else:
        btn_hold_timer = time.perf_counter() # Start hold timer

def capture_video():
    """
    Captures a video from the camera until no objects are present any more (or until the buffer is full), performing object
    detection and controlling the buzzer accordingly.

    Returns:
    - True if the device should go back to sleep, False if it should shut down
    """

    global idle_timer
    global buzzer_timer

    i = 0
    measured_framerate = 0

    while True: # Capture until one of the return statements below is reached

        t = time.perf_counter()

        camera.capture_frame()

        boxes = object_detector.boxes
        labels = object_detector.labels
        scores = object_detector.scores

        if len(labels) > 0: # If we found something

            idle_timer = time.perf_counter() # Reset idle timer

            # Check objects against blacklist/whitelist
            if settings.ENABLE_BUZZER and any(item in settings.OBJECT_BLACKLIST for item in labels) and all(item1 not in settings.OBJECT_WHITELIST for item1 in labels):
                # Activate the buzzer and start the timer
                if buzzer_timer == 0: gpio.set_buzzer_frequency(settings.BUZZER_FREQUENCY)
                buzzer_timer = time.perf_counter()

        if buzzer_timer != 0 and time.perf_counter() - buzzer_timer >= settings.BUZZ_TIME: # If buzzer time has elapsed
            # Deactivate the buzzer and reset the timer
            gpio.set_buzzer_frequency(0)
            buzzer_timer = 0

        camera.annotate_current(boxes, labels, scores, measured_framerate, buzzer_timer != 0)

        if settings.DEV_MODE:
            camera.display_current()

        if not camera.store_current(): # Stop recording if the buffer is full
            log.info("Buffer full, stopping current recording")
            return True

        if time.perf_counter() - idle_timer >= settings.IDLE_TIME: # If idle time has elapsed
            return True

        if state == State.SHUTTING_DOWN:
            return False

        # LED flashing
        if i == 20:
            gpio.set_power_led_state(True)
            i = 0
        else:
            if i == 0:
                gpio.set_power_led_state(False)
            i += 1

        measured_framerate = min(settings.FRAMERATE, round(1/(time.perf_counter() - t), 1))

        # Try to keep a stable framerate by waiting for the rest of the time, if any
        cv2.waitKey(max(1, int(1000 * (1.0/settings.FRAMERATE - (time.perf_counter() - t)))))

### Finish Setup ###

log.info("Setting up callbacks")

gpio.init_pir_callback(on_pir_activated)
gpio.init_btn_callback(on_power_btn_pressed)

gpio.set_power_led_state(False)

log.info("Setup done")

### Main Logic Loop ###

while state != State.SHUTTING_DOWN: # Keep doing this until the program shuts down

    if state == State.ACTIVE:

        idle_timer = time.perf_counter() # Start idle timer

        gpio.set_ir_led_state(True)
        camera.open(settings.SAVE_DIRECTORY) # Do this as close to first capture as possible to minimise delay

        if capture_video():
            # capture_video() returned True so go back to sleep
            log.debug("Going back to sleep, goodnight")
            state = state.INACTIVE
            camera.close()
            gpio.set_buzzer_frequency(0)
            gpio.set_ir_led_state(False)

    time.sleep(0.2) # Save processing power when not doing anything

### Shutdown ###

# Shut down modules and wait for other processes to finish
gpio.prepare_shutdown() # Cancel callbacks first
object_detector.shutdown() # Start shutdown of object detector
camera.shutdown() # Wait for camera to shut down
object_detector.thread.join() # Wait until object detector has finished (it probably has already, but just in case)

# Finally, turn off the power LED and close the GPIO connection once everything is done
gpio.finish_shutdown()

log.info("*** Exiting program ***")

if settings.DEV_MODE:
    sys.exit()
else:
    os.system("poweroff")


import json
import time
from random import choice

import requests

from django.conf import settings

from fitbot.handlers import progress, meals, recipes, default, profanity
from fitbot.models import Person
from fitbot.utils import MessengerEvent

class States:
    WAITING_FOR_MEAL_PHOTO = 'WAITING_FOR_MEAL_PHOTO'
    WAITING_FOR_MEAL_COMMENTS = 'WAITING_FOR_MEAL_COMMENTS'

    WAITING_FOR_PROGRESS_PHOTO = 'WAITING_FOR_PROGRESS_PHOTO'
    WAITING_FOR_PROGRESS_WEIGHT = 'WAITING_FOR_PROGRESS_WEIGHT'
    WAITING_FOR_PROGRESS_BMI = 'WAITING_FOR_PROGRESS_BMI'

    CHECKING_FOOD = "CHECKING_FOOD"
    CHECKING_PROGRESS = "CHECKING_PROGRESS"

class PostBacks:
    SKIP_COMMENTS = "SKIP_COMMENTS"
    SKIP_PHOTO = "SKIP_PHOTO"
    SKIP_WEIGHT = "SKIP_WEIGHT"
    SKIP_BMI = "SKIP_BMI"

    LOG_BREAKFAST = "LOG_BREAKFAST"
    LOG_LUNCH = "LOG_LUNCH"
    LOG_DINNER = "LOG_DINNER"
    LOG_SNACK = "LOG_SNACK"

    LOG_PROGRESS = "LOG_PROGRESS"

    CHECK_FOOD = "CHECK_FOOD"
    CHECK_PROGRESS = "CHECK_PROGRESS"

    PREV_DAY = "PREV_DAY"
    NEXT_DAY = "NEXT_DAY"

    GET_STARTED = "GET_STARTED"
    CONTINUE_INTRO_PART2 = "CONTINUE_INTRO1"

class Intents:
    LOG_MEAL = "LOG_MEAL"
    VIEW_MEAL_DIARY = "VIEW_MEAL_DIARY"
    SUGGEST_RECIPE = "SUGGEST_RECIPE"
    GREETING = "GREETING"
    GOOD_BYE = "GOOD_BYE"
    PROFANITY = "PROFANITY"

class Meals:
    BREAKFAST = 'breakfast'
    LUNCH = 'lunch'
    DINNER = 'dinner'
    SNACK = 'snack'

PB_TO_MEAL_TYPES = {
    PostBacks.LOG_BREAKFAST: Meals.BREAKFAST,
    PostBacks.LOG_LUNCH: Meals.LUNCH,
    PostBacks.LOG_DINNER: Meals.DINNER,
    PostBacks.LOG_SNACK: Meals.SNACK
}

class Chatbot(object):

    _handlers = []

    class SenderActions:
        MARK_SEEN = 'mark_seen'
        TYPING_ON = 'typing_on'
        TYPING_OFF = 'typing_off'

    @property
    def handlers(self):
        return self._handlers

    @staticmethod
    def get_server_uri():
        return "https://graph.facebook.com/v2.6/me/messages?access_token=%s" % settings.FB_ACCESS_TOKEN

    @classmethod
    def send_action(cls, person, action):
        data = {
            'recipient': {'id': person.fb_id},
            'sender_action': action
        }

        response = requests.post(
            cls.get_server_uri(),
            data=json.dumps(data),
            headers={'content-type': 'application/json'}
        )

    @classmethod
    def send_mark_seen(cls, person):
        cls.send_action(person, Chatbot.SenderActions.MARK_SEEN)

    @classmethod
    def send_typing(cls, person, seconds=1.2):
        cls.send_action(person, Chatbot.SenderActions.TYPING_ON)
        time.sleep(seconds)
        cls.send_action(person, Chatbot.SenderActions.TYPING_OFF)

    @classmethod
    def send_text(cls, person, text, quick_replies=None):
        """

        :param text: plain text string
        :param quick_replies: [(display1, postback1), ...]
        :return:
        """

        # If we have more than one choice, choose a random one
        if isinstance(text, (list, tuple)):
            text = choice(text)

        data = {
            'messaging_type': 'RESPONSE',
            'recipient': {'id': person.fb_id},
            'message': {'text': text}
        }

        if quick_replies is not None:
            data['message']['quick_replies'] = [
                {
                    'content_type': "text",
                    'title': qr[0],
                    'payload': qr[1],

                } for qr in quick_replies
            ]

        # print("Outgoing data")
        # print("======================")
        # pprint(data)
        # print("======================")
        response = requests.post(
            cls.get_server_uri(),
            data=json.dumps(data),
            headers={'content-type': 'application/json'}
        )

    @classmethod
    def send_carousel(cls, person, elements, height_ratio="tall"):
        data = {
            'messaging_type': 'RESPONSE',
            "recipient": {"id": person.fb_id},
            "message": {
                "attachment": {
                    "type": "template",
                    "payload": {
                        "template_type": "generic",
                        "elements": []
                    }

                }
            }
        }

        for element in elements:
            title, image_url, subtitle, action, buttons = element
            element_payload = {
                "title": title,
                "image_url": image_url,
                "subtitle": subtitle,
                "default_action": {
                    "webview_height_ratio": height_ratio
                },
                "buttons": []
            }
            if action[:4] == 'http':
                element_payload["default_action"]["type"] = "web_url"
                element_payload["default_action"]["url"] = action
            else:
                element_payload["default_action"]["type"] = "postback"
                element_payload["default_action"]["payload"] = action

            for button in buttons:
                title, action = button
                button_payload = {
                    "title": title
                }
                if action[:4] == 'http':
                    button_payload["type"] = "web_url"
                    button_payload["url"] = action
                else:
                    button_payload["type"] = "postback"
                    button_payload["payload"] = action

                element_payload["buttons"].append(button_payload)

            data["message"]["attachment"]["payload"]["elements"].append(element_payload)

        # print("\n\n")
        # print("Outgoing data")
        # print("======================")
        # pprint(data)
        # print("======================")
        response = requests.post(
            cls.get_server_uri(),
            data=json.dumps(data),
            headers={'content-type': 'application/json'}
        )

    @classmethod
    def register_handler(cls, **conditions):
        """ Decorator for registering a chatbot handler """
        def wrapper(handler):
            cls._handlers.append((conditions, handler))

        return wrapper

    @classmethod
    def check_condition(cls, person, event, condition_name, condition_value):
        chunks = condition_name.split("__")
        if chunks[0] == 'context':
            if condition_name[1] == 'contains':
                return condition_value in person.context

        if len(chunks) == 1:
            chunks.append('')

        if chunks[0] == 'postback':
            print("Checking for postback")
            if not event.has_postback():
                return False

            pb = event.get_postback()
            print("pb=", pb)

            if chunks[1] == 'eq':
                print("pb == value", pb, condition_value, pb == condition_value)
                return pb == condition_value

            if chunks[1] == 'in':
                return pb in condition_value

        if chunks[0] == 'message':
            if not event.has_message():
                return False

            if chunks[1] == 'eq':
                return event.get_message() == condition_value

            if chunks[1] == '':
                return condition_value

        if chunks[0] == 'state':
            if chunks[1] == 'eq':
                return person.state == condition_value

            if chunks[1] == 'in':
                return person.state in condition_value

        if chunks[0] == 'intent':
            if chunks[1] == 'eq':
                if not event.has_intent():
                    return False
                return event.get_intent() == condition_value

    def handle(self, person, event):
        assert isinstance(event, MessengerEvent)
        assert isinstance(person, Person)

        best_handler, best_score = None, -1

        for conditions, handler in self.handlers:
            if len(conditions) <= best_score:
                continue

            passed = True
            for c, v in conditions.items():
                if not self.check_condition(person, event, c, v):
                    passed = False
                    break

            if passed:
                best_handler = handler
                best_score = len(conditions)

        if best_handler is None:
            print("No appropriate handler found ...")
            return

        return best_handler(self, person, event)

# Meal Related handlers
Chatbot.register_handler(postback__in=PB_TO_MEAL_TYPES.keys())(
    meals.handle_log_meal_via_postback)
Chatbot.register_handler(intent__eq=Intents.LOG_MEAL)(
    meals.handle_log_meal_via_intent)
Chatbot.register_handler(postback__eq=PostBacks.SKIP_PHOTO, state__eq=States.WAITING_FOR_MEAL_PHOTO)(
    meals.handle_skip_meal_photo)
Chatbot.register_handler(postback__eq=PostBacks.SKIP_COMMENTS, state__eq=States.WAITING_FOR_MEAL_COMMENTS)(
    meals.handle_skip_comments)
Chatbot.register_handler(state__eq=States.WAITING_FOR_MEAL_PHOTO)(
    meals.handle_meal_photo)
Chatbot.register_handler(state__eq=States.WAITING_FOR_MEAL_COMMENTS)(
    meals.handle_meal_comments)
Chatbot.register_handler(postback__eq=PostBacks.CHECK_FOOD)(
    meals.handle_check_food)
Chatbot.register_handler(postback__eq=PostBacks.PREV_DAY, state__eq=States.CHECKING_FOOD)(
    meals.handle_prev_day)
Chatbot.register_handler(postback__eq=PostBacks.NEXT_DAY, state__eq=States.CHECKING_FOOD)(
    meals.handle_next_day)
Chatbot.register_handler(intent__eq=Intents.VIEW_MEAL_DIARY)(
    meals.handle_check_food_via_intent)

# Log Progress handlers
Chatbot.register_handler(postback__eq=PostBacks.LOG_PROGRESS)(
    progress.handle_log_progress)
Chatbot.register_handler(state__eq=States.WAITING_FOR_PROGRESS_PHOTO)(
    progress.handle_progress_photo)
Chatbot.register_handler(postback__eq=PostBacks.SKIP_PHOTO, state__eq=States.WAITING_FOR_PROGRESS_PHOTO)(
    progress.handle_skip_progress_photo)
Chatbot.register_handler(state__eq=States.WAITING_FOR_PROGRESS_WEIGHT)(
    progress.handle_progress_weight)
Chatbot.register_handler(postback__eq=PostBacks.SKIP_WEIGHT, state__eq=States.WAITING_FOR_PROGRESS_WEIGHT)(
    progress.handle_skip_progress_weight)
Chatbot.register_handler(state__eq=States.WAITING_FOR_PROGRESS_BMI)(
    progress.handle_progress_bmi)
Chatbot.register_handler(postback__eq=PostBacks.SKIP_BMI, state__eq=States.WAITING_FOR_PROGRESS_BMI)(
    progress.handle_skip_progress_bmi)

# Suggest Recipes
Chatbot.register_handler(intent__eq=Intents.SUGGEST_RECIPE)(
    recipes.handle_recipe_request_via_intent)

# Default Answer
Chatbot.register_handler(intent__eq=Intents.GREETING)(default.handle_say_hello)
Chatbot.register_handler(intent__eq=Intents.GOOD_BYE)(default.handle_say_bye)
Chatbot.register_handler(intent__eq=Intents.PROFANITY)(profanity.handle_profanity)
Chatbot.register_handler(postback__eq=PostBacks.GET_STARTED)(default.handle_get_started)
Chatbot.register_handler(postback__eq=PostBacks.CONTINUE_INTRO_PART2)(default.handle_get_started_part2)
Chatbot.register_handler()(default.handle_default)


import cv2

# Download the haarcascade_frontalcatface.xml from this link
# https://github.com/opencv/opencv/blob/master/data/haarcascades/haarcascade_frontalcatface.xml

# Cascade file that contains all what we need to detect faces
face_cascade = cv2.CascadeClassifier('haarcascade_frontalcatface.xml')

def detect():

    # Setting the webcam of our pc
    cap = cv2.VideoCapture(0)

    while True:
        _,img = cap.read()

        # Setting the gray
        gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

        # Face detected
        # face = face_cascade.detectMultiScale(gray, scale_factor, nb_faces )
        face = face_cascade.detectMultiScale(gray, 1.1, 4)

        # Create a rectangle for faces
        for (x, y, w, h) in face:
            cv2.rectangle(img, (x, y), (x+w, y+h), (255,0,0), 2)

        cv2.imshow("Face Detect", img)

        if cv2.waitKey(1)==27:
            break

    cap.release()

detect()


#!/usr/bin/env python
"""
_DCCPGenericImpl_

Implementation of StageOutImpl interface for DCCP for a generic
site.

Assumes the target dir will already exist.
Doesnt support deletion yet: This needs to be added

"""
import os
import logging

from WMCore.Storage.StageOutImplV2 import StageOutImplV2

_CheckExitCodeOption = True

class DCCPGenericImpl(StageOutImplV2):
    """
    _DCCPGenericImpl_

    Implement interface for dccp command

    """

    def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol  ):
        """
            performs a transfer. stageOut tells you which way to go. returns the new pfn or
            raises on failure. StageOutError (and inherited exceptions) are for expected errors
            such as temporary connection failures. Anything else will be handled as an unexpected
            error and skip retrying with this plugin

            if stageOut is true:
                The fromPfn is the LOCAL FILE NAME on the node, without file://
                the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
            if stageOut is false:
                The toPfn is the LOCAL FILE NAME on the node, without file://
                the fromPfn is the target PFN, mapped from the LFN using the TFC or overrrides

            this behavior is because most transfer commands will switch their direction
            simply by swapping the order of the arguments. the stageOut flag is provided
            however, because sometimes you want to pass different command line args

        """
        ourCommand = \
            self.generateCommandFromPreAndPostParts(\
                        ["dccp"],
                        [fromPfn, toPfn],
                        options)
        self.runCommandFailOnNonZero(ourCommand)

        return toPfn

    def doDelete(self, pfn, seName, command, options, protocol  ):
        """
            deletes a file, raises on error
            StageOutError (and inherited exceptions) are for expected errors
            such as temporary connection failures. Anything else will be handled as an unexpected
            error and skip retrying with this plugin
        """
        logging.info("Attempted to delete, but delete not supported")
        logging.info(pfn)


from ninja_ide.core import plugin

import atexit

from IPython.qt.console.rich_ipython_widget import RichIPythonWidget
from IPython.qt.inprocess import QtInProcessKernelManager

from PyQt4 import QtCore
from PyQt4.QtCore import SIGNAL

from ninja_ide.gui.main_panel import itab_item
from ninja_ide.resources import IMAGES
from PyQt4.QtGui import QWidget

class INinjaConsole(plugin.Plugin):
    def initialize(self):
        # Init your plugin
        self.misc_s = self.locator.get_service('misc')
        explorer_container = self.locator.get_service('explorer')
        tree = explorer_container.get_tree_projects()

        kernel_manager = QtInProcessKernelManager()
        kernel_manager.start_kernel()
        kernel = kernel_manager.kernel
        kernel.gui = 'qt4'

        kernel_client = kernel_manager.client()
        kernel_client.start_channels()

        def stop():
            kernel_client.stop_channels()
            kernel_manager.shutdown_kernel()

        self.ipython_console = RichIPythonWidget()
        self.ipython_console.kernel_manager = kernel_manager
        self.ipython_console.kernel_client = kernel_client
        self.ipython_console.exit_requested.connect(stop)
        self.ipython_console.show()

        self.misc_s.add_widget(self.ipython_console, IMAGES["console"],
                                "IPython console")
        addp = SIGNAL("addProjectToConsole(QString)")
        delp = SIGNAL("removeProjectFromConsole(QString)")
        self.connect(tree, addp, self._add_project)
        self.connect(tree, delp, self._del_project)

    def _add_project(self, project_path):
        self.ipython_console.execute("import sys; sys.path += ['%s']" %
                                    project_path)

    def _del_project(self, project_path):
        """This is ugly, but I was in a hurry and copied form console
        widget"""
        self.ipython_console.execute("import sys; "
            "sys.path = [path for path in sys.path "
            "if path != '%s']" % project_path)

    def finish(self):
        # Shutdown your plugin
        pass

    def get_preferences_widget(self):
        # Return a widget for customize your plugin
        pass


# -*- coding: utf-8 -*-
import scrapy

from appbee_crawler.manager.db_manager import DBManager
from appbee_crawler.spiders.parser.app_item_parser import AppItemParser

class AppInfoUpdateSpider(scrapy.Spider):
    name = "AppInfoUpdateSpider"
    allowed_domains = ["play.google.com"]

    def start_requests(self):
        package_names = DBManager.get_package_names_to_update_info()
        for package_name in package_names:
            if package_name == 'com.formakers.fomes':
                continue

            request = scrapy.Request('https://play.google.com/store/apps/details?id=' + package_name,
                                     meta={'packageName': package_name}, dont_filter=True)
            yield request

    def parse(self, response):
        yield AppItemParser.parse(response)


# -*- coding: utf-8 -*-
# @Time    : 2023/2/7 10:52
# @Author  : HongFei Wang

import tqdm
from docx import Document
from docxcompose.composer import Composer

def delete_paragraph(paragraph):
	p = paragraph._element
	p.getparent().remove(p)
	paragraph._p = paragraph._element = None

# 定义合并文档的函数
def merge_doc(source_file_path_list, target_file_path):
	"""
	:param source_file_path_list: 源文件路径列表
	:param target_file_path: 目标文件路径
	"""
	# 填充分页符号文档
	page_break_doc = Document()
	page_break_doc.add_section()
	# 定义新文档
	target_doc = Document(source_file_path_list[0])
	target_composer = Composer(target_doc)
	for i in tqdm.tqdm(range(len(source_file_path_list))):
		# 跳过第一个作为模板的文件
		if i == 0: continue
		target_composer.append(page_break_doc)  # 添加空白
		# 拼接文档内容
		f = source_file_path_list[i]
		f = Document(f)
		target_composer.append(f)
	# 保存目标文档
	target_composer.save(target_file_path)


from datetime import datetime, timedelta
import pandas as pd
from peewee import fn
from database import *
from args import *

def day_interval(date):
    return (
        datetime(date.year, date.month, date.day),
        datetime(date.year, date.month, date.day) + timedelta(days=1),
    )

def report_user_call(date=None, duration=30):
    date_begin, date_end = day_interval(date or datetime.now())

    fields = [
        UsersCall.context,
        UsersCall.processname,
        fn.Count(UsersCall.id).alias('count'),
        fn.Sum(UsersCall.duration).alias('duration'),
        fn.Avg(UsersCall.duration).alias('avg'),
        fn.Max(UsersCall.duration).alias('max'),
    ]

    query = UsersCall.select(*fields).where(
            (UsersCall.date>=date_begin)
            & (UsersCall.date<date_end)
            & (UsersCall.duration>duration*1e6)
        ).group_by(UsersCall.context, UsersCall.processname).order_by(SQL('avg').desc())

    result = list(query.dicts())
    if len(result)==0:
        return "Not data"

    df = pd.DataFrame(result)

    df['duration'] = round(df['duration']/1e6, 3)
    df['avg'] = round(pd.to_numeric(df['avg'])/1e6, 3)
    df['max'] = round(df['max']/1e6, 3)

    df.columns = ['Метод', 'База', 'Количество', 'Суммарная длительность', 'Средняя длительность', 'Максимальная длительность']

    return df.to_html(classes='report')

def report_db_call(date=None, duration=30):
    date_begin, date_end = day_interval(date or datetime.now())

    fields = [
        DBCall.context,
        DBCall.processname,
        DBCall.sql,
        fn.Count(DBCall.id).alias('count'),
        fn.Sum(DBCall.duration).alias('duration'),
        fn.Avg(DBCall.duration).alias('avg'),
        fn.Max(DBCall.duration).alias('max'),
    ]

    query = DBCall.select(*fields).where(
            (DBCall.date>=date_begin)
            & (DBCall.date<date_end)
            & (DBCall.duration>duration*1e6)
        ).group_by(DBCall.context, DBCall.sql, DBCall.processname).order_by(SQL('avg').desc())

    result = list(query.dicts())
    if len(result)==0:
        return "Not data"

    df = pd.DataFrame(result)

    df['duration'] = round(df['duration']/1e6, 3)
    df['avg'] = round(pd.to_numeric(df['avg'])/1e6, 3)
    df['max'] = round(df['max']/1e6, 3)
    df.columns = ['Метод', 'База', 'Текст', 'Количество', 'Суммарная длительность', 'Средняя длительность', 'Максимальная длительность']

    return df.to_html(classes='report')

def report_app_call(date=None, duration=30):
    date_begin, date_end = day_interval(date or datetime.now())

    fields = [
        AppCall.context,
        AppCall.processname,
        fn.Count(AppCall.id).alias('count'),
        fn.Sum(AppCall.duration).alias('duration'),
        fn.Avg(AppCall.duration).alias('avg'),
        fn.Max(AppCall.duration).alias('max'),
        fn.Avg(AppCall.cputime).alias('cpu'),
    ]

    query = AppCall.select(*fields).where(
            (AppCall.date>=date_begin)
            & (AppCall.date<date_end)
            & (AppCall.duration>duration*1e6)
        ).group_by(AppCall.context, AppCall.processname).order_by(SQL('avg').desc())

    result = list(query.dicts())
    #print(result)
    if len(result)==0:
        return "Not data"

    df = pd.DataFrame(result)

    df['duration'] = round(df['duration']/1e6, 3)
    df['avg'] = round(pd.to_numeric(df['avg'])/1e6, 3)
    df['max'] = round(df['max']/1e6, 3)
    df['cpu'] = pd.to_numeric(df['cpu'])
    df.columns = ['Метод', 'База', 'Количество', 'Суммарная длительность', 'Средняя длительность', 'Максимальная длительность', "Сред.CPU"]

    return df.to_html(classes='report')

def report_app_mem(date=None, top=10):
    date_begin, date_end = day_interval(date or datetime.now())

    fields = [
        AppCall.context,
        AppCall.processname,
        fn.Count(AppCall.id).alias('count'),
        fn.Avg(AppCall.memorypeak).alias('avg'),
        fn.Max(AppCall.memorypeak).alias('max'),
    ]

    query = AppCall.select(*fields).where(
            (AppCall.date>=date_begin)
            & (AppCall.date<date_end)
        ).group_by(AppCall.context, AppCall.processname).order_by(SQL('max').desc()).limit(top)

    result = list(query.dicts())
    if len(result)==0:
        return "Not data"

    df = pd.DataFrame(result)

    df['avg'] = round(pd.to_numeric(df['avg']))
    df['max'] = df['max']
    df.columns = ['Метод', 'База', 'Количество', 'Среднее', 'Максимальное']

    return df.to_html(classes='report')

if __name__ == "__main__":
    args = get_args()
    init_database(args.psql_url, drop=False)
    report_app_call(duration=0.001)


# ||AUM||
import scipy.io
import numpy as np
from dfsio import readdfs
from mayavi import mlab
#import h5py
import os
from sklearn.cluster import SpectralClustering
from scipy.stats import trim_mean
p_dir = 'E:\\HCP-fMRI-NLM'
lst = os.listdir(p_dir)
r_factor = 3
ref_dir = os.path.join(p_dir, 'reference')

ref = '100307'
print(ref + '.reduce' + str(r_factor) + '.LR_mask.mat')
fn1 = ref + '.reduce' + str(r_factor) + '.LR_mask.mat'
fname1 = os.path.join(ref_dir, fn1)
msk = scipy.io.loadmat(fname1)  # h5py.File(fname1);
dfs_left = readdfs(os.path.join(p_dir, 'reference', ref + '.aparc.a2009s.32k_fs.reduce3.left.dfs'))
dfs_left_sm = readdfs(os.path.join(p_dir, 'reference', ref + '.aparc.a2009s.32k_fs.reduce3.very_smooth.left.dfs'))
count1 = 0
rho_rho=[];rho_all=[]
for sub in lst: # ['751348']: #; lst:
    try:
        data = scipy.io.loadmat(os.path.join(p_dir, sub, sub + '.rfMRI_REST1_RL.reduce3.ftdata.NLM_11N_hvar_25.mat'))
    except:
        continue

    LR_flag = msk['LR_flag']
    LR_flag = np.squeeze(LR_flag) > 0
    data = data['ftdata_NLM']
    temp = data[LR_flag, :]
    m = np.mean(temp, 1)
    temp = temp - m[:,None]
    s = np.std(temp, 1)+1e-16
    temp = temp/s[:,None]
    msk_small_region = (dfs_left.labels == 46) | (dfs_left.labels == 28) | (dfs_left.labels == 29)  # % motor
    d = temp[msk_small_region, :]
    d_corr = temp[~msk_small_region, :]
    rho = np.corrcoef(d, d_corr)
    rho=rho[range(d.shape[0]),d.shape[0]+1:]
    rho[~np.isfinite(rho)] = 0
    rho_all.append(rho)
    rho_rho.append(np.corrcoef(rho))
    count1+=1
    print(count1),
    #if count1 >10 :
    #    break

A=trim_mean(np.array(rho_rho),.1)#np.mean(np.array(rho_rho),0)
#A=np.mean(np.array(rho_all),0)
print(A.shape)

for nClusters in [2,4,6,8,10,12]:
    SC=SpectralClustering(n_clusters=nClusters,affinity='precomputed',assign_labels='discretize')#,random_state=3425
    labs=SC.fit_predict(A)
    print(labs.shape)

    r = dfs_left_sm;r.labels = r.labels*0;r.labels[msk_small_region] = labs
    mesh = mlab.triangular_mesh(r.vertices[:,0], r.vertices[:,1], r.vertices[:,2], r.faces, representation='surface',
                            opacity=1,scalars=np.float64(r.labels))
    #mlab.pipeline.surface(mesh)
    mlab.gcf().scene.parallel_projection = True
    mlab.view(azimuth=0, elevation=-90)
    mlab.savefig(filename = str(nClusters)+'mean'+'labels1_1.png')
    mlab.view(azimuth=0, elevation=90)
    mlab.savefig(filename = str(nClusters)+'mean'+'labels2_1.png')
    mlab.close()


# (C) 2018-2020, Tom Eulenfeld, MIT license

import numpy as np
import matplotlib.pyplot as plt
import re

def _f(B, x):
    """Linear function y=m*x+b"""
    return B[0]*x + B[1]

def odr_regr(x, y):
    """Orthogonal distance regression, return m and b of line y=m*x+b"""
    from scipy.odr import Data, Model, ODR
    linear = Model(_f)
    mydata = Data(x, y, wd=1, we=1)
    myodr = ODR(mydata, linear, beta0=[1., 2.])
    res = myodr.run()
    return res.beta

def project_coordinates(x, y, m, b):
    """Project coordinates x,y onto line y=mx+b"""
    # direction of line (1, m)
    c = (1 * x + m * (y - b)) / (1 + m ** 2)
    return 1 * c, m * c + b

def plot_projection(x, y, xp, yp, m, b, fname=None):
    """Plot projection

    x,y: coordinates
    xp,yp: projected coordinates
    m,b: line y=m*x+b
    """
    plt.figure()
    plt.plot(x, y, '.')
    plt.plot(x, m*np.array(x)+b)
    plt.plot(xp, yp, '.')
    plt.axis('scaled')
    if fname:
        plt.savefig(fname)

def plot_html(fname_in, fname_out, zone=None, delimiter=',', encoding=None,
              color=None, color_func=None, reverse=False, **kw):
    """Plot html of coordinates in fname_in"""
    import folium
    import utm
    data = np.genfromtxt(fname_in, dtype=None, delimiter=delimiter, encoding=encoding, **kw)
    names, x, y, z, *_ = list(zip(*reversed(data))) if reverse else zip(*data)
    x0 = np.mean(x)
    y0 = np.mean(y)
    latlon0 = utm.to_latlon(x0, y0, *zone) if zone else (y0, x0)
    m = folium.Map(location=latlon0, zoom_start=15, max_zoom=20, control_scale=True)
    m.add_child(folium.LatLngPopup())
    for name, xp, yp, zp in zip(names, x, y, z):
        if color is None and color_func is None:
            c = '#3186cc'
        elif color is None:
            c = color_func(name)
        else:
            c = color
        latlon = utm.to_latlon(xp, yp, *zone) if zone else (yp, xp)
        folium.CircleMarker(latlon, radius=1,
                            popup='%s %.5f %.5f %.2f' % (name, latlon[0], latlon[1], zp),
                            color=c, fill_color=c
                            ).add_to(m)
    m.save(fname_out)

def read_coords(fname, delimiter=',', sort=True):
    """Read coordinates in fname

    name, x, y, z
    """
    data = np.genfromtxt(fname, dtype=None, encoding=None, delimiter=delimiter)
    names, x, y, z, *_ = zip(*data)
    nums = [int(re.sub('[a-zA-Z]+', '', name)) for name in names]
    if sort:
        ind = sorted(range(len(names)), key=lambda i: nums[i])
        nums = [nums[i] for i in ind]
        names = [names[i] for i in ind]
        x = [x[i] for i in ind]
        y = [y[i] for i in ind]
        z = [z[i] for i in ind]
    return names, np.array(nums), np.array(x), np.array(y), np.array(z)

def test_project_coords():
    x =  np.arange(10)
    y = 0.5 * x + 10 + 2*np.random.random(10)
    m, b = odr_regr(x, y)
    xp, yp = project_coordinates(x, y, m, b)
    plot_projection(x, y, xp, yp, m, b)

def write_geo(x, y, z, x0, y0, sps, fname1, fname2, rec0=1, sort=False,
              xyzs=None, shot0=1):
    """Write profil files (receivers and shots)

    x,y,z: profile coordinates
    x0,y0: profile in m relative to this coordinates
    fname1,fname2: file name for receivers, shots
    sps: shotpoints - geofon dictionary from read_shotpoints
    rec0,shot0: Start number of receivers/shots
    xyzs: sps can be set to None. Then, use xyzs.
          tuple of profile coordinates of shotpoints,
          (x, ys, zs)
    """
    fmt = '%3d  %8.3f  %8.3f'
    pos = ((x - x0) ** 2 + (y - y0) ** 2) ** 0.5
    num = np.arange(len(x)) + rec0

    if xyzs is not None:
        x2, y2, z2 = xyzs
        pos2 = ((x2 - x0) ** 2 + (y2 - y0) ** 2) ** 0.5
        num2 = np.arange(len(x2)) + shot0
    elif sps is not None:
        shot_ind=np.array(list(sps.values())) - 1
        num2 = sps.keys()
        pos2 = pos[shot_ind]
        z2 = z[shot_ind]
    else:
        raise

    if sort:
        z = [a for _, a in sorted(zip(pos, z))]
        z2 = [a for _, a in sorted(zip(pos2, z2))]
        pos = sorted(pos)
        pos2 = sorted(pos2)
    np.savetxt(fname2, list(zip(num2, pos2, z2)), fmt=fmt)
    np.savetxt(fname1, list(zip(num, pos, z)), fmt=fmt)

def profilemeter2xy(meters, x, y, print_lines=False):
    dy = y[-1] - y[0]
    dx = x[-1] - x[0]
    alpha = np.arctan2(dy, dx)
    res = [(meter * np.cos(alpha) + x[0],
            meter * np.sin(alpha) + y[0]) for meter in meters]
    if print_lines:
        for i, (x2, y2), in enumerate(res):
            print(f'x{i}={x2:.3f} y{i}={y2:.3f} z{i}=',)
    return res

def interpolate_geophones(nums, x, y, z, nnums=None):
    if nnums is None:
        nnums = np.arange(nums[0], nums[-1]+1)
    res = [np.interp(nnums, nums, coordpart) for coordpart in (x, y, z)]
    return res

if __name__ == '__main__':
    test_project_coords()


import csv
from typing import Tuple, Set, Dict
from pathlib import Path

import numpy as np
from pygame import Surface

from game.cell import Cell
from game.configs import Colors

class CellPosition(object):
    """Stores a cell position in a board, in units"""

    def __init__(self, x: int, y: int):
        self.x = x
        self.y = y

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y

    def __hash__(self):
        return hash(self.__repr__())

    def __repr__(self):
        return f"CellPosition( x = {self.x}, y = {self.y} )"

    def as_tuple(self) -> Tuple[int, int]:
        return self.x, self.y

def get_position_neighbours(position: 'CellPosition') -> Set['CellPosition']:
    neighbours: Set[CellPosition] = {
        CellPosition(position.x - 1, position.y - 1),
        CellPosition(position.x, position.y - 1),
        CellPosition(position.x + 1, position.y - 1),
        CellPosition(position.x + 1, position.y),
        CellPosition(position.x + 1, position.y + 1),
        CellPosition(position.x, position.y + 1),
        CellPosition(position.x - 1, position.y + 1),
        CellPosition(position.x - 1, position.y)}        # Left

    return neighbours

class Board(object):

    def __init__(self, columns: int, rows: int, cell_side: int,
                 window_size: Tuple[int, int]):

        self.columns = columns
        self.rows = rows
        self.cell_side = cell_side
        self.window_size = window_size

        self.window = Surface(window_size)
        self.window.fill(Colors.BLACK)

        self.board: np.array = np.array([
            [Cell(self.cell_side, (row * cell_side, column * cell_side))
             for column in range(rows)]
            for row in range(columns)
        ])

        self.active_cells: Set[CellPosition] = set()

    def reset(self):
        for cell_position in self.active_cells:
            self.board[cell_position.as_tuple()].die()

    def render(self, window: Surface, position: Tuple[int, int]):

        [[cell.render(self.window) for cell in column] for column in self.board]
        window.blit(self.window, position)

    def handle_click(self, x: int, y: int):

        cell_x = int(x / self.cell_side)
        cell_y = int(y / self.cell_side)
        cell_position = CellPosition(cell_x, cell_y)

        self.board[cell_position.as_tuple()].change_state()

        if self.board[cell_position.as_tuple()].is_alive():
            self.active_cells.add(CellPosition(cell_x, cell_y))
        else:
            self.active_cells.remove(CellPosition(cell_x, cell_y))

    def get_alive_neighbours(self, position: CellPosition) -> int:
        alive_neighbours = 0
        for neighbour in get_position_neighbours(position):
            if self.valid_position(neighbour):
                alive_neighbours += int(self.board[neighbour.as_tuple()].is_alive())

        return alive_neighbours

    def valid_position(self, position: CellPosition):
        x_valid = 0 <= position.x < self.columns
        y_valid = 0 <= position.y < self.rows

        return x_valid and y_valid

    def update_state(self):
        cells_alive_neighbours: Dict[CellPosition, int] = {}
        for active_cell in self.active_cells:

            if active_cell not in cells_alive_neighbours:
                cells_alive_neighbours[active_cell] = self.get_alive_neighbours(active_cell)

            for neighbour in get_position_neighbours(active_cell):
                if neighbour not in cells_alive_neighbours and self.valid_position(neighbour):
                    cells_alive_neighbours[neighbour] = self.get_alive_neighbours(neighbour)

        new_alive: Set[CellPosition] = set()
        for cell_position, alive_neighbours in cells_alive_neighbours.items():
            cell = self.board[cell_position.as_tuple()]
            cell.update(alive_neighbours)
            if cell.is_alive():
                new_alive.add(cell_position)

        self.active_cells = new_alive

    def load_board(self, board_file: Path):
        self.reset()
        with open(board_file, "r", newline="") as f:
            reader = csv.reader(f)

            for y, row in enumerate(reader):
                for x, value in enumerate(row):
                    if value == "1":
                        self.board[(x, y)].populate()
                        self.active_cells.add(CellPosition(x, y))

    def save_board(self, board_file: Path):
        with open(board_file, "w", newline="") as f:
            writer = csv.writer(f)

            for y in range(self.rows):
                write_row = []
                for x in range(self.columns):
                    if self.board[(x, y)].is_alive():
                        write_row.append("1")
                    else:
                        write_row.append("0")
                writer.writerow(write_row)


import logging

from django.http import HttpResponseRedirect
from django.shortcuts import render, redirect
from django.urls import reverse

from stu.models import Student, StudentInfo, Class

logger = logging.getLogger('stu')

def index(request):

    if request.method == 'GET':
        stuinfos = StudentInfo.objects.all()
        logger.info('url: %s method:%s 获取学生信息成功' % (request.path, request.method))
        return render(request, 'index.html', {'stuinfos': stuinfos})

def addStu(request):
    if request.method == 'GET':
        return render(request, 'addStu.html')

    if request.method == 'POST':
        name = request.POST.get('name')
        num = request.POST.get('num')

        stu = Student.objects.create(
            stu_name=name,
            stu_num=num
        )
        return redirect(
            reverse('s:addinfo', kwargs={'stu_id': stu.id})
        )

def addStuInfo(request, stu_id):

    if request.method == 'GET':

        return render(request, 'addStuInfo.html', {'stu_id':stu_id})

    if request.method == 'POST':

        stu_id = request.POST.get('stu_id')

        # 添加头像图片
        img = request.FILES.get('img')

        StudentInfo.objects.create(s_id=stu_id, i_image=img)

        return HttpResponseRedirect('/stu/index/')

def addClass(request):

    if request.method == 'GET':

        return render(request, 'addClass.html')

    if request.method == 'POST':

        name = request.POST.get('name')
        num = request.POST.get('num')
        clas = request.POST.get('clas')

        Student.objects.create(
            s_name=name,
            s_num=num,
            s_class=clas
            )
        Class.objects.create(s_class=clas,
                             s_name=name,
                             s_num=num)
        return HttpResponseRedirect('/stu/index/')

def addGrade(request):

    if request.method == 'GET':

        return render(request, 'addGrade.html')

    if request.method == 'POST':

        s_num = request.POST.get('num')
        s_name = request.POST.get('name')
        s_sex = request.POST.get('sex')
        s_Chinese = request.POST.get('Chinses')
        s_Math = request.POST.get('Math')
        s_English = request.POST.get('English')

        StudentInfo.objects.create(s_num=s_num,
                                   s_name=s_name,
                                   s_sex=s_sex,
                                   s_Chinese=s_Chinese,
                                   s_Math=s_Math,
                                   s_English=s_English)

        return HttpResponseRedirect('/stu/index/')

def selectStu(request):
    stus = Student.objects.all()

    if request.method == 'POST':
        s_name = request.POST.get('stuname')
        stus = Student.objects.filter(stu_name=s_name)
        return render(request, 'sel_stut.html', {'stus': stus})

    return render(request, 'sel_stu.html', {'stus': stus})


import numpy as np
import pandas as pd
import plotly.express as px
from dash import Dash, Input, Output, dcc, html
from umap import UMAP

app = Dash(__name__)
app.title = "UMAP Projections"

app.layout = html.Div(
    [
        html.Div(
            children=[
                html.H1(
                    "UMAP Projections for MNIST and Fashion-MNIST Datasets",
                    style={"text-align": "center"},
                ),
                dcc.Markdown(
                    """
                    Uniform Manifold Approximation and Projection (UMAP) is a general-purpose dimension reduction algorithm. Similar to t-distributed stochastic neighbor embedding (t-SNE), you can use UMAP to visualize the relationships between data points. In this example, we are training a three-component UMAP model on MNIST datasets and then displaying the 3D graph of the result. The color of the point in the graph is based on the label. In the resulting graph, blobs of colors show that UMAP clustered data points with similar labels together.
                """,
                ),
            ],
            style={"padding": 10},
        ),
        html.Div(
            [
                html.Div(
                    children=[
                        html.H1("Input"),
                        html.Label("Dataset"),
                        dcc.Dropdown(
                            ["MNIST-Digits", "MNIST-Fashion"], "MNIST-Digits", id="dataset_dropdown"
                        ),
                    ],
                    style={"padding": 10, "flex": 1},
                ),
                html.Div(
                    children=[
                        html.H1("Output"),
                        dcc.Loading(
                            id="loading-1",
                            children=[dcc.Graph(id="graph")],
                            type="circle",
                        ),
                    ],
                    style={"padding": 10, "flex": 3},
                ),
            ],
            style={"display": "flex", "flex-direction": "row"},
        ),
    ]
)

@app.callback(
    Output("graph", "figure"),
    Input("dataset_dropdown", "value"),
)
def update_figure(selected_dataset):
    if selected_dataset == "MNIST-Digits":
        X = pd.read_csv(
            "https://saturn-public-data.s3.us-east-2.amazonaws.com/MNIST-1000/mnist-1000-input.csv"
        )
        y = pd.read_csv(
            "https://saturn-public-data.s3.us-east-2.amazonaws.com/MNIST-1000/mnist-1000-labels.csv"
        )
        y = np.unique(y, return_inverse=True)[1]

    elif selected_dataset == "MNIST-Fashion":
        X = pd.read_csv(
            "https://saturn-public-data.s3.us-east-2.amazonaws.com/MNIST-1000/fashion-1000-input.csv"
        )
        y = pd.read_csv(
            "https://saturn-public-data.s3.us-east-2.amazonaws.com/MNIST-1000/fashion-1000-labels.csv"
        )
        y = np.unique(y, return_inverse=True)[1]

    else:
        return None, "Please select a dataset."

    umap_3d = UMAP(n_components=3, init="random", random_state=0)

    proj_3d = umap_3d.fit_transform(X, y=y)

    fig = px.scatter_3d(proj_3d, x=0, y=1, z=2, color=y)

    fig.update_layout(transition_duration=500, height=1000)
    fig.update(layout_coloraxis_showscale=False)
    fig.update_traces(marker_size=2)

    return fig

if __name__ == "__main__":
    app.run_server(host="0.0.0.0", port="8000")


from bs4 import BeautifulSoup
import requests
import pandas as pd
import time
import regex as re
import random

data = {"title" : [], "link" : []}

temp = open("links.txt",'r').readlines()
adresses = []
for i in temp:
    adresses.append(i.replace("\n",""))

# Here's a function which gathers links from steam search page if particular title link covers searched game (including dlc, soundtrack etc.).
# I use this if statement because I don't know which game from the list is available on Steam, and result may not be a polish game if I search absent title.
def link_soup(adres):
    url = adres
    cookies = {"birthtime" : "",
        "wants_mature_content" : "1",
        "lastagecheckage" : ""} # here were cookies from my browser to avoid age checking
    headers = {
        "user-agent" : "",
        "referer" : "https://store.steampowered.com/"
        }
    r = requests.get(url, headers = headers)
    print(r.status_code)

    soup = BeautifulSoup(r.content, "html.parser")
    results_container = soup.find("div", id="search_resultsRows")
    query = soup.find("input",  id="term")["value"]

    if results_container is None:
        print("Not found " + str(query))
    else:
        for i in results_container.find_all("a"):
            link = str(i["href"])
            title = str(i.span.string)
            if re.match(query, title):
                data["title"].append(title)
                data["link"].append(link)

for i in adresy:
    time.sleep(random.uniform(1, 1.5))
    link_soup(i)

frame = pd.DataFrame(data)
frame.drop_duplicates(inplace=True)
frame.to_csv("links_and_titles.csv", encoding = "utf-8", index = False)


import logging

import faker
from django.urls import reverse
from django.utils import timezone
from rest_framework import status

from polls.models import Question, Choice
from tests import BaseTestCase, HttpMethod

fake = faker.Faker()

logger = logging.getLogger(__name__)

class QuestionTests(BaseTestCase):
    """
    Given the following failing tests, do what is required externally to this module to make them pass.

    i.e. There should be no changes to this file.  Instead, find the areas in the application to augment
    and fix the failing tests.
    """
    __test__ = True

    def create_question(self):
        pub_date = timezone.make_aware(fake.date_time_this_year())
        question = Question.objects.create(question_text=fake.catch_phrase(), pub_date=pub_date)
        for __ in range(fake.random_digit()):
            Choice.objects.create(question=question, choice_text=fake.bs(), votes=fake.pyint())
        return question

    def test_multi_update(self):
        """
        Augment the endpoint to handle updating multiple objects within a single request.
        """
        # Create some test objects
        expected_count = 10
        self.assertFalse(Question.objects.exists())

        for __ in range(expected_count):
            self.create_question()

        self.assertEqual(Question.objects.count(), expected_count)

        url = reverse("question-list")
        self.authenticate()

        # Build a payload for the request.  In this case we're sending a non-standard payload
        # that includes multiple objects to update.
        payload = []
        for obj in Question.objects.all():
            payload.append(dict(url=reverse("question-detail", kwargs=dict(pk=obj.pk)),
                                question_text=fake.bs()))

        # Send the request - we're doing a partial update in this case (i.e. PATCH vs a PUT)
        response, data = self.request(HttpMethod.PATCH, url, data=payload, authenticated=True)
        self.assertResponseStatus(response, status_code=status.HTTP_200_OK)
        self.assertEqual(len(data["results"]), expected_count)

    def test_has_date_created(self):
        """
        Augment the model to have a `date_created` field.

        This field should be automatically be set with a datetime value representing the point in time
        that the instance was created.  Make sure to create a migration file as well.  (python manage.py makemigrations)

        Furthermore, augment the serializer to include the new field.
        """
        obj = self.create_question()
        url = reverse("question-detail", kwargs=dict(pk=obj.pk))
        response, data = self.request(HttpMethod.GET, url, authenticated=True)
        self.assertResponseStatus(response, status_code=status.HTTP_200_OK)
        self.assertIn("pub_date", data)
        self.assertIn("url", data)
        self.assertIn("question_text", data)
        self.assertIn("choices", data)
        self.assertIn("date_created", data, msg="date_created isn't in the serialized data yet")
        self.assertIsNotNone(data["date_created"])

    def test_query_count_is_off(self):
        """
        Something is causing a higher than expected number of queries executed during a list request.

        Find and fix the area of the application that would be responsible for this.
        """
        # Create a bunch of test objects
        for __ in range(fake.pyint()):
            self.create_question()

        url = reverse("question-list")
        self.authenticate()

        # We only expect 11 queries to execute in total if this request was optimized
        with self.assertNumQueries(11):
            response, data = self.request(HttpMethod.GET, url, authenticated=True)
            self.assertResponseStatus(response, status_code=status.HTTP_200_OK)
            self.assertGreaterEqual(len(data["results"]), 1)
            obj = data["results"][0]
            self.assertIn("pub_date", obj)
            self.assertIn("url", obj)
            self.assertIn("question_text", obj)
            self.assertIn("choices", obj)


import sys
import types

from utils import Set

class PDA:
    def __init__(self, Q, Σ, Γ, δ, q0, Z0, F):
        """

        Q  : set of states
        Σ  : finite alphabet
        Γ  : set of alphabet of Stack
        δ  : Q × Σ → Q transition function
        q0 : q0 ∈ Q initial states
        Z0 : initial symbol in Stack
        F  : F ⊆ Q set of accepting states

        """
        import PDA

        assert isinstance(Q, Set), "Q must be a Set of states"
        assert isinstance(Σ, Set), "Σ must be a Set of alphabet"
        if not (isinstance(δ, PDA.δ) or isinstance(δ, PDA.ô)):
            raise AssertionError(
                f"δ must be a transition function PDA.δ, but is {type(δ)}")
        assert q0 in Q, f"condition q0 ∈ Q not met, {q0} not in {Q}"
        assert isinstance(F, Set), "F must be a Set of accepting states"
        assert all(qf in Q for qf in F) or F.empty(
        ), f"condition F ⊆ Q not met, element of {F} not in {Q}"

        self.__dict__.update({
            "Q": Q.map(str), "Σ": Σ.map(str), "Γ": Γ.map(str), "δ": δ, "q0": str(q0), "Z0": str(Z0), "F": F.map(str)})

        self.δ.pda = self

    def __str__(self):
        return (f"A = ({self.Q}, {self.Σ}, {self.Γ}, δ, {self.q0}, {self.Z0}, {self.F}) where\n" +
                f"δ = {self.δ if getattr(self, 'δ', False) else self.ô}\n")

class δ(dict):
    def __setitem__(self, key, val):
        super().__setitem__(tuple(map(str, key)), Set(map(str, val)))

    def __getitem__(self, key):
        if key not in self:
            self.__setitem__(key, Set())
        return super().__getitem__(key)

class ô(dict):
    def __setitem__(self, key, val):
        super().__setitem__(key, val)

    def __getitem__(self, key):
        if key not in self:
            self.__setitem__(key, Set())
        return super().__getitem__(key)

class call(types.ModuleType):
    """
    make the module callable simplifying
    from PDA import PDA
    to
    import PDA
    """

    def __init__(self):
        types.ModuleType.__init__(self, __name__)
        # or super().__init__(__name__) for Python 3
        self.__dict__.update(sys.modules[__name__].__dict__)

    def __call__(self, *args, **kwargs):
        return PDA(*args, **kwargs)

sys.modules[__name__] = call()


from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField, SubmitField, DecimalField

class QuerytableForm(FlaskForm):

	ticker = StringField('Input stock ticker:')
	year = StringField('Input fiscal period (2010Q3):', default="2019Q3")
	submit = SubmitField('Submit')

class DCF_Form(FlaskForm):

	ticker = StringField('Input stock ticker:')
	fiscal_period = StringField('Input the fiscal period for analysis (2010Q3):', default="2019Q3")
	years_project = IntegerField('Input the number of years in the future to project FCFs:', default="5")
	growth_rte = DecimalField('Input the expected FCF growth rate:', default=0.09, places=3)
	decline_rte = DecimalField('Input the rate of decline:', default=0.05, places=3)
	discount_rte = DecimalField('Input the discount rate:', default=0.09, places=3)
	terminal_growth_rt = DecimalField('Input the terminal growth rate (recommend between 2.5-3%):', default=0.025, places=3)
	submit = SubmitField('Submit')

class PE_Form(FlaskForm):

	ticker = StringField('Input stock ticker:')
	fiscal_period = StringField('Input the fiscal period for analysis (2019Q3):', default="2019Q3")
	years_project = IntegerField('Input the number of years in the future to project EPS:', default="5")
	growth_rte = DecimalField('Input the expected EPS growth rate:', default=0.09, places=3)
	discount_rte = DecimalField('Input the discount rate:', default=0.09, places=3)
	submit = SubmitField('Submit')


#!/usr/bin/env python
# encoding: utf-8
from flask import Flask, jsonify, request, send_file
app = Flask(__name__)

import os
# import gnews
import openai # added
from langchain.chat_models import ChatOpenAI
from dotenv import load_dotenv
# from datetime import datetime
# import requests
# import datetime
load_dotenv()
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from moviepy.editor import concatenate_audioclips, AudioFileClip
import uuid
# from elevenlabs import voices, generate, play, save

openai.api_key = os.getenv("OPENAI_API_KEY")
weather_api_key = os.getenv("WEATHER_API_KEY")
ELEVEN_API_KEY = os.getenv("ELEVEN_API_KEY")
# # Initialize chat model
chat_model = ChatOpenAI(model_name='gpt-3.5-turbo-16k')  # Make sure to set the API key as an environment variable

class Podcast:
    def __init__(self, user_prompt, num_minutes, category):
        self.ID = generate_name()
        self.user_prompt = user_prompt
        self.num_minutes = int(num_minutes)
        self.num_words = int(num_minutes) * 135
        self.category = category
        self.text_list = []
        self.sound_file_name = self.ID + ".wav"
        self.drive_name = ""

    def get_ID(self):
        return self.ID

    def get_drive_name(self):
        return self.drive_name

    def gen_story_podcast(self):

        if self.num_words < 1000:
            prompt = "Create a story about '{}'. The story should be approximately {} number of words".format(self.user_prompt, self.num_words)
            self.text_list = [chat_model.predict(prompt)]
            print("--------")
            print(self.text_list)
        else:
            chapters = int(self.num_words / 300)
            prompt = "Create a detailed outline for a story about '{}' with {} scenes. Return your answer in the following format:".format(self.user_prompt, str(chapters*3))
            for i in range(1, 3*chapters+1):
                prompt += "\nScene {}:".format(str(i))

            outline = chat_model.predict(prompt)
            scene_list = []
            for i in range(1, chapters*3+1):
                scene_before_index = outline.index("Scene {}".format(str(i)))
                if i < chapters*3:
                    scene_after_index = outline.index("Scene {}".format(str(i+1)))
                    scene = outline[scene_before_index:scene_after_index]
                else:
                    scene = outline[scene_before_index:]
                scene_list.append(scene)

            prompt = "Specify the characters and in what scene they enter the story based on this outline:\n{}\n\nReturn in the format:\nName of character - Description - Scene Number".format(outline)
            characters = chat_model.predict(prompt)
            extended_scenes = []

            for i in range(chapters):
                number_of_scenes = 3
                response_format = "Return in the format:\nPart {}:\nPart {}:\nPart {}:".format(str(i*3+1), str(i*3+2), str(i*3+3))
                scenes = scene_list[i*3] + "\n" + scene_list[i*3+1] + "\n" + scene_list[i*3+2]
                if i < chapters - 1:
                        scenes += "\n" + scene_list[i*3+3]
                        response_format += "\nPart {}:".format(str(i*3+4))
                        number_of_scenes = 4

                #prompt = "Turn these {} scenes into storytelling format, add details, return just the text:\n{}\n\nHere is a list of characters and in what scene they will or have been introduced:\n{}\n\n{}".format(str(number_of_scenes), scenes, characters, response_format)
                prompt = "Here are the story's characters and in what scenes they appear:\n{}\n\nRewrite the following scenes into a plain text story that can written in a book. Add unnecessary details so the story feels more alive:\n{}\n\n{}".format(characters, scenes, response_format)
                response = chat_model.predict(prompt)

                k = 0
                for j in range(i*3+1,i*3+number_of_scenes+1):
                    k += 1
                    scene_before_index = response.lower().index("part {}".format(str(j)))
                    if j < i*3+number_of_scenes:
                        scene_after_index = response.lower().index("part {}".format(str(j+1)))
                        scene = response[scene_before_index:scene_after_index]
                    else:
                        scene = response[scene_before_index:]

                    if k < 4:
                        extended_scenes.append(scene)

            cleaned_text = []
            for row in extended_scenes:
                split_row = row.split("\n")[1:]  # Remove the first element
                restored_row = "\n".join(split_row)  # Join the remaining elements with newline
                restored_row = restored_row.strip("\n")
                cleaned_text.append(restored_row)

            self.text_list = cleaned_text
            print("--------")
            print(self.text_list)

    def audiofy_story(self, voice="Adam", sound_effect_name="fairytale"):

        if not os.path.exists("story_audio"):
            os.makedirs("story_audio")

        file_names = []
        # for i in range(len(self.text_list)):
        #     audio = generate(text=self.text_list[i], voice=voice, model="eleven_monolingual_v1", api_key=ELEVEN_API_KEY)
        #     save(audio, "./story_audio/story_{}_{}.wav".format(self.ID, str(i)))

        #     file_names.append("./story_audio/story_{}_{}.wav".format(self.ID, str(i)))
        #     file_names.append("./sound_effects/{}.wav".format(sound_effect_name))

        # file_names = ["./sound_effects/fairytale.wav","./sound_effects/fairytale.wav"]
        file_names = ["./sound_effects/original_transition.wav","./sound_effects/original_transition.wav"]
        concatenate_audio_moviepy(file_names, "./story_audio/{}".format(self.sound_file_name))

    def upload_wav_file_and_get_ID(self):
        print(777777)
        gauth = GoogleAuth()

        # Try to load saved client credentials
        gauth.LoadCredentialsFile("mycreds.txt")

        if gauth.credentials is None:
            # Authenticate if they're not there
            gauth.GetFlow()
            gauth.flow.params.update({'access_type': 'offline'})
            gauth.flow.params.update({'approval_prompt': 'force'})
            gauth.LocalWebserverAuth()
        elif gauth.access_token_expired:
            # Refresh them if expired
            gauth.Refresh()
        else:
            # Initialize the saved creds
            gauth.Authorize()

        # Save the current credentials to a file
        gauth.SaveCredentialsFile("mycreds.txt")

        drive = GoogleDrive(gauth)

        title = generate_name() + ".txt"
        team_drive_id = '1WdeZhQ_vegXPMA-JeoM0pldAOKbCCVcx'
        parent_folder_id = '1TtWk3uo0jTC0BR2CAlaH8DgcRtp0hDCb'
        f = drive.CreateFile({
            'title': title,
            'parents': [{
                'kind': 'drive#fileLink',
                'teamDriveId': team_drive_id,
                'id': parent_folder_id
            }]
        })
        f.SetContentFile("./story_audio/" + self.sound_file_name)
        f.Upload(param={'supportsTeamDrives': True})

        files = drive.ListFile({"q": "'" + parent_folder_id + "' in parents and mimeType!='application/vnd.google-apps.folder'"}).GetList()

        name = ""
        for file in files:
            if file['title'] == title:
                name = file['id']

        self.drive_name = name

    def list_to_text(self):
        text = ""
        for row in self.text_list:
            text += row + "\n"
        return text

def generate_name():
    # Generate a unique ID
    unique_id = str(uuid.uuid4())
    return unique_id

def concatenate_audio_moviepy(audio_clip_paths, output_path):
        """Concatenates several audio files into one audio file using MoviePy
        and save it to `output_path`. Note that extension (mp3, etc.) must be added to `output_path`"""
        clips = [AudioFileClip(c) for c in audio_clip_paths]
        final_clip = concatenate_audioclips(clips)
        final_clip.write_audiofile(output_path)

@app.route('/')
def home():
    return 'Hello, World adam!'

@app.route('/about')
def about():
    return 'adam test'

@app.route('/create')
def get_create():
    print(00000)

    # Argument fetched from url params
    user_prompt = request.args.get('topic') # User input can be any string
    num_minutes = request.args.get('min') # Time [0-5, 5-10, 10-15]
    podcast_type = request.args.get("style") # style ["NEWS", "STORY"]
    # num_words = int(num_minutes)*135

    # Pausar detta för tillfället
    # prompt = "Can you create a documentary/news broadcast/podcast/fantasy story about this input? Respond only by returning 'YES' or explain why it does not work. Input = {}".format(user_prompt)
    # response = chat_model.predict(prompt)
    # if "YES" != response:
    #     return print(response)

    new_podcast = Podcast(user_prompt, num_minutes, podcast_type)

    if podcast_type == "NEWS":
        # Todo fix
        print("IN NEWS")
        podcast_name = temp_gen()
        data = {'text': "TODO", 'name': podcast_name}
        return jsonify(data)
    elif podcast_type == "STORY":
        new_podcast.gen_story_podcast()
        new_podcast.audiofy_story()
        new_podcast.upload_wav_file_and_get_ID()

    data = {'text': new_podcast.list_to_text(), 'name': new_podcast.get_drive_name()}

    return jsonify(data)