import itertools
from collections import UserDict
from operator import itemgetter
from typing import Dict, Iterator, List, Set, Tuple, TypeVar, Union, cast

from pyproj import Proj, Transformer
from shapely.geometry import MultiLineString, MultiPolygon, Polygon
from shapely.geometry.base import BaseGeometry
from shapely.geometry.collection import GeometryCollection
from shapely.ops import polygonize, transform, unary_union

from ....utils.cache import cached_property
from ....utils.geometry import reorient
from .. import Overpass
from ..core import Relation

class RelationsDict(UserDict):
    def __missing__(self, key):
        value = self[key] = list()
        return value

    def include(self, chunk: BaseGeometry, role: str):
        if isinstance(chunk, MultiLineString):
            for c in chunk.geoms:
                self[role].append(c)
        else:
            self[role].append(chunk)

T = TypeVar("T", bound="Boundary")

class Boundary(Relation):
    """A class to parse boundary=* relations.

    boundary=* relations are used for grouping boundaries and marking
    enclaves/exclaves.

    Reference: https://wiki.openstreetmap.org/wiki/Relation:boundary

    Tags:
      - type (boundary)
      - boundary (administrative)
      - land_area (administrative)
      - name
      - admin_level

    Relation members:
      - outer 1+
      - inner 0+
      - admin_center 0-1
      - label 0-1
      - subarea 0+
    """

    def __init__(self, json):
        super().__init__(json)

        self.known_chunks = RelationsDict()
        self.parent: Overpass = json["_parent"]

        self.parsed_keys = cast(
            Dict[int, Dict[str, Union[int, str]]],
            dict(
                (elt["ref"], elt)
                for elt in self.parent.all_members[self.json["id_"]]
            ),
        )

        self._build_geometry_parts()
        if len(self.known_chunks) > 0:
            self._make_geometry(self.known_chunks)

    def simplify(
        self: T,
        resolution: float,
        bounds: Union[None, Tuple[float, float, float, float]] = None,
    ) -> T:
        if bounds is None:
            bounds = self.parent.bounds
        b0, b1, b2, b3 = bounds

        def simplify_shape(shape_: BaseGeometry) -> BaseGeometry:
            proj = Proj(
                proj="aea",  # equivalent projection
                lat_1=b1,
                lat_2=b3,
                lat_0=(b1 + b3) / 2,
                lon_0=(b0 + b2) / 2,
            )

            forward = Transformer.from_proj(
                Proj("EPSG:4326"), proj, always_xy=True
            )
            backward = Transformer.from_proj(
                proj, Proj("EPSG:4326"), always_xy=True
            )
            return transform(
                backward.transform,
                transform(forward.transform, shape_).simplify(resolution),
            )

        rel_dict = RelationsDict()
        for role, shapes in self.known_chunks.items():
            for elt in shapes:
                rel_dict.include(simplify_shape(elt), role)

        self._make_geometry(rel_dict)
        return self

    def _make_geometry(self, parts: RelationsDict):
        outer: List[Polygon] = list(polygonize((unary_union(parts["outer"]))))
        inner: List[Polygon] = list(polygonize((unary_union(parts["inner"]))))

        if len(outer) == 0:
            return GeometryCollection()

        if len(outer) > 1:
            list_ = [
                Polygon(o, holes=list(i for i in inner if o.contains(i)))
                for o in outer
            ]
            shape = MultiPolygon(list_)

        else:
            shape = Polygon(outer[0], inner)

        self.shape = reorient(shape)
        self.json["geometry"] = self.shape

    def intersections(self) -> Iterator[Tuple[int, Set[int]]]:
        all_sets: Dict[int, Set[int]] = dict(
            (key, set(e["ref"] for e in list_))
            for key, list_ in self.parent.all_members.items()
        )
        key_i = self.json["id_"]
        set_i = all_sets[key_i]
        for key_j, set_j in all_sets.items():
            if key_i == key_j:
                continue
            intersection = set_i.intersection(set_j)
            if len(intersection) > 0:
                yield key_j, intersection

    @cached_property
    def neighbours(self) -> Set[int]:
        return set(key for key, _value in self.intersections())

    def include(self, elements: Set[int]) -> None:
        for role, it in itertools.groupby(
            (self.parsed_keys[key] for key in elements), key=itemgetter("role")
        ):
            elts = list(
                elt["geometry"] for elt in it if elt["geometry"] is not None
            )
            if len(elts) > 0:
                chunk = unary_union(elts)
                self.known_chunks.include(chunk, role)

    def _build_geometry_parts(self) -> BaseGeometry:
        all_: Set[int] = set(self.parsed_keys.keys())

        for _id, intersection in self.intersections():
            self.include(intersection)
            all_ -= intersection

        self.include(all_)

# Forcing the inheritance for proper registration
class Land_Area(Boundary):
    pass


import logging
import re

import requests

from .base import ParserBase

bot = logging.getLogger("rse.main.parsers.zenodo")

class ZenodoParser(ParserBase):
    name = "zenodo"
    matchstring = "^10[.][0-9]{4}/zenodo[.][0-9]{7}$"

    def _set_uid(self, uid):
        """Given some kind of Zenodo uri, parse it"""
        match = re.search(self.matchstring, uid)
        if not match:
            raise RuntimeError(f"{uid} does match a Zenodo DOI.")
        return match.group()

    def load_secrets(self):
        """load secrets, namely the access token, check if required and
        exit if not provided
        """
        self.token = self.get_setting("TOKEN")

    def get_url(self, data=None):
        """a common function for a parser to return the html url for the
        upper level of metadata
        """
        data = data or self.data
        return data.get("links", {}).get("html")

    def get_avatar(self, data=None):
        """a common function for a parser to return an image."""
        data = data or self.data
        return data.get("links", {}).get("badge")

    def get_description(self, data=None):
        """a common function for a parser to return a description."""
        data = data or self.data
        return data.get("metadata", {}).get("description")

    def get_metadata(self, uri=None, require_repo=True):
        """Retrieve repository metadata. The common metadata (timestamp) is
        added by the software repository parser, and here we need to
        ensure that the url field is populated with a correct url.

        Arguments:
        uri (str) : a repository uri string to override one currently set
        require_repo (bool) : require a repository to parse.
        """
        from rse.main.parsers import get_parser
        from rse.utils.urls import repository_regex

        repository_regex = repository_regex.rstrip("$")

        if uri:
            self.set_uri(uri)
        self.load_secrets()

        # Get the record number from the doi
        record = self.uid.split("/")[-1].replace("zenodo.", "")

        # Token isn't required for public entries
        if self.token:
            response = requests.get(
                "https://zenodo.org/api/records/%s" % record,
                json={"access_token": self.token},
            )
        else:
            response = requests.get("https://zenodo.org/api/records/%s" % record)

        # Successful query!
        if response.status_code == 200:
            self.data = response.json()

            # For Zenodo, we require a GitHub or GitLab related identifier to add
            repo_url = None
            for identifier in self.data["metadata"].get("related_identifiers", []):
                match = re.search(repository_regex, identifier["identifier"])
                if match:
                    repo_url = "https://%s" % match.group()
                    break

            # If we return None, the entry is not added
            if repo_url is None and require_repo is True:
                bot.warning(
                    "Repository url not found with Zenodo record, skipping add."
                )
                return repo_url

            # Convert the class into another parser type
            elif repo_url is not None:
                uid = self.uid
                self = get_parser(repo_url)
                self.get_metadata()
                self.data["doi"] = uid
                return self
            return self.data

        elif response.status_code == 404:
            bot.error(f"Cannot find doi {self.uid}.")

        elif response.status_code in [400, 401, 403]:
            bot.error(f"Permission denied to query {self.uid}")

        else:
            bot.error(
                f"Cannot get doi {self.uid}: {response.status_code}, {response.reason}"
            )
        return None


from sklearn.metrics import mean_squared_error,accuracy_score,auc,f1_score
import torch
import numpy as np

def rmse(y_true,y_pred):
    if isinstance(y_true,torch.Tensor):
        return ((y_pred-ytrue)**2).mean().pow(0.5).item()
    elif isinstance(y_true,np.ndarray):
        return mean_squared_error(y_true,y_pred,squared=False)

def accuracy(y_true,y_pred,reduction=True):
    if isinstance(y_true,torch.Tensor):
        y_pred = torch.argmax(y_true,dim=-1)
        if reduction:
            return y_pred.eq(y_true.view_as(preds)).mean().item()
        else:
            return y_pred.eq(y_true.view_as(preds)).sum().item()

    elif isinstance(y_true,np.ndarray):
        y_pred = np.argmax(y_pred,axis=-1)
        if reduction:
            return accuracy_score(y_true,y_pred)
        else:
            return accuracy_score(y_true,y_pred,normalize=False)

class f1(object):
    def __init__(self,average='macro'):
        self.average=average
    def __call__(self,y_true,y_pred):
        if isinstance(y_true,torch.Tensor):
            preds = y_pred.cpu().detach().numpy()
            truths = y_true.cpu().detach().numpy()

        elif isinstance(y_true,list):
            preds = np.array(y_preds)
            truths = np.array(y_true)
        else:
            preds = y_pred
            truths = y_true
        preds = np.argmax(preds,axis=1)
        return f1_score(truths,preds,average = self.average)
    @property
    def __name__(self):
        return self.__class__.__name__

_metrics = {'accuracy':accuracy,'f1_score':f1,'rmse':rmse}

def get_metric(name=''):

    if name not in _metrics:
        raise ValueError('Metric should be either accuracy,f1_score and rmse')
    return _metrics[name]


import csv
import requests
import random
import json
import pymysql
import time
from lxml import etree
allResult = []
# 从json文件当中获取当前的数据条数以及总数据
# with open('小说.json','r',encoding='utf-8') as file:
# with open('小说test.json','r',encoding='utf-8') as file:
#     # jsonDic = json.load(file)
#     count = jsonDic['count']
#     allResult = jsonDic['data']
db = pymysql.connect('localhost', 'root', 'root', 'python', 8889)
cursor = db.cursor()
count = 0
for i in range(1, 600):
    url = "https://api.hongdouyuedu.com/honor/category/info?category=%E5%A5%B3%E9%A2%91%3B%E7%8E%B0%E4%BB%A3%E8%A8%80%E6%83%85%3B%E6%80%BB%E8%A3%81%E8%B1%AA%E9%97%A8&channel=oppo&device_name=PDPM00&identifier=&physic_size=6.56&platform=wwa1&plattype=aphone&scale_screen=2400_1080&ts=1598649593222&uuid=FF91EABB42C32DD1CBC71BCE6A528208&version=2.5.9&virtual=0&sig=b221f5e56806176a35b9d433da5014bc"
    parameter = {"page_index": i}
    headers = {
        "User-Agent": "okhttp/3.12.1",
        "Host": "api.hongdouyuedu.com",
        "Connection": "Keep-Alive",
        "Accept-Encoding": "identity",
    }
    response = requests.get(url, headers=headers, params=parameter)
    print(response.text)
    content = response.json()["data"]["data"]
    for book in content:
        print()
        print("开始爬取下一本小说")
        book_name = book["name"]
        print(book_name)
        author_name = book["author_name"]
        print(author_name)
        tags = book["label_name"]
        print(tags)
        categories = book["category_name"]
        print(categories)
        intro = book["described"]
        print(intro)
        words_num = book["words_total"]
        popularity = book["views_total"]
        print(book_name + " 小说爬取完毕")

        # 分享链接
        # share_url = secondary_content["data"]["share_url"]
        share_url = "空"

        dic = {
            '书名': book_name,
            '作者名': author_name,
            '标签': tags,
            '可选分类': categories,
            '简介': intro,
            '字数': words_num,
            '人气数': popularity,
            '分享链接': share_url
        }

        count += 1
        allResult.append(dic)
        # 写入json文件，"红豆小说1000本总裁文.json"
        data = {'data': allResult, 'count': count}
        with open('小说.json', 'w', encoding='utf-8') as file:
            # 直接存入json
            json.dump(data, file, ensure_ascii=False, indent=4)

        count += 1
        # 写入csv文件，"红豆小说1000本总裁文.csv"
        header = ['书名', '作者名', '标签', '可选分类', '简介', '字数', '人气数', '分享链接']
        with open('小说.csv', 'a', encoding='utf-8-sig', newline='') as file:
            huo = csv.DictWriter(file, header)
            if count == 1:
                huo.writeheader()
            huo.writerow(dic)

        # 写入数据库表
        sql = "INSERT INTO xiaoshuo (book_name, author_name, tags, categories, intro, words_num, popularity, share_url) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s');" % (
            book_name, author_name, tags, categories, intro, words_num, popularity, share_url)
        cursor.execute(sql)
        # 数据库提交操作
        db.commit()

# 从json文件当中获取当前的数据条数以及总数据
# with open('小说.json','r',encoding='utf-8') as file:
# with open('小说test.json','r',encoding='utf-8') as file:
#     # jsonDic = json.load(file)
#     count = jsonDic['count']
#     allResult = jsonDic['data']

# 关闭数据库
db.close()


import itertools

list = ['0001','0011','0101','1011','1101','0111']

new_list = [int(binary , 2) for binary in list]

new_list.sort()

print(new_list)

sum= sum(new_list)

print(sum)

combination = itertools.combinations(new_list,2)

last = None

for combo in combination:
    last = combo
    print(combo)

if last != None:
    print(last[0])
    print(last[1])

# jo minus krane wala concpt hai vo karna hai like 2 array lekar ek mai sab add karke ek ek elemnt se subtract karna hai and phir itertools lagana hai


#!/usr/bin/env python3
# script fetches FMI Open data and imports it to DB
from tendo import singleton
me = singleton.SingleInstance() # will sys.exit(-1) if other instance is running

from db_setup import *
from cleaner import *
import configparser
from configurator import *
import os.path

tiff_dir_base = config.get("locations","wms_data_dir")
api_key = config.get( "settings","fmi_api_key")

# set logger
logger = set_logger( "fmi_open" )

from datetime import datetime,timedelta
import os
from urllib.request import urlopen,urlparse

from xml.etree import ElementTree
gml_namespace = "http://www.opengis.net/gml/3.2"

# sections in config file must match with layer names
wfs_layers = {
    'fmi_radar_single_dbz':
    'http://opendata.fmi.fi/wfs?request=GetFeature&storedquery_id=fmi::radar::single::dbz',
    'fmi_radar_single_vrad':
    'http://opendata.fmi.fi/wfs?request=GetFeature&storedquery_id=fmi::radar::single::vrad',
    'fmi_radar_single_etop_20':
    'http://opendata.fmi.fi/wfs?request=GetFeature&storedquery_id=fmi::radar::single::etop_20',
    'fmi_radar_single_hclass':
    'http://opendata.fmi.fi/wfs?request=GetFeature&storedquery_id=fmi::radar::single::hclass'

}

config_dataset_names,config_sections = get_sections_from_config()

def update():
    add_datasets = []
    return_datasets = []
    logger.debug( "Start updating of DB" )
    for layer in wfs_layers:
        if not layer in config_dataset_names:
            continue
        else:
            logger.debug( "Found layer %s from config" % layer )
            section = config_sections[config_dataset_names.index(layer)]
        # check that tiff directory exists
        tiff_dir = "%s/%s" % (tiff_dir_base,layer)
        if not os.path.exists(tiff_dir):
            logger.debug( "GeoTIFF directory does not exist. Create it." )
            os.makedirs(tiff_dir)
        # get WFS to get WMS urls
        try:
            wfs_url = wfs_layers[layer].replace("{key}",api_key)
            response = urlopen( wfs_url )
            logger.debug( "Data from url %s fetched" % wfs_url )
        # ignore network related problems
        except:
            logger.error( "Network error occurred while fetching url %s" % wfs_url )
            continue
        logger.debug( "Parse WFS response for layer %s" % layer )
        wfs_response = ElementTree.fromstring(response.read())
        file_references = wfs_response.findall('.//{%s}fileReference' % gml_namespace)
        for ref in file_references:
            url = ref.text
            query = urlparse(url).query
            query = query.split("&")
            for q in query:
                if q[0:4].lower()=="time":
                    time_value = q.split("=")[-1]
                elif q[0:3].lower()=="srs":
                    projdef = q.split("=")[-1]
                elif q[0:4].lower()=="bbox":
                    bbox = q.split("=")[-1]
                elif q[0:6].lower()=="layers":
                    radar_name = q.split("=")[-1]
                elif q[0:9].lower()=="elevation":
                    radar_elevation = q.split("=")[-1]
            timestamp = datetime.strptime(time_value,\
                    "%Y-%m-%dT%H:%M:%SZ")
            # search if dataset already exists
            if (radar_name.split("_")[0] != "Radar:kesalahti") :
                continue
            if (layer!='fmi_radar_single_etop_20' and float(radar_elevation) > 0.5) :
                continue
            output_filename = tiff_dir + "/" + radar_name.replace("Radar:","")+ time_value.replace(":","")\
                .replace("-","") + "_"+radar_elevation+ ".tif"
            # skip file fetching if it already exists
            if os.path.isfile(output_filename):
                continue
            # save file to disk
            try:
                response = urlopen( url )
                logger.debug( "Fetched data from url %s" % url )
            # ignore network related problems
            except:
                logger.error( "Network error or invalid api-key occurred while fetching url %s" % url )
                continue
            f = open(output_filename,"wb")
            f.write( response.read() )
            f.close()
            # import dataset to db
            logger.info( "Add new dataset: %s:%s to DB." % (layer,str(timestamp)) )
            add_datasets.append(
                RadarDataset(
                    name = layer,
                    timestamp = timestamp,
                    geotiff_path = output_filename,
                    projdef = projdef,
                    bbox_original = bbox,
                    dataset_type = config.get(section,"dataset_type"),
                    unit = config.get(section,"unit"),
                    style = config.get(section,"style"),
                    title = config.get(section,"title")
                )
            )
            return_datasets.append({"name": layer,
                                    "timestamp": timestamp })
    # finally add new datasets to DB
    if (len(add_datasets)>0):
        session.add_all(add_datasets)
        session.commit()
    logger.info ( "Added %i results." % len(add_datasets) )
    logger.debug( "Updating of DB finished" )
    session.close()
    return return_datasets

if __name__ == '__main__':
    update()
    # delete old
    for layer in wfs_layers:
        if not layer in config_dataset_names:
            continue
        else:
            section = config_sections[config_dataset_names.index(layer)]
            clean_up(config.get(section,"name"),
                     int( config.get(section,"cleanup_time") ))


# -*- coding:utf8 -*-

from __future__ import print_function
import os
import numpy as np
import pandas as pd
import codecs
from keras.datasets import imdb
from config import Config

class Data_Loader(object):

    def __init__(self):
        print('Loading data...')

    def load_from_file(self, config, mode='train'):

        if mode == 'train':
            path = config.train_path
        else:
            path = config.test_path

        neg_path = path + 'neg/'
        pos_path = path + 'pos/'

        neg_list = os.listdir(neg_path)
        pos_list = os.listdir(pos_path)

        data = []
        label = []

        print("Load neg...")
        for neg in neg_list:
            with codecs.open(neg_path + neg) as fp:
                text = fp.read()
                data.append(text.strip())
                label.append(0)

        print("neg data is", len(data))
        # print("example of neg:\n", data[0])

        print("Loading pos...")
        for pos in pos_list:
            with codecs.open(pos_path + pos) as fp:
                text = fp.read()
                data.append(text.strip())
                label.append(1)

        print("all data is", len(data))
        # print("example of pos:\n", data[-1])

        return data, label

    def load(self, config):
        """Load data

        :param config:
        :return: the data of imdb
        """
        (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=config.max_features)
        print(len(X_train), 'train sequences')
        print(len(X_test), 'test sequences')
        print('x_train set\' shape', np.shape(X_train))
        print('y_train set\' shape', np.shape(y_train))
        print('x_test set\' shape', np.shape(X_test))
        print('y_test set\' shape', np.shape(y_test))

        return X_train, y_train, X_test, y_test

    def stat(self):
        """Statistic data

        :return: None
        """
        print('statistic...')
        config = Config()
        X_train, y_train, X_test, y_test = self.load(config)
        len_num = []
        for X in X_train+X_test:
            len_num.append(len(X))
        print('max len num is', max(len_num))
        len_num = pd.Series(len_num)
        print('len num count is\n', len_num.value_counts())

    def show_example(self):
        print('show example...')
        config = Config(word_vocb_path='./data/imdb.vocab')
        index_word = config.get_index_word()
        X_train, _, _, _ = self.load(config)
        print(X_train[0])
        new_text = []
        for index in X_train[0]:
            new_text.append(index_word[index].strip())
        print(' '.join(new_text))

    def test(self):
        config = Config()
        X_train, y_train, X_test, y_test = self.load(config)
        print(X_train[0])
        print(y_train[0])

if __name__ == '__main__':
    data_loader = Data_Loader()
    data_loader.show_example()
    # data_loader.stat()


#import PIL.Image
import dlib
import numpy as np
import cv2
from .faceID_models import faceID_models
import itertools
import sys
import os
import re
import multiprocessing
import shutil

face_detector = dlib.get_frontal_face_detector()

predictor_68_point_model = faceID_models.pose_predictor_model_location()
pose_predictor_68_point = dlib.shape_predictor(predictor_68_point_model)

predictor_5_point_model = faceID_models.pose_predictor_five_point_model_location()
pose_predictor_5_point = dlib.shape_predictor(predictor_5_point_model)

cnn_face_detection_model = faceID_models.cnn_face_detector_model_location()
cnn_face_detector = dlib.cnn_face_detection_model_v1(cnn_face_detection_model)

face_recognition_model = faceID_models.face_recognition_model_location()
face_encoder = dlib.face_recognition_model_v1(face_recognition_model)

def load_image_file(file):
    """
    Loads an image file (.jpg, .png, etc) into a numpy array using opencv

    :param file: image file name or file object to load
    :return: image contents as numpy array (RGB order)
    """

    return cv2.cvtColor(np.array(cv2.imread(file)), cv2.COLOR_BGR2RGB)

def imwrite(file_name,image):
    """
    Write numpy array into disk

    :param file: image file name or file object to write
    :return: None
    """

    cv2.imwrite(file_name,cv2.cvtColor(image,cv2.COLOR_RGB2BGR))

def face_distance(face_encodings, face_to_compare):
    """
    Given a list of face encodings, compare them to a known face encoding and get a euclidean distance
    for each comparison face. The distance tells you how similar the faces are.

    :param faces: List of face encodings to compare
    :param face_to_compare: A face encoding to compare against
    :return: A numpy ndarray with the distance for each face in the same order as the 'faces' array
    """
    if len(face_encodings) == 0:
        return np.empty((0))

    return np.linalg.norm(face_encodings - face_to_compare, axis=1)

def show_image_cv(screen_name,image,wait_time=0):
    '''
    Show out image

    :param screen_name: Screen name of the showing window
    :param image: Numpy array of image
    :param wait_time: Delay time of the screen
    '''
    cv2.imshow(screen_name,cv2.cvtColor(image,cv2.COLOR_RGB2BGR))
    cv2.waitKey(wait_time)

def _rect_to_css(rect):
    """
    Convert a dlib 'rect' object to a plain tuple in (top, right, bottom, left) order

    :param rect: a dlib 'rect' object
    :return: a plain tuple representation of the rect in (top, right, bottom, left) order
    """
    return rect.top(), rect.right(), rect.bottom(), rect.left()

def _css_to_rect(css):
    """
    Convert a tuple in (top, right, bottom, left) order to a dlib `rect` object

    :param css:  plain tuple representation of the rect in (top, right, bottom, left) order
    :return: a dlib `rect` object
    """
    return dlib.rectangle(css[3], css[0], css[1], css[2])

def _trim_css_to_bounds(css, image_shape):
    """
    Make sure a tuple in (top, right, bottom, left) order is within the bounds of the image.

    :param css:  plain tuple representation of the rect in (top, right, bottom, left) order
    :param image_shape: numpy shape of the image array
    :return: a trimmed plain tuple representation of the rect in (top, right, bottom, left) order
    """
    return max(css[0], 0), min(css[1], image_shape[1]), min(css[2], image_shape[0]), max(css[3], 0)

def _raw_face_locations(img, number_of_times_to_upsample=1, model="hog"):
    """
    Returns an array of bounding boxes of human faces in a image

    :param img: An image (as a numpy array)
    :param number_of_times_to_upsample: How many times to upsample the image looking for faces. Higher numbers find smaller faces.
    :param model: Which face detection model to use. "hog" is less accurate but faster on CPUs. "cnn" is a more accurate
                  deep-learning model which is GPU/CUDA accelerated (if available). The default is "hog".
    :return: A list of dlib 'rect' objects of found face locations
    """
    if model == "cnn":
        return cnn_face_detector(img, number_of_times_to_upsample)
    else:
        return face_detector(img, number_of_times_to_upsample)

def face_locations(img, number_of_times_to_upsample=1, model="hog"):
    """
    Returns an array of bounding boxes of human faces in a image

    :param img: An image (as a numpy array)
    :param number_of_times_to_upsample: How many times to upsample the image looking for faces. Higher numbers find smaller faces.
    :param model: Which face detection model to use. "hog" is less accurate but faster on CPUs. "cnn" is a more accurate
                  deep-learning model which is GPU/CUDA accelerated (if available). The default is "hog".
    :return: A list of tuples of found face locations in css (top, right, bottom, left) order
    """
    if model == "cnn":
        return [_trim_css_to_bounds(_rect_to_css(face.rect), img.shape) for face in _raw_face_locations(img, number_of_times_to_upsample, "cnn")]
    else:
        return [_trim_css_to_bounds(_rect_to_css(face), img.shape) for face in _raw_face_locations(img, number_of_times_to_upsample, model)]

def _raw_face_locations_batched(images, number_of_times_to_upsample=1, batch_size=128):
    """
    Returns an 2d array of dlib rects of human faces in a image using the cnn face detector

    :param img: A list of images (each as a numpy array)
    :param number_of_times_to_upsample: How many times to upsample the image looking for faces. Higher numbers find smaller faces.
    :return: A list of dlib 'rect' objects of found face locations
    """
    return cnn_face_detector(images, number_of_times_to_upsample, batch_size=batch_size)

def batch_face_locations(images, number_of_times_to_upsample=1, batch_size=128):
    """
    Returns an 2d array of bounding boxes of human faces in a image using the cnn face detector
    If you are using a GPU, this can give you much faster results since the GPU
    can process batches of images at once. If you aren't using a GPU, you don't need this function.

    :param img: A list of images (each as a numpy array)
    :param number_of_times_to_upsample: How many times to upsample the image looking for faces. Higher numbers find smaller faces.
    :param batch_size: How many images to include in each GPU processing batch.
    :return: A list of tuples of found face locations in css (top, right, bottom, left) order
    """
    def convert_cnn_detections_to_css(detections):
        return [_trim_css_to_bounds(_rect_to_css(face.rect), images[0].shape) for face in detections]

    raw_detections_batched = _raw_face_locations_batched(images, number_of_times_to_upsample, batch_size)

    return list(map(convert_cnn_detections_to_css, raw_detections_batched))

def _raw_face_landmarks(face_image, face_locations=None, model="large"):
    if face_locations is None:
        face_locations = _raw_face_locations(face_image)
    else:
        face_locations = [_css_to_rect(face_location) for face_location in face_locations]

    pose_predictor = pose_predictor_68_point

    if model == "small":
        pose_predictor = pose_predictor_5_point

    return [pose_predictor(face_image, face_location) for face_location in face_locations]

def face_encodings(face_image, known_face_locations=None, num_jitters=1):
    """
    Given an image, return the 128-dimension face encoding for each face in the image.

    :param face_image: The image that contains one or more faces
    :param known_face_locations: Optional - the bounding boxes of each face if you already know them.
    :param num_jitters: How many times to re-sample the face when calculating encoding. Higher is more accurate, but slower (i.e. 100 is 100x slower)
    :return: A list of 128-dimensional face encodings (one for each face in the image)
    """
    raw_landmarks = _raw_face_landmarks(face_image, known_face_locations, model="large")
    return [np.array(face_encoder.compute_face_descriptor(face_image, raw_landmark_set, num_jitters)) for raw_landmark_set in raw_landmarks]

def compare_faces(known_face_encodings, face_encoding_to_check, tolerance=0.6):
    """
    Compare a list of face encodings against a candidate encoding to see if they match.

    :param known_face_encodings: A list of known face encodings
    :param face_encoding_to_check: A single face encoding to compare against the list
    :param tolerance: How much distance between faces to consider it a match. Lower is more strict. 0.6 is typical best performance.
    :return: A list of True/False values indicating which known_face_encodings match the face encoding to check
    """

    return list(face_distance(known_face_encodings, face_encoding_to_check) <= tolerance)

#######################################################################################################################################
def image_files_in_folder(folder):
    return [os.path.join(folder, f) for f in os.listdir(folder) if re.match(r'.*\.(jpg|jpeg|png|JPG)', f, flags=re.I)]

def test_image(target_encodings,image_to_check, model,output_folder):
    '''
    Given an unknown image, compare the face appear in it with sample encodings

    :param target_encodings: encodings extracted from sample image
    :param image_to_check: image path that needs to be check
    :param model: face detection model, either 'hog' or 'cnn'
    :param output_folder: output folder of images that has target_encodings in

    '''
    unknown_image = load_image_file(image_to_check)

    unknown_image_locations = face_locations(unknown_image,model=model)
    test_encodings = face_encodings(unknown_image,unknown_image_locations)

    for idx,test_encoding in enumerate(test_encodings):
        compare_results = compare_faces(target_encodings,test_encoding,0.6)
        print (compare_results)
        if any(compare_results):
            shutil.copyfile(image_to_check, os.path.join(output_folder,os.path.basename(image_to_check)))

def process_images_in_process_pool(target_encodings,images_to_check, number_of_cpus, model,output_folder):
    if number_of_cpus == -1:
        processes = None
    else:
        processes = number_of_cpus

    # macOS will crash due to a bug in libdispatch if you don't use 'forkserver'
    context = multiprocessing
    if "forkserver" in multiprocessing.get_all_start_methods():
        context = multiprocessing.get_context("forkserver")

    pool = context.Pool(processes=processes)

    function_parameters = zip(
        itertools.repeat(target_encodings),
        images_to_check,
        itertools.repeat(model),
        itertools.repeat(output_folder)
    )

    pool.starmap(test_image, function_parameters)

def find_ID(sample,input,output_folder,cpus, model):
    '''
    find the images that has at lease one ID that appear in sample image

    :param sample: sample image path
    :param input: folder contains testing images
    :param cpus: number of cpus that you want to use
    :param model: face detection model, either 'hog' or 'cnn'

    '''
    # Multi-core processing only supported on Python 3.4 or greater
    if (sys.version_info < (3, 4)) and cpus != 1:
        click.echo("WARNING: Multi-processing support requires Python 3.4 or greater. Falling back to single-threaded processing!")
        cpus = 1

    if not (os.path.exists(input) or os.path.exists(sample)):
        click.echo("ERROR: " + input + " is not found")
        sys.exit(1)

    if not os.path.exists(output_folder):
        os.mkdir(output_folder)

    try:
        image_sample = load_image_file(sample)
        target_locations = face_locations(image_sample,model=model)
        target_encodings = face_encodings(image_sample,target_locations)
        print (target_encodings)
        print(len(target_encodings)," faces are found in sample image")

    except:
        print("Cannot encode from the sample image")
        quit()

    if os.path.isdir(input):
        if cpus == 1:
            [test_image(target_encodings,image_file, model,output_folder) for image_file in image_files_in_folder(input)]
        else:
            process_images_in_process_pool(target_encodings,image_files_in_folder(input), cpus, model,output_folder)
    else:
        test_image(target_encodings,image_to_check, model,output_folder)


import glob
import re
import jieba
from collections import defaultdict

jieba.enable_paddle()
jieba.initialize()

def convert_to_halfwidth(line):
    line = line.translate(str.maketrans("".join([chr(i) for i in range(ord('！'), ord('～') + 1)]), "".join([chr(i) for i in range(ord('!'), ord('~') + 1)]))).strip()
    line = line.translate(str.maketrans("《》、。“”", "(),.\"\"")).strip()
    return line

ch_radio_fnames = glob.glob("../datasets/man_ch_news_txt/ch_radio/cr??????", recursive=True)
p_daily_fnames = glob.glob("../datasets/man_ch_news_txt/p_daily/pd????", recursive=True)
xinhua_fnames = glob.glob("../datasets/man_ch_news_txt/xinhua/xh????_?", recursive=True)

with open("../extracted/raw_ch_radio_sents.txt", "w") as ch_radio_sents_f:
    for ch_radio_fname in ch_radio_fnames:
        with open(ch_radio_fname, encoding='gb18030') as ch_radio_f:
            while True:
                line = ch_radio_f.readline()
                if len(line) == 0:
                    break
                if line[:4] == "<HL>" and line[-5:] == "</HL>":
                    line = line[4:-5]
                    line = convert_to_halfwidth(line.strip())
                    line = " ".join(jieba.cut(line, use_paddle=True))
                    ch_radio_sents_f.write(line + '\n')
                    # tokens = jieba.cut(line[4:-5], use_paddle=True)
                    # for token in tokens:
                    #     w2i[token] += 1
                elif line.strip() == "<TEXT>":
                    sents = []
                    while True:
                        line = ch_radio_f.readline().strip()
                        if line.strip() == "</TEXT>":
                            break
                        if line != "<p>":
                            print("unexpected line that is not <p>")
                            print(line)
                            input("Press enter")
                        line = ch_radio_f.readline().strip()
                        for line_1 in re.split(r'[!?.]', line):
                            line_1 = convert_to_halfwidth(line_1.strip())
                            sents.append(line_1)

                    sents.pop()
                    for sent in sents:
                        sent = sent.translate(str.maketrans('', '', '#"\'%-`~()[]<>.,·¡™=§*?_/\\@&—;:'))
                        sent = " ".join(jieba.cut(sent, use_paddle=True))
                        if len(sent.strip()) == 0:
                            continue
                        ch_radio_sents_f.write(sent + '\n')
        print(f'Done with {ch_radio_fname}')

with open("../extracted/raw_p_daily_sents.txt", "w") as p_daily_sents_f:
    for p_daily_fname in p_daily_fnames:
        print(p_daily_fname)
        with open(p_daily_fname, encoding='gb18030') as p_daily_f:
            while True:
                line = p_daily_f.readline()
                if len(line) == 0:
                    break
                if line[:4] == "<HL>" and line[-5:] == "</HL>":
                    line = line[4:-5]
                elif line[0] == '<':
                    continue
                for line_1 in re.split(r'[.!?]', line):
                    line_1 = convert_to_halfwidth(line_1.strip())
                    line_1 = line_1.translate(str.maketrans('', '', '#"\'%-`~()[]<>.,·¡™=§*?_/\\@&—;:'))
                    if len(line_1.strip()) == 0:
                        continue
                    line_1 = " ".join(jieba.cut(line_1, use_paddle=True))
                    if len(line_1.strip()) == 0:
                        continue
                    p_daily_sents_f.write(line_1 + '\n')
        print(f'Done with {p_daily_fname}')

with open("../extracted/raw_xinhua_sents.txt", "w") as xinhua_sents_f:
    for xinhua_fname in xinhua_fnames:
        with open(xinhua_fname, encoding='gb18030') as xinhua_f:
            while True:
                line = xinhua_f.readline()
                if len(line) == 0:
                    break
                line = convert_to_halfwidth(line.strip())
                if line[:10] == "<headline>" and line[-11:] == "</headline>":
                    line = convert_to_halfwidth(line[10:-11].strip())
                elif line[:3] == "<s>" and line[-4:] == "</s>":
                    line = convert_to_halfwidth(line[3:-4].strip())
                else:
                    continue
                for line_1 in re.split(r'[.!?]', line):
                    line_1 = convert_to_halfwidth(line_1.strip())
                    if len(line_1.strip()) == 0:
                        continue
                    line_1 = line_1.translate(str.maketrans('', '', '#"\'%-`~()[]<>.,·¡™=§*?_/\\@&—;:'))
                    line_1 = " ".join(jieba.cut(line_1, use_paddle=True))
                    if line_1 == "完":
                        continue
                    if len(line_1.strip()) == 0:
                        continue
                    xinhua_sents_f.write(line_1 + '\n')
        print(f'Done with {xinhua_fname}')


from typing import Union, List

from simpleperf_utils import AdbHelper

def get_logcat_for_aut(aut: str, adb: AdbHelper) -> Union[str, None]:
    status, pid = adb.run_and_return_output(['shell', 'pidof', aut])
    if not status:
        print("failed to get pid")
        return None
    print(pid)
    status, logcat_str = adb.run_and_return_output(
        ['shell', 'logcat', '-d', '--pid=' + str(pid)], True, True)
    if not status:
        print("something went wrong with logcat!")
        return None
    return logcat_str

def filter_logcat_str_for_tags(logcat: List[str], tags: Union[str, List[str]]) -> List[str]:
    if isinstance(tags, str):
        def filter_fun(line):
            return tags in line
    elif isinstance(tags, List):
        def filter_fun(line):
            return any(tag in line for tag in tags)
    else:
        raise TypeError(f'Unknown type for tags: {type(tags)}')

    return list(filter(filter_fun, logcat))

def split_logcat(logcat: str, latest_first: bool = True) -> List[str]:
    log_list = logcat.split('\n')
    if latest_first:
        log_list.reverse()
    return log_list


import tkinter as tk
import tkinter.font as tkFont
from time import strftime
import datetime as dd

class App:
    def __init__(self, root):
        #setting title
        root.title("clock")
        #setting window size
        width=1000
        height=580
        screenwidth = root.winfo_screenwidth()
        screenheight = root.winfo_screenheight()
        alignstr = '%dx%d+%d+%d' % (width, height, (screenwidth - width) / 2, (screenheight - height) / 2)
        root.geometry(alignstr)
        root.resizable(width=False, height=False)

        def time():
            string = strftime('%H:%M:%S %p')
            lbl.config(text=string)
            lbl.after(1000, time)

        date = dd.datetime.now()

        GLabel_70=tk.Label(root)
        GLabel_70["activeforeground"] = "#542121"
        #GLabel_70["anchor"] = "nw"
        ft = tkFont.Font(family='Times',size=15)
        GLabel_70["font"] = ft
        GLabel_70["fg"] = "#333333"
        GLabel_70["justify"] = "center"
        GLabel_70["text"] = "Current Time and Date"
        GLabel_70.place(x=80,y=20)

        lbl = tk.Label(root, font=('calibri', 40, 'bold'),background='purple',foreground='white')

        lbl.place(x=80,y=60)
        time()
        lbl1 = tk.Label(root, text=f"{date:%A, %B %d, %Y}",font=('calibri', 20, 'bold'),background='black',foreground='white')

        lbl1.place(x=80,y=120)

        GLabel_70=tk.Label(root)
        GLabel_70["activeforeground"] = "#542121"
        #GLabel_70["anchor"] = "nw"
        ft = tkFont.Font(family='Times',size=15)
        GLabel_70["font"] = ft
        GLabel_70["fg"] = "#333333"
        GLabel_70["justify"] = "center"
        GLabel_70["text"] = "Previos Date"
        GLabel_70.place(x=80,y=180)

        GLabel_70=tk.Label(root)
        GLabel_70["activeforeground"] = "#542121"
        #GLabel_70["anchor"] = "nw"
        ft = tkFont.Font(family='Times',size=15)
        GLabel_70["font"] = ft
        GLabel_70["fg"] = "#333333"
        GLabel_70["justify"] = "center"
        GLabel_70["text"] = "Office Opening Time"
        GLabel_70.place(x=700,y=20)

        lbl2 = tk.Label(root, text="09:15 AM",font=('calibri', 20, 'bold'),background='black',foreground='white')

        lbl2.place(x=700,y=80)

        GLabel_70=tk.Label(root)
        GLabel_70["activeforeground"] = "#542121"
        #GLabel_70["anchor"] = "nw"
        ft = tkFont.Font(family='Times',size=15)
        GLabel_70["font"] = ft
        GLabel_70["fg"] = "#333333"
        GLabel_70["justify"] = "center"
        GLabel_70["text"] = "Office Closing Time"
        GLabel_70.place(x=700,y=150)

        lbl3 = tk.Label(root, text="15:30 PM",font=('calibri', 20, 'bold'),background='black',foreground='white')

        lbl3.place(x=700,y=210)

        GButton_235=tk.Button(root)
        GButton_235["bg"] = "red"
        ft = tkFont.Font(family='Times',size=10)
        GButton_235["font"] = ft
        GButton_235["fg"] = "#000000"
        GButton_235["justify"] = "center"
        GButton_235["text"] = "Quit"
        GButton_235.place(x=80,y=550,width=70,height=25)
        GButton_235["command"] = self.GButton_235_command

        GButton_125=tk.Button(root)
        GButton_125["bg"] = "Green"
        ft = tkFont.Font(family='Times',size=10)
        GButton_125["font"] = ft
        GButton_125["fg"] = "#000000"
        GButton_125["justify"] = "center"
        GButton_125["text"] = "Last Date"
        GButton_125.place(x=700,y=550,width=70,height=25)
        GButton_125["command"] = self.GButton_125_command

        #root.Listbox = Listbox(width=50, height=10, highlightthickness=0, font=('Andalus', 12, 'bold'),
        #                     selectmode=SINGLE, bg='#044f19', fg='#65eb8a', selectbackground='#aed6b9',
         #                    selectforeground='#68786d', selectborderwidth=3, activestyle=NONE)
        #root.Listbox.place(x=460,y=450,width=111,height=30)
        #root.Listbox.grid(row=0, column=1, padx=(20, 0), pady=(20, 0), sticky="nsew")

    def GButton_235_command(self):
        quit()

    def GButton_125_command(self):
        date = dd.datetime.now()
        import datetime as dt
        date = dt.datetime.now()
        a=f"{date:%A, %B %d, %Y}"
        print(a)
        print(date)
        Previous_Date = date - dt.timedelta(days=1)
        print (Previous_Date)

        lbl4 = tk.Label(root, text=Previous_Date,font=('calibri', 20, 'bold'),background='black',foreground='white')

        lbl4.place(x=80,y=210)

if __name__ == "__main__":
    root = tk.Tk()
    app = App(root)
    root.mainloop()


#Coded by Sumanjay on 29th Feb 2020
from flask import Flask, request, jsonify
from inshorts import getNews
from flask_cors import CORS

app = Flask(__name__)
app.secret_key = "i_am_not_feeling_sleepy_so_i_am_coding_this"
CORS(app)

@app.route('/')
def home():
    return 'News API is UP!

A part of Sj Projects'

@app.route('/news')
def news():
    if request.method == 'GET':
        category = request.args.get("category")
        if not category:
            return jsonify({
                "error": "please add category in query params"
            }), 404
        return jsonify(getNews(category)), 200

if __name__ == '__main__':
    app.debug = True
    app.run(host='0.0.0.0',port=5000,use_reloader=True)


#!/usr/bin/env python
#-*- coding: utf-8 -*-
#pylint: disable=C0301,C0103,R0914,R0903

"""
WMArchive command line tool
"""
from __future__ import print_function
__author__ = "Valentin Kuznetsov"

# system modules
import os
import sys
import pwd
if  sys.version_info < (2, 6):
    raise Exception("WMArchive requires python 2.6 or greater")

WMArchive_CLIENT = 'wma-client/1.1::python/%s.%s' % sys.version_info[:2]

import re
import time
import json
import urllib
import urllib2
import httplib
import cookielib
from   optparse import OptionParser

# define exit codes according to Linux sysexists.h
EX_OK           = 0  # successful termination
EX__BASE        = 64 # base value for error messages
EX_USAGE        = 64 # command line usage error
EX_DATAERR      = 65 # data format error
EX_NOINPUT      = 66 # cannot open input
EX_NOUSER       = 67 # addressee unknown
EX_NOHOST       = 68 # host name unknown
EX_UNAVAILABLE  = 69 # service unavailable
EX_SOFTWARE     = 70 # internal software error
EX_OSERR        = 71 # system error (e.g., can't fork)
EX_OSFILE       = 72 # critical OS file missing
EX_CANTCREAT    = 73 # can't create (user) output file
EX_IOERR        = 74 # input/output error
EX_TEMPFAIL     = 75 # temp failure; user is invited to retry
EX_PROTOCOL     = 76 # remote error in protocol
EX_NOPERM       = 77 # permission denied
EX_CONFIG       = 78 # configuration error

class HTTPSClientAuthHandler(urllib2.HTTPSHandler):
    """
    Simple HTTPS client authentication class based on provided
    key/ca information
    """
    def __init__(self, key=None, cert=None, level=0):
        if  level > 1:
            urllib2.HTTPSHandler.__init__(self, debuglevel=1)
        else:
            urllib2.HTTPSHandler.__init__(self)
        self.key = key
        self.cert = cert

    def https_open(self, req):
        """Open request method"""
        #Rather than pass in a reference to a connection class, we pass in
        # a reference to a function which, for all intents and purposes,
        # will behave as a constructor
        return self.do_open(self.get_connection, req)

    def get_connection(self, host, timeout=300):
        """Connection method"""
        if  self.key:
            return httplib.HTTPSConnection(host, key_file=self.key,
                                                cert_file=self.cert)
        return httplib.HTTPSConnection(host)

def x509():
    "Helper function to get x509 either from env or tmp file"
    proxy = os.environ.get('X509_USER_PROXY', '')
    if  not proxy:
        proxy = '/tmp/x509up_u%s' % pwd.getpwuid( os.getuid() ).pw_uid
        if  not os.path.isfile(proxy):
            return ''
    return proxy

def check_glidein():
    "Check glideine environment and exit if it is set"
    glidein = os.environ.get('GLIDEIN_CMSSite', '')
    if  glidein:
        msg = "ERROR: das_client is running from GLIDEIN environment, it is prohibited"
        print(msg)
        sys.exit(EX__BASE)

def check_auth(key):
    "Check if user runs das_client with key/cert and warn users to switch"
    if  not key:
        msg  = "WARNING: das_client is running without user credentials/X509 proxy, create proxy via 'voms-proxy-init -voms cms -rfc'"
        print(msg, file=sys.stderr)

def fullpath(path):
    "Expand path to full path"
    if  path and path[0] == '~':
        path = path.replace('~', '')
        path = path[1:] if path[0] == '/' else path
        path = os.path.join(os.environ['HOME'], path)
    return path

class WMArchiveOptionParser:
    """
    WMArchive cache client option parser
    """
    def __init__(self):
        usage  = "Usage: %prog [options]\n"
        usage += "For more help please visit https://github.com/dmwm/WMArchive/wiki"
        self.parser = OptionParser(usage=usage)
        self.parser.add_option("--verbose", action="store", type="int",
            default=0, dest="verbose", help="verbosity level")
        self.parser.add_option("--spec", action="store", type="string",
            default="query.json", dest="spec",
            help="specify query spec file, defalt query.json")
        host = 'https://vocms013.cern.ch'
        msg  = "host name of WMArchive server, default is %s" % host
        self.parser.add_option("--host", action="store", type="string",
                       default=host, dest="host", help=msg)
        msg  = 'specify private key file name, default $X509_USER_PROXY'
        self.parser.add_option("--key", action="store", type="string",
                               default=x509(), dest="ckey", help=msg)
        msg  = 'specify private certificate file name, default $X509_USER_PROXY'
        self.parser.add_option("--cert", action="store", type="string",
                               default=x509(), dest="cert", help=msg)
    def get_opt(self):
        """
        Returns parse list of options
        """
        return self.parser.parse_args()

def get_data(host, spec_file, ckey, cert, verbose=0):
    """Contact WMArchive server and retrieve data for given spec"""
    try:
        query = json.load(open(spec_file))
    except Exception as exp:
        msg = 'Unable to open spec JSON file'
        print(msg)
        print(str(exp))
        sys.exit(EX_DATAERR)
    if  'spec' not in query or 'fields' not in query:
        print('The input query JSON file must contain a "spec" and "fields"')
        print('The "spec" represents a dict of conditions or list of wmaids')
        print('    Examples: "spec":{"lfn":"bla.root", "timerange":[YYYYMMDD,YYYYMMDD]}')
        print('              "spec":[wmaid1, wmaid2]')
        print('The "fields" provides list of keys to look-up')
        print('    Examples: "fields":["wmaid", "lfn"]')
        sys.exit(EX_NOINPUT)
    path = '/wmarchive/data'
    pat  = re.compile('http[s]{0,1}://')
    if  not pat.match(host):
        msg = 'Invalid hostname: %s' % host
        print(msg)
        sys.exit(EX_UNAVAILABLE)
    url = host + path
    data = json.dumps(query)
    client = '%s (%s)' % (WMArchive_CLIENT, os.environ.get('USER', ''))
    headers = {"Accept": "application/json", "User-Agent": client, "Content-type":"application/json"}
    req  = urllib2.Request(url=url, data=data, headers=headers)
    ckey = fullpath(ckey)
    cert = fullpath(cert)
    http_hdlr  = HTTPSClientAuthHandler(ckey, cert, verbose)
    proxy_handler  = urllib2.ProxyHandler({})
    cookie_jar     = cookielib.CookieJar()
    cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
    opener = urllib2.build_opener(http_hdlr, proxy_handler, cookie_handler)
    fdesc = opener.open(req)
    response = json.load(fdesc)
    fdesc.close()
    return response

def main():
    """Main function"""
    optmgr  = WMArchiveOptionParser()
    opts, _ = optmgr.get_opt()
    data = get_data(opts.host, opts.spec, opts.ckey, opts.cert, opts.verbose)
    print(json.dumps(data))

#
# main
#
if __name__ == '__main__':
    main()


from setuptools import setup, find_packages
import os
# import pkg_resources
# installed_pkg = {pkg.key for pkg in pkg_resources.working_set}

# # Determine version on branch name
# active_branch = os.environ.get('CI_COMMIT_REF_NAME')
# if active_branch is None:
#     if 'git' in installed_pkg:
#         import git
#         local_repo = git.Repo('./')
#         active_branch = local_repo.active_branch.name

# if not(active_branch is None) and ('packaging' in installed_pkg):
#     import packaging.version  # noqa: 401

#     # CI case
#     try:
#         version = str(packaging.version.Version(active_branch))

#     except packaging.version.InvalidVersion:
#         if active_branch in ['devel', 'master']:
#             version = active_branch
#         else:
#             version = "unstable"
# else:
#     # No CI
#     version = None

version = "0.0.23"

setup(name='databayes',
      version=version,
      url='https://github.com/alphabayes/databayes',
      author='Roland Donat',
      author_email='roland.donat@gmail.com, roland.donat@edgemind.fr, roland.donat@alphabayes.fr',
      maintainer='Roland Donat',
      maintainer_email='roland.donat@gmail.com',
      keywords='pronostic datascience machine-learning ',
      classifiers=[
          'Development Status :: 3 - Alpha',
          'Intended Audience :: Science/Research',
          'License :: OSI Approved :: MIT License',
          'Operating System :: POSIX :: Linux',
          'Programming Language :: Python :: 3.7',
          'Topic :: Scientific/Engineering :: Artificial Intelligence'
      ],
      packages=find_packages(
          exclude=[
              "*.tests",
              "*.tests.*",
              "tests.*",
              "tests",
              "log",
              "log.*",
              "*.log",
              "*.log.*"
          ]
      ),
      description='Model as a service machine learning library',
      license='GPL V3',
      platforms='ALL',
      python_requires='>=3.6',
      install_requires=[
          "typing-extensions",
          "pandas",
          "PyYAML",
          "pydantic",
          "tqdm",
          "scikit-learn",
          "pyAgrum",
          "colored",
          "gspread_pandas",
          "openpyxl",
          "XlsxWriter",
          # "psycopg2", # not needed for now
          "intervals",  # Really needed ?
          "deepmerge",
      ],
      zip_safe=False,
      )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QGraphicsDropShadowEffect, QWidget

class EffectTool:
    @staticmethod
    def setBlur(widget: QWidget, shadowRadius: int = 12, shadowColor: int = Qt.gray):
        # 去除默认边框
        widget.setWindowFlags(Qt.FramelessWindowHint)
        # 背景透明（就是ui中黑色背景的那个控件）
        widget.setAttribute(Qt.WA_TranslucentBackground, True)

        # 添加阴影
        effect = QGraphicsDropShadowEffect(widget)
        effect.setBlurRadius(shadowRadius)
        effect.setOffset(0, 0)
        effect.setColor(shadowColor)
        widget.setGraphicsEffect(effect)

    @staticmethod
    def setBlurs(widgets: [QWidget], blurRadius: int = 12, color: int = Qt.gray):
        for widget in widgets:
            EffectTool.setBlur(widget, blurRadius, color)


import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.patches import Rectangle

'''
0 -- up
1 -- right
2 -- down
3 -- left
'''
def get_motion_direction(x, epsilon):
	if np.random.uniform(0,1) > epsilon:
		return x
	return np.random.randint(0, 4)

def is_wall(x, y):
	if x == 0 or x == 49:
		return True
	if y == 0 or y == 24:
		return True
	if x == 25 or x == 26:
		if y != 12:
			return True
	return False

def is_goal(x, y):
	if x == 48 and y == 12:
		return True
	return False

def reward(x, y, hit_wall):
	if is_goal(x, y):
		return 100
	if hit_wall:
		return -1
	return 0

def transition(x, y, dir, epsilon):
	# add arbitrariness
	dir = get_motion_direction(dir, epsilon)

	xf = x
	yf = y
	if dir == 0:
		yf += 1
	elif dir == 1:
		xf += 1
	elif dir == 2:
		yf -= 1
	else:
		xf -= 1
	return (xf, yf)

def get_next_state(qsa_grid, x, y, epsilon):
	dir = np.argmax(qsa_grid[x][y])
	xf, yf = transition(x, y, dir, epsilon)
	hit_wall = False
	if is_wall(xf, yf):
		hit_wall = True
		xf, yf = x, y
	r = reward(xf, yf, hit_wall)
	return (xf, yf, r, dir)

def initialise_qsa_grid():
	qsa_grid = np.zeros((50, 25, 4))
	for x in range(50):
		for y in range(25):
			for a in range(4):
				qsa_grid[x][y][a] = 1
	return qsa_grid

def get_start_state():
	x = 0
	y = 0
	while (is_wall(x, y) or is_goal(x, y)):
		x = np.random.randint(0, 50)
		y = np.random.randint(0, 25)
	return (x, y)

def run_one_episode(qsa_grid, learning_rate, epsilon, episode_length):
	discount_factor = 0.99
	x, y = get_start_state()
	reward_per_episode = 0
	for ep_len in range(episode_length):
		xf, yf, r, action = get_next_state(qsa_grid, x, y, epsilon)
		qsa_grid[x][y][action] += learning_rate*(r + discount_factor*np.max(qsa_grid[xf][yf]) - qsa_grid[x][y][action])
		reward_per_episode += r
		x, y = xf, yf
		if is_goal(xf, yf):
			break
	return qsa_grid, reward_per_episode

def q_learning(learning_rate, epsilon, episodes = 4000, episode_length = 1000):
	qsa_grid = initialise_qsa_grid()
	reward_per_episode = []
	for eps in range(episodes):
		qsa_grid, r = run_one_episode(qsa_grid, learning_rate, epsilon, episode_length)
		reward_per_episode.append(r)

	val_grid = np.zeros((50, 25))
	action_grid = np.zeros((50, 25))
	for x in range(qsa_grid.shape[0]):
		for y in range(qsa_grid.shape[1]):
			if is_wall(x, y) or is_goal(x, y):
				action_grid[x][y] = -1
			else:
				action_grid[x][y] = np.argmax(qsa_grid[x][y])
				val_grid[x][y] = np.max(qsa_grid[x][y])
	return (val_grid, action_grid, reward_per_episode)

def show_heatmap(arr):
	ax = sns.heatmap(arr.T, cmap='gray', linewidth=0.1)
	ax.invert_yaxis()
	ax.add_patch(Rectangle((48, 12), 1, 1, fill=False, edgecolor='red', lw=2))
	ax.add_patch(Rectangle((0, 0), 1, 25))
	ax.add_patch(Rectangle((0, 24), 50, 1))
	ax.add_patch(Rectangle((49, 0), 1, 25))
	ax.add_patch(Rectangle((0, 0), 50, 1))
	ax.add_patch(Rectangle((25, 0), 2, 12))
	ax.add_patch(Rectangle((25, 13), 2, 12))
	plt.show()

def plot_reward(r):
	y = [i for i in range(len(r))]
	plt.plot(y, r)
	plt.ylabel('Reward')
	plt.xlabel('Episodes')
	plt.show()

def plot_reward_avg(r):
	r = [np.sum(r[10*i:10*(i+1)])/10 for i in range(len(r)//10)]
	y = [10*i for i in range(len(r))]
	plt.plot(y, r)
	plt.ylabel('Reward')
	plt.xlabel('Episodes')
	plt.show()

def action_indicator_diag(arr):
	x = np.arange(0.5, 50.5, 1)
	y = np.arange(0.5, 25.5, 1)

	X, Y = np.meshgrid(x, y)
	u = np.zeros((25, 50))
	v = np.zeros((25, 50))

	for i in range(50):
		for j in range(25):
			if (arr[i][j] == 0):
				v[j][i] = 1
			elif (arr[i][j] == 1):
				u[j][i] = 1
			elif (arr[i][j] == 2):
				v[j][i] = -1
			else:
				u[j][i] = -1

	# creating plot
	# fig, ax = plt.subplots(figsize =fsize)
	fig, ax = plt.subplots()
	ax.quiver(X, Y, u, v, angles='xy', pivot='mid')

	major_ticks_x = np.arange(0, 50, 1)
	#minor_ticks_x = np.arange(0, 49, 1)
	major_ticks_y = np.arange(0, 25, 1)
	#minor_ticks_y = np.arange(0, 24, 1)

	ax.axis([0, 50, 0, 25])
	ax.set_xticks(major_ticks_x)
	ax.set_yticks(major_ticks_y)
	ax.add_patch(Rectangle((0, 0), 1, 25))
	ax.add_patch(Rectangle((0, 24), 50, 1))
	ax.add_patch(Rectangle((49, 0), 1, 25))
	ax.add_patch(Rectangle((0, 0), 50, 1))
	ax.add_patch(Rectangle((25, 0), 2, 12))
	ax.add_patch(Rectangle((25, 13), 2, 12))
	ax.add_patch(Rectangle((48, 12), 1, 1, color = 'red', alpha = 0.3))

	ax.grid(True)
	plt.minorticks_on
	ax.set_axisbelow(True)

	plt.show()

def part_b():
	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.05)
	show_heatmap(val_grid)
	action_indicator_diag(action_grid)

def part_c():
	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.005)
	show_heatmap(val_grid)
	action_indicator_diag(action_grid)

	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.05)
	show_heatmap(val_grid)
	action_indicator_diag(action_grid)

	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.5)
	show_heatmap(val_grid)
	action_indicator_diag(action_grid)

def part_d():
	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.05)
	plot_reward(reward_per_episode)
	plot_reward_avg(reward_per_episode)

	np.random.seed(1)
	val_grid, action_grid, reward_per_episode = q_learning(0.25, 0.5)
	plot_reward(reward_per_episode)
	plot_reward_avg(reward_per_episode)

part_b()
part_c()
part_d()


# Author: Steven C. Dang

# Main script for importing provided D3M dataset schemas for operation on datasets

import logging
import os.path as path
import os
import csv

# Workflow component specific imports
from ls_utilities.ls_logging import setup_logging
from ls_utilities.cmd_parser import get_default_arg_parser
from ls_utilities.ls_wf_settings import *
from ls_dataset.d3m_dataset import D3MDataset

__version__ = '0.1'

if __name__ == '__main__':

    # Parse argumennts
    parser = get_default_arg_parser("Import List of available D3M Datasets")
    args = parser.parse_args()

    if args.is_test is not None:
        is_test = args.is_test == 1
    else:
        is_test = False

    # Get config file
    config = SettingsFactory.get_settings(path.join(args.programDir, 'program', 'settings.cfg'),
                                          program_dir=args.programDir,
                                          working_dir=args.workingDir,
                                          is_test=is_test
                                          )

    # Setup Logging
    setup_logging(config)
    logger = logging.getLogger('datasets_importer')

    ### Begin Script ###
    logger.info("Importing List of available datasets")
    logger.debug("Running Dataset Importer with arguments: %s" % str(args))

    # Read in the dataset json
    ds_root = config.get_dataset_path()
    datasets = set()
    for root, dirs, files in os.walk(ds_root):
        for f in files:
            if f == 'datasetDoc.json':
                logger.debug("Found dataset in directory: %s" % root)
                try:
                    ds = D3MDataset.from_dataset_json(path.join(root, f))
                    if ds.name not in datasets:
                        logger.info("Found dataset name: %s\nAt path: %s" % (ds.name,  ds.dpath))
                        datasets.add(ds.name)
                except:
                    # Don't choke on unsupported dataset jsons
                    logger.warning("Encountered unsupported dataset: %s" % str(path.join(root, f)))

    logger.debug("Found datasets: %s" % str(datasets))

    # # Write dataset info to output file
    out_file_path = path.join(args.workingDir, config.get('Dataset', 'out_file'))
    logger.info("Writing dataset list of %i datasets to file: %s" % (len(datasets), out_file_path))
    with open(out_file_path, 'w') as out_file:
        out_csv = csv.writer(out_file, delimiter='\t')
        out_csv.writerow(datasets)


import argparse
import numpy as np

from maml import MAML
from Dynamics import Dynamics
import gym
from cost_functions import cheetah_cost_fn, ant_cost_fn
from my_envs.mujoco import *

import tensorflow as tf

import utils
from loginfo import log
import time
from controllers import MPCcontroller
from utils import configure_log_dir, Logger
from dataset.Gymdata import dataset

def argsparser():
    parser = argparse.ArgumentParser("Tensorflow Implementation of MAML")
    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--env_name', type=str,
                        default='HalfCheetahEnvDisableEnv-v0')  # HalfCheetahEnvDisableEnv   HalfCheetahVaryingEnv
    # Dataset
    parser.add_argument('--dataset', help='environment ID', choices=['sin,gym'], default='gym',
                        required=False)
    # MAML
    parser.add_argument('--K', type=int, default=32)  # horizon num_paths
    parser.add_argument('--num_paths', type=int, default=1)  # num_paths
    parser.add_argument('--model_type', type=str, default='fc')
    parser.add_argument('--loss_type', type=str, default='MSE')
    parser.add_argument('--num_updates', type=int, default=1)
    parser.add_argument('--norm', choices=['None', 'batch_norm'], default='batch_norm')
    # Train
    parser.add_argument('--is_train', action='store_true', default=True)

    parser.add_argument('--beta', type=float, default=0.01)  # learning rate for new taks during update
    parser.add_argument('--max_epochs', type=int, default=5)  # for update_training using experience

    # Test
    parser.add_argument('--restore_checkpoint', type=str)
    parser.add_argument('--restore_dir', type=str,
                        default='Hyper/log-files/HalfCheetahEnvDisableEnv-v0/Jun-03_21:18:39Meta_Train_train_meta_KXX_alXX_LabExp1/Jun-03_21:18:39Meta_Train_train_meta_K32_up1_pa1_al0.01_be0.01_batch500_LabExp1/checkpoint/MAML.HalfCheetahEnvDisableEnv-v0_gym_32-shot_1-updates_500-batch_norm-batch_norm-EXP_Meta_Train_train_meta_K32_up1_pa1_al0.01_be0.01_batch500_LabExp1')

    parser.add_argument('--NumOfExp', type=int, default=32)
    parser.add_argument('--horizon',  type=int, default=1000)
    parser.add_argument('--num_itr', type=int, default=5)
    parser.add_argument('--task_goal', type=float, default=5)

    # MPC Controller
    parser.add_argument('--mpc_horizon', '-m', type=int, default=10)  # mpc simulation H  10
    parser.add_argument('--simulated_paths', '-sp', type=int, default=5000)  # mpc  candidate  K 10000

    parser.add_argument('--note', type=str, default='Control_MPC_EXP02')
    args = parser.parse_args()
    print(args)
    return args

def rollout(env,
            controller,
            dyn_model,
            task_goal,
            experiences,
            NumOfExp = 100,
            horizon=1000,
            cost_fn=cheetah_cost_fn,
            render=False,
            verbose=False,
            save_video=False,
            ignore_done=True,
            ):
    """
        Write a sampler function which takes in an environment, a controller (either random or the MPC controller),
        and returns rollouts by running on the env.
        Each path can have elements for observations, next_observations, rewards, returns, actions, etc.
    """
    paths = []

    """ YOUR CODE HERE """
    obs = env.reset(reset_args=task_goal)
    observations, actions, next_observations, rewards ,meta_train_losses= [], [], [], [],[]
    done = False
    t = 0

    start = time.time()  # caculate run time --start time point

    while not done:
        if render:
            env.render()
        obs = obs.astype(np.float64).reshape((1, -1))
        meta_train_loss = dyn_model.update(experiences)
        if meta_train_loss is not None:
            meta_train_losses.append(meta_train_loss)
        start = time.time()
        action = controller.get_action5(dyn_model, obs)  # highly time-consuming
        action = action.astype(np.float64).reshape((1, -1))
        end = time.time()
        runtime = end - start
        next_obs, reward, done, _ = env.step(action)

        end2 = time.time()
        runtime1 = end2 - end

        next_obs = next_obs.astype(np.float64).reshape((1, -1))
        current_experience = np.concatenate((obs, action, next_obs - obs), axis=1)
        obs = next_obs

        experiences.append(current_experience)

        if not isinstance(reward, float):
            reward = np.asscalar(reward)
        rewards.append(reward)

        if len(experiences) >  NumOfExp:
            del experiences[0]

        end3 = time.time()
        runtime2 = end3 - end2

        t += 1
        if not ignore_done:
            if done:
                print("Episode finished after {} timesteps".format(t + 1))
                break
        else:
            if t >= horizon:
                break

    end = time.time()
    runtime1 = end - start

    return sum(rewards), sum(meta_train_losses)/horizon

def main(args):
    tf.set_random_seed(args.seed)
    np.random.seed(args.seed)

    env_name = args.env_name  # HalfCheetah-v2  My3LineDirect-v1
    print(env_name)

    if args.env_name == 'HalfCheetahEnvDisableEnv-v0':
        cost_fn = cheetah_cost_fn
        sample_task_fun = np.random.randint
    elif args.env_name == 'HalfCheetahVaryingEnv-v0':
        cost_fn = cheetah_cost_fn
        sample_task_fun = np.random.uniform
    else:
        print('env is error!!! ')

    env = gym.make(env_name)
    dim_input = env.observation_space.shape[0]+env.action_space.shape[0]
    dim_output = env.observation_space.shape[0]

    logdir = configure_log_dir(logname=env_name, txt=args.note)
    # save args prameters
    with open(logdir + '/info.txt', 'wt') as f:
        print('Hello World!\n', file=f)
        print(args, file=f)

    mpc_horizon = args.mpc_horizon
    num_simulated_paths = args.simulated_paths #10000

    dyn_model = Dynamics(args.env_name,

                 args.NumOfExp,
                 args.model_type,
                 args.loss_type,
                 dim_input,
                 dim_output,
                 beta = args.beta,#args.beta,
                 max_epochs=args.max_epochs,
                 is_train =args.is_train,
                 norm = args.norm,
                 task_Note=args.note,
                 restore_checkpoint=args.restore_checkpoint,
                 restore_dir=args.restore_dir,
                 logdir = logdir
                 )

    mpc_controller = MPCcontroller(env=env,
                                   dyn_model=dyn_model,
                                   horizon=mpc_horizon,
                                   cost_fn=cost_fn,
                                   num_simulated_paths=num_simulated_paths,
                                   )
    logger = Logger(logdir, csvname='log')

    num_itr = args.num_itr
    experiences,costs =[],[]
    print('MPC is beginning...' )
    for itr in range(num_itr):
        reward, model_loss_mean = rollout(env, mpc_controller, task_goal = args.task_goal,
                       dyn_model=dyn_model,experiences=experiences, NumOfExp= args.NumOfExp,horizon=args.horizon, cost_fn=cheetah_cost_fn,
                render=False, verbose=False, save_video=False, ignore_done=True, )

        #print(time.asctime( time.localtime(time.time()) ), ' itr :', itr, 'Average reward :' , cost)
        log.infov("Itr {}/{} Accumulated Reward: {:.4f}  Model loss mean:{:.4f}".format(itr, num_itr,reward, model_loss_mean))

        logger.log({'itr': itr,
                    'Accumulated Reward': reward,
                    'Model loss mean': model_loss_mean,
                    })

    print('MPC is over....')

    logger.write(display=False)

if __name__ == '__main__':
    args = argsparser()
    main(args)


"""added decision_by_id fields to works

Revision ID: 622cb040a90e
Revises: 2b164207f899
Create Date: 2023-02-02 11:38:06.597732

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '622cb040a90e'
down_revision = '2b164207f899'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.execute('TRUNCATE works RESTART IDENTITY CASCADE')
    op.add_column('work_statuses', sa.Column('work_status_notes', sa.Text(), nullable=True))
    op.add_column('works', sa.Column('eac_decision_by_id', sa.Integer(), nullable=False))
    op.add_column('works', sa.Column('decision_by_id', sa.Integer(), nullable=False))
    op.create_foreign_key(None, 'works', 'staffs', ['decision_by_id'], ['id'])
    op.create_foreign_key(None, 'works', 'staffs', ['eac_decision_by_id'], ['id'])
    op.drop_column('works', 'work_status_notes')
    op.drop_column('works', 'work_short_status')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('works', sa.Column('work_short_status', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
    op.add_column('works', sa.Column('work_status_notes', sa.TEXT(), autoincrement=False, nullable=True))
    op.drop_constraint(None, 'works', type_='foreignkey')
    op.drop_constraint(None, 'works', type_='foreignkey')
    op.drop_column('works', 'decision_by_id')
    op.drop_column('works', 'eac_decision_by_id')
    op.drop_column('work_statuses', 'work_status_notes')
    # ### end Alembic commands ###


import json
import boto3
import os
import logging

def lambda_handler(event, context):
    connection_id = event["requestContext"]["connectionId"]
    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table('Connection')

    ###### $connect
    if event["requestContext"]["eventType"] == "CONNECT":
        table.put_item(
            Item={
            'connection_Id': connection_id,
            'matched_truckid' : 1,
            'from_lon' : 140,
            'from_lat' : 40,
            'to_lon' : 100,
            'to_lat' : 60
            }
        )
        return {"statusCode": 200, "body": "Connect successful."}
    ###### $disconnect
    elif event['requestContext']['eventType'] == "DISCONNECT":
        table.delete_item(Key={'connection_Id': connection_id})
        return {"statusCode": 200, "body": "Disconnect successful."}
    ###### $default
    else:
        return {"statusCode": 500, "body": "Unrecognized eventType."}


import requests
import json
if __name__ == '__main__':
    url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36'
    }

    place = input('enter a place:')
    page = 1  # 从第1页开始
    fileName = place + 'KFC餐厅位置信息' + '.json'
    for i in range(0, 20):  # 设置一个较大参数直到爬完所有页码
        param = {
            'cname': '',
            'pid': '',
            'keyword': place,  # 查询地点
            'pageIndex': page,  # 查询页码
            'pageSize': '10',  # 每页最多显示10个
        }
        response = requests.post(url=url, params=param, headers=headers)
        page_text = response.text
        # print(page_text)
        with open(fileName, 'a', encoding='utf-8') as fp:
            json.dump(page_text, fp=fp, ensure_ascii=False)
            fp.write('\n')  # 注意这里还是在for循环当中，每爬取完一页内容，就敲个回车
        page = page + 1  # 佛如循环的循环变量，注意前文默认为1
    print('over!!!')

05-肯德基餐厅位置查询.py


from django.shortcuts import render, redirect
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.views.generic import ListView, DetailView
from django.contrib.auth import login
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.decorators import login_required
# Import the mixin for class-based views
from django.contrib.auth.mixins import LoginRequiredMixin
from .models import Dove, Toy
from .forms import FeedingForm
# Create your views here.
# Define the home view
def signup(request):
  error_message = ''
  if request.method == 'POST':
    # This is how to create a 'user' form object
    # that includes the data from the browser
    form = UserCreationForm(request.POST)
    if form.is_valid():
      # This will add the user to the database
      user = form.save()
      # This is how we log a user in via code
      login(request, user)
      return redirect('index')
    else:
      error_message = 'Invalid credentials - try again'
  # A bad POST or a GET request, so render signup.html with an empty form
  form = UserCreationForm()
  context = {'form': form, 'error_message': error_message}
  return render(request, 'registration/signup.html', context)

class DoveCreate(LoginRequiredMixin, CreateView):
  model = Dove
  fields = '__all__'

  def form_valid(self, form):
    # Assign the logged in user
    form.instance.user = self.request.user
    # Let the CreateView do its job as usual
    return super().form_valid(form)

class DoveUpdate(UpdateView):
  model = Dove
  # Let's disallow the renaming of a dove by excluding the name field!
  fields = ['breed', 'description', 'age']

class DoveDelete(DeleteView):
  model = Dove
  success_url = '/doves/'

def home(request):
  return render(request, 'home.html')

def about(request):
  return render(request, 'about.html')
#   return HttpResponse('<h1>About Dove Collector<h1>')

@login_required
def doves_index(request):
  doves = Dove.objects.filter(user=request.user)
  # You could also retrieve the logged in user's doves like this
  # doves = request.user.dove_set.all()
  return render(request, 'doves/index.html', { 'doves': doves })
  # doves = Dove.objects.all()
  # return render(request, 'doves/index.html', { 'doves': doves})

@login_required
def doves_detail(request, dove_id):
  dove = Dove.objects.get(id=dove_id)
  # Get the toys the cat doesn't have
  toys_dove_doesnt_have = Toy.objects.exclude(id__in = dove.toys.all().values_list('id'))
  # Instantiate FeedingForm to be rendered in the template
  feeding_form = FeedingForm()
  return render(request, 'doves/detail.html', {
    # Pass the cat and feeding_form as context
    'dove': dove, 'feeding_form': feeding_form,
    # Add the toys to be displayed
    'toys': toys_dove_doesnt_have
    })

@login_required
def add_feeding(request, dove_id):
  # create a ModelForm instance using the data in request.POST
  form = FeedingForm(request.POST)
  # validate the form
  if form.is_valid():
    # don't save the form to the db until it
    # has the cat_id assigned
    new_feeding = form.save(commit=False)
    new_feeding.dove_id = dove_id
    new_feeding.save()
  return redirect('detail', dove_id=dove_id)

@login_required
def assoc_toy(request, dove_id, toy_id):
  Dove.objects.get(id=dove_id).toys.add(toy_id)
  return redirect('detail', dove_id=dove_id)

@login_required
def unassoc_toy(request, dove_id, toy_id):
  Dove.objects.get(id=dove_id).toys.remove(toy_id)
  return redirect('detail', dove_id=dove_id)

class ToyList(ListView):
  model = Toy

class ToyDetail(DetailView):
  model = Toy

class ToyCreate(CreateView):
  model = Toy
  fields = '__all__'

class ToyUpdate(UpdateView):
  model = Toy
  fields = ['name', 'color']

class ToyDelete(DeleteView):
  model = Toy
  success_url = '/toys/'


import re
import json
import string
from bs4 import BeautifulSoup
from selenium import webdriver
import csv

def get_soup(url):
    driver = webdriver.Chrome(executable_path=r'C:\Users\Ashish Mishra\Desktop\chromedriver.exe')
    driver.get(url)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')

    driver.close()
    return soup

def get_job_links(soup):
    urls = []
    # Loop thru all the posting links

    for link in soup.find_all('h2', {'class': 'jobtitle'}):
        # Since sponsored job postings are represented by "a target" instead of "a href", no need to worry here
        partial_url = link.a.get('href')
        # This is a partial url, we need to attach the prefix
        url = 'https://www.indeed.co.in' + partial_url
        # Make sure this is not a sponsored posting
        urls.append(url)
    return urls

def get_urls(query, num_pages, location, state):
    # # h  https: // www.indeed.co. in / jobs?q ={}& l ={} + % 2C + Karnataka & start = 10
    #
    hey_url = 'https: // www.indeed.co.in / jobs?q = {}  & l = {}+,+{}'.format(query, location, state)

    base_url = hey_url.translate(({ord(c): None for c in string.whitespace}))
    print(base_url)
    soup = get_soup(base_url)
    urls = get_job_links(soup)
    # https: // www.indeed.co.in / jobs?q = [ % 27 % 22data + scientist % 22 % 27] & l = Bengaluru, +Karnataka
    # Get the total number of postings found
    # # https: // www.indeed.co. in / jobs?q = % 22data + engineer % 22 & l = Bengaluru % 2C + Karnataka
    posting_count_string = soup.find(name='div', attrs={'id': "searchCount"}).get_text()
    posting_count_string = posting_count_string[posting_count_string.find('of') + 2:].strip()
    try:
        posting_count = int(posting_count_string)
    except ValueError:  # deal with special case when parsed string is "360 jobs"
        posting_count = int(re.search('\d+', posting_count_string).group(0))
    finally:
        posting_count = 330  # setting to 330 when unable to get the total
        pass
    max_pages = round(posting_count / 10) - 3
    if num_pages > max_pages:
        print('returning max_pages!!')
        return max_pages
    if num_pages >= 2:
        for i in range(2, num_pages + 1):
            num = (i - 1) * 10
            # https: // www.indeed.co. in / jobs?q = % 22
            # etl + developer % 22 & l = Bengaluru, +Karnataka & start = 0
            # https: // www.indeed.co. in / jobs?q = % 22data + engineer % 22 & l = Bengaluru, +Karnataka & start = 0
            hey_url = 'https: // www.indeed.co.in / jobs?q = {}  & l = {}+,+{}&start={}'.format(query, location, state, num)

            base_url = hey_url.translate(({ord(c): None for c in string.whitespace}))
            try:
                soup = get_soup(base_url)
                urls += get_job_links(soup)
            except:
                continue

    return urls

def get_posting(url):
    soup = get_soup(url)
#h3 tag
    title = soup.find(name='h3').getText().lower()
    posting = soup.find(name='div', attrs={'class': "jobsearch-JobComponent"}).get_text()

    return title, posting.lower()

def get_data(query, num_pages, location):
    postings_dict = {}
    urls = get_urls(query, num_pages, location, state)

    if isinstance(urls, list):
        num_urls = len(urls)
        for i, url in enumerate(urls):
            try:
                title, posting = get_posting(url)
                postings_dict[i] = {}
                postings_dict[i]['title'], postings_dict[i]['posting'], postings_dict[i]['url'] = \
                    title, posting, url
            except:
                continue

        # with open('file_name'.csv, 'w') as file_name:
        #   csv.writer(file_name, postings_dict)

        file_name = query.replace('"', "") + '.json'
        with open(file_name, 'w+') as f:
            print('hey')
            print((file_name))
            json.dump(postings_dict, f)

        print('{} JobPostings have been scraped : '.format(num_urls))

if __name__ == "__main__":

    fs = open('watching_list2.csv', 'r+')
    lines = fs.readlines()

    i = 0

    while i < len(lines):
        lines[i] = lines[i].lower().rstrip()
        queries = lines
        i = i + 1
        d = queries[0]
    query = '"'+d+'"'
    print(query)
    num_pages = 1
    location = 'Bengaluru'
    state = 'Karnataka'

    get_data(query, num_pages, location)


#!/usr/bin/env python3

import cheminfo

import sys

import numpy as np
import qml
import scipy
from qml.kernels import (get_atomic_local_gradient_kernel,
                         get_atomic_local_kernel)
from qml.math import svd_solve
from qml.representations import generate_fchl_acsf
import rmsd

from train_test import csvdir_to_reps

np.random.seed(666)

def train(dataname, n_train=100):

    SIGMA = 10.0

    # Read training data from file
    # X,  dX,  Q,  E,  F  = get_data_from_file(filename_train, n=n_train)

    Xall,  dXall,  Qall,  Eall,  Fall =  csvdir_to_reps(dataname)

    if  len(Eall) < n_train:
        print("Not enough training data for", n_train)
        exit()

    idx = list(range(len(Eall)))
    np.random.shuffle(idx)

    train = idx[:n_train]

    print(len(train))

    X  = Xall[train]
    dX = dXall[train]
    Q  = [Qall[i] for i in train]
    E  = Eall[train]
    F  = [Fall[i] for i in train]

    offset = 0.0
    print("OFFSET: ", offset)

    F = np.concatenate(F)
    Y = np.concatenate((E, F.flatten()))

    print("Generating Kernels ...")
    Kte = get_atomic_local_kernel(X, X,  Q, Q,  SIGMA)
    Kt = get_atomic_local_gradient_kernel(X, X,  dX,  Q, Q,  SIGMA)

    C = np.concatenate((Kte, Kt))

    print("Alphas operator ...")
    alpha = svd_solve(C, Y, rcond=1e-11)

    np.save("data/"+dataname+"_offset.npy", offset)
    np.save("data/"+dataname+"_sigma.npy", SIGMA)
    np.save("data/"+dataname+"_alphas.npy", alpha)
    np.save("data/"+dataname+"_Q.npy", Q, allow_pickle=True)
    np.save("data/"+dataname+"_X.npy", X)

    return

if __name__ == "__main__":

    import argparse
    import sys

    parser = argparse.ArgumentParser()
    parser.add_argument('--correct', action='store_true', help='Add penalty and correction molecules to kernel')
    parser.add_argument('dataname', metavar='NAME', type=str)

    args = parser.parse_args()

    if len(sys.argv[1:]) < 1:
        print("choose some data")
        quit()

    dataname = args.dataname

    train(dataname, n_train=150)

    # # Get sample from test
    # data = np.load("data/" + dataname + "-test.npz")
    # coordinates = data["R"][0]
    # nuclear_charges = data["z"]
    # nuclear_charges = [cheminfo.convert(x) for x in nuclear_charges]

    # txt = rmsd.set_coordinates(nuclear_charges, coordinates)
    # f = open("examples/"+dataname+".xyz", 'w')
    # f.write(txt)
    # f.close()


import requests
from bs4 import BeautifulSoup
import pandas as pd

def gameFinder(gameName):
    gameName = gameName.replace(' ', '_')
    page = requests.get(f"https://en.wikipedia.org/wiki/{gameName}")
    soup = BeautifulSoup(page.content, 'html.parser')
    main = soup.find(id='bodyContent')
    infos = main.find_all('p')
    definition = infos[1].get_text() + infos[2].get_text()
    game = '\t\t***Definition***\n' + definition + '\n'
    gameName = gameName.replace("_", "+")
    ytTrailler = '\n\n\t\t***Trailer && Gameplay ***\n' + f"https://www.youtube.com/results?search_query={gameName}+trailer"
    downlodForFree = '\n\n\t\t***Download For Free***\n' + f'https://fitgirlrepacks.co/search/{gameName}'
    return game + ytTrailler + downlodForFree


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from get_KONICA_data import get_KONICA_data
import json
import os

# Sigmoid函数
def sigmoid(x, k, x0):
    return 1 / (1 + np.exp(-k * (x - x0)))
# root_path = r'B:\Datasets\Temporal_Flicker_Meter_log\deltaL_L_2s\2023-12-17-05-06-02'
# root_path = r'B:\Datasets\Temporal_Flicker_Meter_log\deltaL_L_5s\2023-12-18-01-08-47'
root_path = r'B:\Datasets\Temporal_Flicker_Meter_log\deltaL_L_10s\2023-12-18-17-18-03'
with open(os.path.join(root_path, 'config.json'), 'r') as fp:
    config_data = json.load(fp)
with open(r'B:\Py_codes\VRR_Real\dL_L\Temporal_Results/dl_L_results_10s.json', 'r') as fp:
    json_result_dict = json.load(fp)
Temporal_L_array = np.array(json_result_dict['L'])
Temporal_dL_array = np.array(json_result_dict['dL'])
Temporal_abnormal_array = np.array(json_result_dict['abnormal'])
size_num, vrr_f_num,repeat_num, color_num = Temporal_L_array.shape
Temporal_dL_L_array = Temporal_dL_array / Temporal_L_array
x_fit = np.linspace(np.min(np.log10(Temporal_L_array))-2, np.max(np.log10(Temporal_L_array))+2, 100)
VRR_Frequency_list = config_data['change_params']['VRR_Frequency']
Size_list = config_data['change_params']['Size']
initial_guess_for_k = 1.0
initial_guess_for_x0 = 0.0
p0 = [initial_guess_for_k, initial_guess_for_x0]
fig1, ax1 = plt.subplots(figsize=(8,6))
for size_i in range(size_num):
    for vrr_f_i in range(vrr_f_num):
        current_L_array = Temporal_L_array[size_i, vrr_f_i, :, :].mean(axis=0)
        current_dl_L_array = Temporal_dL_L_array[size_i, vrr_f_i, :, :].mean(axis=0)
        x = np.log10(current_L_array)
        y = current_dl_L_array
        popt, pcov = curve_fit(sigmoid, x, y, p0=p0)

        # 绘制散点和拟合曲线
        plt.scatter(x, y)
        plt.scatter(x, y, label=f'Size {Size_list[size_i]} VRR_F {VRR_Frequency_list[vrr_f_i]}')
        # plt.plot(x, y, label=f'Size {Size_list[size_i]} VRR_F {VRR_Frequency_list[vrr_f_i]}', linewidth=0.5)
        # plt.plot(x_fit, sigmoid(x_fit, *popt),
        #          label=f'Fit - Size {Size_list[size_i]} VRR_F {VRR_Frequency_list[vrr_f_i]}', linestyle='--')

# Fit all:
current_L_array = Temporal_L_array.mean(axis=2).flatten()
current_dl_L_array = Temporal_dL_L_array.mean(axis=2).flatten()
x = np.log10(current_L_array)
y = current_dl_L_array
popt, pcov = curve_fit(sigmoid, x, y, p0=p0)

# 绘制拟合曲线
plt.plot(x_fit, sigmoid(x_fit, *popt), label=f'Fit - All Sizes', linewidth=4, color='red')

plt.legend(bbox_to_anchor=(1.05, 0.5), loc='center left', borderaxespad=0., prop={'size': 10})
pos1 = ax1.get_position()
pos1.x1 = 0.7
ax1.set_position(pos1)
plt.title('Sigmoid Fit for All Sizes')
plt.xlabel('log10(Luminance)')
plt.ylabel('deltaL/L')
plt.show()


# exercise 1.1
# use function 'bisect' to realize
# looking for the root of equation by bisect

from math import log2, log, e
from math import ceil, floor
from math import cos, sin
import matplotlib.pyplot as plt
import numpy as np

def bisect(f, a, b, tol):
    '''
    calculate the approximate solution of f(x)=0

    Input:
    - f : quation f
    - a : interval lower bound
    - b : interval upper bound  f(a)*f(b)<0
    - tol : tolerance

    Output:
    - xc : approximate solution of f
    '''
    if (f(a) < 0) == (f(b) < 0):
        raise Exception('Incorrect interval!')

    fa, fb = f(a), f(b)

    while (b-a)/2 > tol:
        c = (a+b)/2
        fc = f(c)

        if fc == 0:
            return c

        if (fa < 0) == (fc < 0):
            a, fa = c, fc

        else:
            b, fb = c, fc

    return (a+b) / 2

def bisect_digits(f, a, b, p):
    '''
    calculate the approximate solution of f(x) to
    the nearest p decimal places

    Input:
    - f : quation f
    - a : interval lower bound
    - b : interval upper bound  f(a)*f(b)<0
    - p : solution to the nearest p decimal places

    Output:
    - xc : approximate solution
    '''
    if (f(a) < 0) == (f(b) < 0):
        raise Exception('Incorrect interval!')

    n = ceil(log2((b-a) / (0.5 * 10**(-p))) - 1)
    print('Original Interval is [{},{}]'.format(a, b))
    print('To get {} decimal, we need {} steps.'.format(p, n))

    fa, fb = f(a), f(b)

    for i in range(n):
        c = (a+b) / 2
        fc = f(c)

        #print('Before {} loop, a={}, b={}, c={}'.format(i, a, b, c))

        if (fa < 0) == (fc < 0):
            a, fa = c, fc

        else:
            b, fb = c, fc

        #print('After this loop, a={}, b={}, c={}'.format(a, b, c))

    return round((a+b) / 2, p)

def plot_functions(f1, f2, f3):
    x = np.linspace(0,1,1000)
    plt.subplot(311)
    plt.plot(x, f1(x))
    plt.xlim(-2,2)
    plt.ylim(-1, 1)
    plt.subplot(312)
    plt.plot(x, f2(x))
    plt.xlim(-2,2)
    plt.ylim(-1, 1)
    plt.subplot(313)
    plt.plot(x, f3(x))
    plt.xlim(-2,2)
    plt.ylim(-1, 1)

    plt.show()

    return plt

if __name__ == '__main__':
    print('Test result:')
    f1 = lambda x: x**3 + x - 1
    print(bisect(f1, 0, 1, tol=0.00005))
    print('='*30)

    print('Problem 1.1-1')
    f1a = lambda x: x**3 - 9
    print('(a):', bisect_digits(f1a, 2, 3, 6))
    f1b = lambda x: 3*x**3 + x**2 - x - 5
    print('(b):', bisect_digits(f1b, 1, 2, 6))
    f1c = lambda x: cos(x)**2 -x + 6
    print('(c):', bisect_digits(f1c, 6, 7, 6))
    print('='*30)

    print('Problem 1.1-2')
    f2a = lambda x: x**5 + x - 1
    print('(a):', bisect_digits(f2a, 0, 1, 8))
    f2b = lambda x:sin(x) -6*x -5
    print('(b):', bisect_digits(f2b, -2, 0, 8))
    f2c = lambda x: log(x) + x**2 - 3
    print('(c):', bisect_digits(f2c, 1, 10, 8))
    print('='*30)

    print('Problem 1.1-3')
    f3a = lambda x: 2*x**3 - 6*x - 1
    f3b = lambda x: e**(x-2) + x**3 - x
    f3c = lambda x: 1 + 5*x - 6*x**3 - e*(2*x)
    #plot_functions(f3a, f3b, f3c)
    print('='*30)

    ziplist = [(2, 'a'), (3, 'b'), (5, 'c')]
    print('Problem 1.1-4')
    for i, j in ziplist:
        f4 = lambda x: x**2 - i
        print(j, bisect_digits(f4, 1, 3, 8))
    print('='*30)

    print('Problem 1.1-5')
    for i, j in ziplist:
        f5 = lambda x: x**3 - i
        print(j, bisect_digits(f5, 1, 3, 8))
    print('='*30)

    print('Problem 1.1-6')
    f6 = lambda x: cos(x)-sin(x)
    print('Approximate solution is:', bisect_digits(f6, 0, 1, 6))
    print('='*30)

    print('Problem 1.1-7')
    f7 = lambda x: np.linalg.det([[1,2,3,x],[4,5,x,6],[7,x,8,9],[x,10,11,12]]) - 1000
    x1 = bisect_digits(f7, -18, -17, 6)
    det1 = f7(x1)
    acc1 = ceil(log(abs(det1*2), 10))
    print('First root is:', x1)
    print('The det is: {} with {} decimal acc.'.format(det1 + 1000, -acc1))
    x2 = bisect_digits(f7, 9, 10, 6)
    det2 = f7(x2)
    acc2 = ceil(log(abs(det2*2), 10))
    print('Second root is:', x2)
    print('The det is:', det2)
    print('The det is: {} with {} decimal acc.'.format(det2 + 1000, -acc2))
    print('='*30)

    print('Problem 1.1-8')
    hibm = 1 / (np.arange(0, 5) + np.arange(1, 6)[:,np.newaxis])
    def f8(x):
        hibm[0][0] = x
        return np.linalg.eig(hibm)[0].max() - np.pi
    print('A_11 is:', bisect_digits(f8, 2, 3, 6))
    print('='*30)

    print('Problem 1.1-9')
    # v=4/3*pi*r^3 > 1*2
    f9 = lambda x: np.pi * x**2 * (1 - 1/3 * x) - 1
    print('The height is:', bisect(f9, 0, 1, 1e-3))
    print('='*30)