'''This spider is used to crawl and save spot
    value of equites like SBIN, YESBANK,
    etc. You need to provide SYMBOL, STARTDATE,
    & ENDDATE as argument to this spider.
'''

import sys
from os import path
sys.path.append( path.dirname( path.dirname( path.abspath(__file__) ) ) )

import datetime
import scrapy, scraper
from scrapy.loader import ItemLoader
from scrapy.selector import Selector
from scrapy.loader.processors import Join, MapCompose
from scraper.items import kHistoricEquityItem
from scraper import utility

class kHistoricEquitySpider(scrapy.Spider):

    name = 'kHistoricEquitySpider'
    allowed_domains = ['www.nseindia.com']

    def __init__(self, symbol, startDate, endDate, databaseTableName, coldStart, * args, **kwargs):
        super(kHistoricEquitySpider, self).__init__(*args, **kwargs)
        self.symbol = symbol
        self.startDate = startDate
        self.endDate = endDate
        self.databaseTableName = databaseTableName + symbol
        self.coldStart = coldStart

        # working_url = https://www.nseindia.com/products/dynaContent/common/productsSymbolMapping.jsp?symbol=sbin&segmentLink=3&symbolCount=1&series=EQ&dateRange=+&fromDate=01-01-2018&toDate=01-03-2018&dataType=PRICEVOLUMEDELIVERABLE
        # url = https://www.nseindia.com/products/dynaContent/common/productsSymbolMapping.jsp?symbol={symbol}&segmentLink=3&symbolCount=1&series=EQ&dateRange=+&fromDate={startDate}&toDate={endDate}&dataType=PRICEVOLUMEDELIVERABLE
        # Referer: https://www.nseindia.com/products/content/equities/equities/eq_security.htm

    def start_requests(self):
        scrapy.Spider.start_requests(self)

        symbolCount = 1
        try:
            symbolCount = utility.SYMBOL_COUNT[self.symbol]
        except Exception:
            # do nothing
            symbolCount = 1

        endDate = datetime.datetime.now()
        startDate = endDate - datetime.timedelta(days=3)

        if (self.coldStart == True):
            startDate =  datetime.datetime.strptime(self.startDate, "%d-%b-%Y")
            endDate = datetime.datetime.strptime(self.endDate, "%d-%b-%Y")

        while (endDate > startDate):

            tempEndDate = startDate + datetime.timedelta(days=90)

            startDateStr = startDate.strftime("%d-%m-%Y")
            tempEndDatStr = tempEndDate.strftime("%d-%m-%Y")

            forceBreak = False
            if tempEndDate >= datetime.datetime.now():
                tempEndDate = datetime.datetime.now() - datetime.timedelta(days=1)
                forceBreak = True

            url = "https://www.nseindia.com/products/dynaContent/common/productsSymbolMapping.jsp?symbol=%s&segmentLink=3&symbolCount=%s&series=EQ&dateRange=+&fromDate=%s&toDate=%s&dataType=PRICEVOLUMEDELIVERABLE" %(self.symbol.lower(), symbolCount, startDateStr, tempEndDatStr)

            yield scrapy.Request(url,self.parse, headers={'Referer': 'https://www.nseindia.com/products/content/equities/equities/eq_security.htm'})

            startDate = tempEndDate
            if forceBreak:
                break

    # used to save respective xpath value with their item,
    # later we can iterate through it.
    item_fields = {
                'Date':'./td[3]//text()',
                'Open':'./td[5]//text()',
                'High':'./td[6]//text()',
                'Low':'./td[7]//text()',
                'Close':'./td[9]//text()',
                'SharesTraded':'./td[11]//text()',
                'Turnover':'./td[12]//text()'
                }

    def parse(self, response):

        # checking for response, wheather it contains records or not
        if (response.xpath("//tr[2]//text()").extract()[0].find("No Records") < 0):
            # Storing all selector except top 2(which contains waste data)
            res = response.xpath('//tr[position()>2]')
            # val will iterate from 0 to (last -1), because last res item is waste
            for val in res[0:len(res)]:
                loader = ItemLoader(kHistoricEquityItem(), val)
                loader.default_input_processor = MapCompose(unicode.strip)
                loader.default_output_processor = Join()

                for item, xpath in self.item_fields.iteritems():
                    loader.add_xpath(item, xpath)
                loader.add_value('DatabaseTableName', unicode(self.databaseTableName))
                yield loader.load_item()


from sklearn.datasets import fetch_20newsgroups  # 导入fetch_20newsgroups函数，用于获取新闻数据集
from sklearn.naive_bayes import MultinomialNB  # 导入MultinomialNB朴素贝叶斯分类器
from sklearn.feature_extraction.text import TfidfTransformer  # 导入TfidfTransformer，用于计算TF-IDF特征
from sklearn.feature_extraction.text import CountVectorizer  # 导入CountVectorizer，用于将文本转换为词频向量

category_map = {'talk.politics.misc': 'Politics',  # 定义新闻类别的映射关系
                'rec.autos': 'Autos',
                'rec.sport.hockey': 'Hockey',
                'sci.electronics': 'Electronics',
                'sci.med': 'Medicine'}

training_data = fetch_20newsgroups(subset='train',  # 获取训练数据集
                                   categories=category_map.keys(),
                                   shuffle=True,
                                   random_state=5)

count_vectorizer = CountVectorizer()  # 创建CountVectorizer对象，用于将文本转换为词频向量
train_tc = count_vectorizer.fit_transform(training_data.data)  # 对训练数据进行词频向量化

tfidf = TfidfTransformer()  # 创建TfidfTransformer对象，用于计算TF-IDF特征
train_tfidf = tfidf.fit_transform(train_tc)  # 对词频向量进行TF-IDF转换

input_data = [  # 待分类的输入数据
    'Be sure to take medicine when you are ill',
    'The player made a mistake in passing the ball',
    'Be sure to fasten your seat belt when you drive a car',
    'Sensing technology has been well applied in this device'
]

classifier = MultinomialNB().fit(train_tfidf, training_data.target)  # 创建并训练MultinomialNB分类器

input_tc = count_vectorizer.transform(input_data)  # 对输入数据进行词频向量化

input_tfidf = tfidf.transform(input_tc)  # 对词频向量进行TF-IDF转换

predictions = classifier.predict(input_tfidf)  # 对输入数据进行分类预测

for sent, category in zip(input_data, predictions):  # 遍历输入数据和预测结果
    print('\nInput sentence:', sent, '\nPredicted category:', \
          category_map[training_data.target_names[category]])  # 打印输入句子和预测的类别名称


#!/usr/bin/env python
# coding: utf-8

import argparse

def parse_config():
    parser = argparse.ArgumentParser()
    parser.add_argument('--train_target', type=str,
                        default='data/uspto_full/tgt-train.txt')
    parser.add_argument('--val_target', type=str,
                        default='data/uspto_full/tgt-val.txt')
    parser.add_argument('--candidate_file', type=str,
                        default='data/uspto_full/candidates.txt')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_config()
    target = set()

    with open(args.train_target, encoding='utf-8') as f:
        for line in f.readlines():
            target.add(line.rstrip())

    with open(args.val_target, encoding='utf-8') as f:
        for line in f.readlines():
            target.add(line.rstrip())

    with open(args.candidate_file, 'w', encoding='utf-8') as f:
        for t in target:
            f.writelines(t + '\n')

    print(f'Collect {len(target)} candidates in {args.candidate_file}.')


import cv2
img1=cv2.imread('1.jpg')
img2=cv2.imread('2.jpg')
#使用SIFT算法获取图像特征的关键点和描述符
sift=cv2.xfeatures2d.SIFT_create()
kp1,des1=sift.detectAndCompute(img1,None)
kp2,des2=sift.detectAndCompute(img2,None)

#定义FLANN匹配器
indexParams=dict(algorithm=0,trees=10)
searchParams=dict(checks=50)
flann=cv2.FlannBasedMatcher(indexParams,searchParams)
#使用KNN算法实现图像匹配，并对匹配结果排序
matches=flann.knnMatch(des1,des2,k=2)
matches=sorted(matches,key=lambda x:x[0].distance)

#去除错误匹配，0.5是系数，系数大小不同，匹配的结果页不同
goodMatches=[]
for m,n in matches:
    if m.distance<0.5*n.distance:
        goodMatches.append(m)

#获取某个点的坐标位置
#index是获取匹配结果的中位数
index=int(len(goodMatches)/2)
#queryIdx是目标图像的描述符索引
x,y=kp1[goodMatches[index].queryIdx].pt
#将坐标位置勾画在2.png图片上，并显示
cv2.rectangle(img1,(int(x),int(y)),(int(x)+5,int(y)+5),(0,255,0),2)
cv2.imshow('baofeng',img1)
cv2.waitKey()


import numpy as npy
import cvxpy as cvx
import matplotlib as mpl
mpl.use('QT4Agg')
import matplotlib.pyplot as plt

def p_fcn(u):
    tmp = npy.zeros(u.shape)
    for idx in npy.arange(u.size):
        if u[idx] <= -1:
            tmp[idx] = float("-inf")
        elif u[idx] <= 8:
            tmp[idx] = 11 + u[idx] - 6 * npy.sqrt(1+u[idx])
        else:
            tmp[idx] = 1
    return tmp

N = 100
u = npy.linspace(-2, 10, num = N)
p = p_fcn(u)
plt.plot(u, p, label = 'p*(u)')
xli = plt.gca().get_xlim();
plt.plot(0, 5, marker = 'o', label = 'p*(0)')
plt.xlabel('u')
plt.legend()
plt.show()


from collections import defaultdict
from collections import OrderedDict
from encoded.cart_view import CartWithElements
from encoded.reports.constants import ANNOTATION_METADATA_COLUMN_TO_FIELDS_MAPPING
from encoded.reports.constants import METADATA_ALLOWED_TYPES
from encoded.reports.constants import METADATA_COLUMN_TO_FIELDS_MAPPING
from encoded.reports.constants import METADATA_AUDIT_TO_AUDIT_COLUMN_MAPPING
from encoded.reports.constants import PUBLICATION_DATA_METADATA_COLUMN_TO_FIELDS_MAPPING
from encoded.reports.constants import SERIES_METADATA_COLUMN_TO_FIELDS_MAPPING
from encoded.reports.constants import METADATA_SERIES_TYPES
from encoded.reports.csv import CSVGenerator
from encoded.reports.decorators import allowed_types
from encoded.reports.inequalities import map_param_values_to_inequalities
from encoded.reports.inequalities import try_to_evaluate_inequality
from encoded.reports.search import BatchedSearchGenerator
from encoded.reports.serializers import make_experiment_cell
from encoded.reports.serializers import make_file_cell
from encoded.reports.serializers import map_strings_to_booleans_and_ints
from encoded.search_views import search_generator
from encoded.vis_defines import is_file_visualizable
from pyramid.httpexceptions import HTTPBadRequest
from pyramid.response import Response
from pyramid.view import view_config
from snosearch.interfaces import EXISTS
from snosearch.interfaces import MUST
from snosearch.interfaces import RANGES
from snosearch.parsers import QueryString
from snovault.util import simple_path_ids

def includeme(config):
    config.add_route('metadata', '/metadata{slash:/?}')
    config.scan(__name__)

def file_matches_file_params(file_, positive_file_param_set):
    # Expects file_param_set where FILES_PREFIX (e.g. 'files.') has been
    # stripped off of key (files.file_type -> file_type)
    # and params with field negation (i.e. file_type!=bigWig)
    # have been filtered out. Param values should be
    # coerced to ints ('2' -> 2) or booleans ('true' -> True)
    # and put into a set for comparison with file values.
    for field, set_of_param_values in positive_file_param_set.items():
        file_value = list(simple_path_ids(file_, field))
        if not file_value:
            return False
        if '*' in set_of_param_values:
            continue
        if not set_of_param_values.intersection(file_value):
            return False
    return True

def some_value_satisfies_inequalities(values, inequalities):
    return all(
        any(
            try_to_evaluate_inequality(inequality, value)
            for value in values
        )
        for inequality in inequalities
    )

def file_satisfies_inequality_constraints(file_, positive_file_inequalities):
    for field, inequalities in positive_file_inequalities.items():
        file_value = list(simple_path_ids(file_, field))
        if not file_value:
            return False
        if not some_value_satisfies_inequalities(file_value, inequalities):
            return False
    return True

def group_audits_by_files_and_type(audits):
    grouped_file_audits = defaultdict(lambda: defaultdict(list))
    grouped_other_audits = defaultdict(list)
    for audit_type, audit_column in METADATA_AUDIT_TO_AUDIT_COLUMN_MAPPING:
        for audit in audits.get(audit_type, []):
            path = audit.get('path')
            if '/files/' in path:
                grouped_file_audits[path][audit_type].append(audit.get('category'))
            else:
                grouped_other_audits[audit_type].append(audit.get('category'))
    return grouped_file_audits, grouped_other_audits

class MetadataReport:

    SEARCH_PATH = '/search/'
    EXCLUDED_COLUMNS = (
        'Restricted',
        'No File Available',
    )
    DEFAULT_PARAMS = [
        ('field', 'audit'),
        ('field', 'files.@id'),
        ('field', 'files.restricted'),
        ('field', 'files.no_file_available'),
        ('field', 'files.file_format'),
        ('field', 'files.file_format_type'),
        ('field', 'files.status'),
        ('field', 'files.assembly'),
        ('limit', 'all'),
    ]
    CONTENT_TYPE = 'text/tsv'
    CONTENT_DISPOSITION = 'attachment; filename="metadata.tsv"'
    FILES_PREFIX = 'files.'

    def __init__(self, request):
        self.request = request
        self.query_string = QueryString(request)
        self.param_list = self.query_string.group_values_by_key()
        self.split_file_filters = {}
        self.positive_file_param_set = {}
        self.positive_file_inequalities = {}
        self.header = []
        self.experiment_column_to_fields_mapping = OrderedDict()
        self.file_column_to_fields_mapping = OrderedDict()
        self.visualizable_only = self.query_string.is_param('option', 'visualizable')
        self.raw_only = self.query_string.is_param('option', 'raw')
        self.csv = CSVGenerator()

    def _get_column_to_fields_mapping(self):
        return METADATA_COLUMN_TO_FIELDS_MAPPING

    def _build_header(self):
        for column in self._get_column_to_fields_mapping():
            if column not in self.EXCLUDED_COLUMNS:
                self.header.append(column)
        for audit, column in METADATA_AUDIT_TO_AUDIT_COLUMN_MAPPING:
            self.header.append(column)

    def _split_column_and_fields_by_experiment_and_file(self):
        for column, fields in self._get_column_to_fields_mapping().items():
            if fields[0].startswith(self.FILES_PREFIX):
                self.file_column_to_fields_mapping[column] = [
                    field.replace(self.FILES_PREFIX, '')
                    for field in fields
                ]
            else:
                self.experiment_column_to_fields_mapping[column] = fields

    def _set_split_file_filters(self):
        file_params = self.query_string.get_filters_by_condition(
            key_and_value_condition=lambda k, _: k.startswith(self.FILES_PREFIX)
        )
        self.split_file_filters = self.query_string.split_filters(
            params=file_params
        )

    def _set_positive_file_param_set(self):
        grouped_positive_file_params = self.query_string.group_values_by_key(
            params=self.split_file_filters[MUST] + self.split_file_filters[EXISTS]
        )
        self.positive_file_param_set = {
            k.replace(self.FILES_PREFIX, ''): set(map_strings_to_booleans_and_ints(v))
            for k, v in grouped_positive_file_params.items()
        }

    def _set_positive_file_inequalities(self):
        grouped_positive_file_inequalities = self.query_string.group_values_by_key(
            params=self.split_file_filters[RANGES]
        )
        self.positive_file_inequalities = {
            k.replace(self.FILES_PREFIX, ''): map_param_values_to_inequalities(v)
            for k, v in grouped_positive_file_inequalities.items()
        }

    def _add_positive_file_filters_as_fields_to_param_list(self):
        self.param_list['field'] = self.param_list.get('field', [])
        self.param_list['field'].extend(
            (
                k
                for k, v in self.query_string._get_original_params()
                if k.startswith(self.FILES_PREFIX) and '!' not in k
            )
        )

    def _add_fields_to_param_list(self):
        self.param_list['field'] = self.param_list.get('field', [])
        for column, fields in self._get_column_to_fields_mapping().items():
            self.param_list['field'].extend(fields)
        self._add_positive_file_filters_as_fields_to_param_list()

    def _initialize_at_id_param(self):
        self.param_list['@id'] = self.param_list.get('@id', [])

    def _maybe_add_cart_elements_to_param_list(self):
        # Don't need to limit max_cart_elements here since
        # search is batched.
        cart = CartWithElements(
            self.request,
            max_cart_elements=None
        )
        self.param_list['@id'].extend(cart.elements)
        self.param_list.pop('cart', None)

    def _get_json_elements_or_empty_list(self):
        try:
            return self.request.json.get('elements', [])
        except ValueError:
            return []

    def _maybe_add_json_elements_to_param_list(self):
        self.param_list['@id'].extend(
            self._get_json_elements_or_empty_list()
        )

    def _get_field_params(self):
        return [
            ('field', p)
            for p in self.param_list.get('field', [])
        ]

    def _get_at_id_params(self):
        return [
            ('@id', p)
            for p in self.param_list.get('@id', [])
        ]

    def _get_default_params(self):
        return self.DEFAULT_PARAMS

    def _build_query_string(self):
        self.query_string.drop('limit')
        self.query_string.drop('option')
        self.query_string.extend(
            self._get_default_params()
            + self._get_field_params()
            + self._get_at_id_params()
        )

    def _get_search_path(self):
        return self.SEARCH_PATH

    def _build_new_request(self):
        self._build_query_string()
        request = self.query_string.get_request_with_new_query_string()
        request.path_info = self._get_search_path()
        return request

    def _get_search_results_generator(self):
        return BatchedSearchGenerator(
            self._build_new_request()
        ).results()

    def _should_not_report_file(self, file_):
        conditions = [
            not file_matches_file_params(file_, self.positive_file_param_set),
            not file_satisfies_inequality_constraints(file_, self.positive_file_inequalities),
            self.visualizable_only and not is_file_visualizable(file_),
            self.raw_only and file_.get('assembly'),
            file_.get('restricted'),
            file_.get('no_file_available'),
        ]
        return any(conditions)

    def _get_experiment_data(self, experiment):
        return {
            column: make_experiment_cell(fields, experiment)
            for column, fields in self.experiment_column_to_fields_mapping.items()
        }

    def _get_file_data(self, file_):
        file_['href'] = self.request.host_url + file_['href']
        return {
            column: make_file_cell(fields, file_)
            for column, fields in self.file_column_to_fields_mapping.items()
        }

    def _get_audit_data(self, grouped_audits_for_file, grouped_other_audits):
        return {
            audit_column: ', '.join(
                set(
                    grouped_audits_for_file.get(audit_type, [])
                    + grouped_other_audits.get(audit_type, [])
                )
            ) for audit_type, audit_column in METADATA_AUDIT_TO_AUDIT_COLUMN_MAPPING
        }

    def _output_sorted_row(self, experiment_data, file_data):
        row = []
        for column in self.header:
            row.append(
                file_data.get(
                    column,
                    experiment_data.get(column)
                )
            )
        return row

    def _generate_rows(self):
        yield self.csv.writerow(self.header)
        for experiment in self._get_search_results_generator():
            if not experiment.get('files', []):
                continue
            grouped_file_audits, grouped_other_audits = group_audits_by_files_and_type(
                experiment.get('audit', {})
            )
            experiment_data = self._get_experiment_data(experiment)
            for file_ in experiment.get('files', []):
                if self._should_not_report_file(file_):
                    continue
                file_data = self._get_file_data(file_)
                audit_data = self._get_audit_data(
                    grouped_file_audits.get(file_.get('@id'), {}),
                    grouped_other_audits
                )
                file_data.update(audit_data)
                yield self.csv.writerow(
                    self._output_sorted_row(experiment_data, file_data)
                )

    def _validate_request(self):
        type_params = self.param_list.get('type', [])
        if len(type_params) != 1:
            raise HTTPBadRequest(explanation='URL requires one "type" parameter.')
        return True

    def _initialize_report(self):
        self._build_header()
        self._split_column_and_fields_by_experiment_and_file()
        self._set_split_file_filters()
        self._set_positive_file_param_set()
        self._set_positive_file_inequalities()

    def _build_params(self):
        self._add_fields_to_param_list()
        self._initialize_at_id_param()
        self._maybe_add_cart_elements_to_param_list()
        self._maybe_add_json_elements_to_param_list()

    def generate(self):
        self._validate_request()
        self._initialize_report()
        self._build_params()
        return Response(
             content_type=self.CONTENT_TYPE,
             app_iter=self._generate_rows(),
             content_disposition=self.CONTENT_DISPOSITION,
        )

class AnnotationMetadataReport(MetadataReport):

    def _get_column_to_fields_mapping(self):
        return ANNOTATION_METADATA_COLUMN_TO_FIELDS_MAPPING

class PublicationDataMetadataReport(MetadataReport):
    '''
    PublicationData objects don't embed file attributes so
    we have to get file metadata with separate search request.
    We try to get all the file metadata together in a batched request
    instead of making a request for every file. This requires some
    extra machinery compared to normal MetdataReport.
    '''

    DEFAULT_PARAMS = [
        ('limit', 'all'),
        ('field', 'files')
    ]
    DEFAULT_FILE_PARAMS = [
        ('type', 'File'),
        ('limit', 'all'),
        ('field', '@id'),
        ('field', 'href'),
        ('field', 'restricted'),
        ('field', 'no_file_available'),
        ('field', 'file_format'),
        ('field', 'file_format_type'),
        ('field', 'status'),
        ('field', 'assembly'),
    ]

    def __init__(self, request):
        super().__init__(request)
        self.file_query_string = QueryString(request)
        self.file_params = []
        self.file_at_ids = []

    # Overrides parent.
    def _get_column_to_fields_mapping(self):
        return PUBLICATION_DATA_METADATA_COLUMN_TO_FIELDS_MAPPING

    # Overrides parent.
    def _build_header(self):
        for column in self._get_column_to_fields_mapping():
            if column not in self.EXCLUDED_COLUMNS:
                self.header.append(column)

    # Overrides parent.
    def _add_fields_to_param_list(self):
        self.param_list['field'] = []
        for column, fields in self.experiment_column_to_fields_mapping.items():
            self.param_list['field'].extend(fields)

    def _add_default_file_params_to_file_params(self):
        self.file_params.extend(self.DEFAULT_FILE_PARAMS)

    def _add_report_file_fields_to_file_params(self):
        for column, fields in self.file_column_to_fields_mapping.items():
            self.file_params.extend(
                [
                    ('field', field)
                    for field in fields
                ]
            )

    def _convert_experiment_params_to_file_params(self):
        return [
            (k.replace(self.FILES_PREFIX, ''), v)
            for k, v in self.query_string._get_original_params()
            if k.startswith(self.FILES_PREFIX)
        ]

    def _add_experiment_file_filters_as_fields_to_file_params(self):
        self.file_params.extend(
            ('field', k)
            for k, v in self._convert_experiment_params_to_file_params()
        )

    def _add_experiment_file_filters_to_file_params(self):
        self.file_params.extend(
            self._convert_experiment_params_to_file_params()
        )

    def _build_file_params(self):
        self._add_default_file_params_to_file_params()
        self._add_report_file_fields_to_file_params()
        self._add_experiment_file_filters_as_fields_to_file_params()
        self._add_experiment_file_filters_to_file_params()

    def _filter_file_params_from_query_string(self):
        self.query_string.params = [
            (k, v)
            for k, v in self.query_string.params
            if not k.startswith(self.FILES_PREFIX)
        ]

    # Overrides parent.
    def _build_params(self):
        super()._build_params()
        self._build_file_params()
        self._filter_file_params_from_query_string()

    def _get_at_id_file_params(self):
        return [
            ('@id', file_at_id)
            for file_at_id in self.file_at_ids
        ]

    def _build_new_file_request(self):
        self.file_query_string.params = (
            self.file_params + self._get_at_id_file_params()
        )
        request = self.file_query_string.get_request_with_new_query_string()
        request.path_info = self._get_search_path()
        return request

    def _get_file_search_results_generator(self):
        request = self._build_new_file_request()
        bsg = BatchedSearchGenerator(request)
        return bsg.results()

    # Overrides parent.
    def _generate_rows(self):
        yield self.csv.writerow(self.header)
        for experiment in self._get_search_results_generator():
            self.file_at_ids = experiment.get('files', [])
            if not self.file_at_ids:
                continue
            experiment_data = self._get_experiment_data(experiment)
            for file_ in self._get_file_search_results_generator():
                if self._should_not_report_file(file_):
                    continue
                file_data = self._get_file_data(file_)
                yield self.csv.writerow(
                    self._output_sorted_row(experiment_data, file_data)
                )

class SeriesMetadataReport(MetadataReport):

    DEFAULT_PARAMS = [
        ('limit', 'all'),
        ('field', 'series_files.@id'),
        ('field', 'series_files.href'),
        ('field', 'series_files.restricted'),
        ('field', 'series_files.no_file_available'),
        ('field', 'series_files.file_format'),
        ('field', 'series_files.file_format_type'),
        ('field', 'series_files.preferred_default'),
        ('field', 'series_files.status'),
        ('field', 'series_files.assembly'),
    ]
    FILES_PREFIX = 'series_files.'

    def _get_column_to_fields_mapping(self):
        return SERIES_METADATA_COLUMN_TO_FIELDS_MAPPING

    def _generate_rows(self):
        yield self.csv.writerow(self.header)
        for series in self._get_search_results_generator():
            series_data = self._get_experiment_data(series)
            for file_ in series.get('series_files', []):
                if self._should_not_report_file(file_):
                    continue
                file_data = self._get_file_data(file_)
                yield self.csv.writerow(
                    self._output_sorted_row(series_data, file_data)
                )

def _get_metadata(context, request):
    metadata_report = MetadataReport(request)
    return metadata_report.generate()

def _get_annotation_metadata(context, request):
    annotation_metadata_report = AnnotationMetadataReport(request)
    return annotation_metadata_report.generate()

def _get_publication_data_metadata(context, request):
    publication_data_metadata_report = PublicationDataMetadataReport(request)
    return publication_data_metadata_report.generate()

def _get_series_metadata(context, request):
    series_metadata_report = SeriesMetadataReport(request)
    return series_metadata_report.generate()

def metadata_report_factory(context, request):
    qs = QueryString(request)
    specified_type = qs.get_one_value(
            params=qs.get_type_filters()
    )
    if specified_type == 'Annotation':
        return _get_annotation_metadata(context, request)
    elif specified_type == 'PublicationData':
        return _get_publication_data_metadata(context, request)
    elif specified_type in METADATA_SERIES_TYPES:
        return _get_series_metadata(context, request)
    else:
        return _get_metadata(context, request)

@view_config(route_name='metadata', request_method=['GET', 'POST'])
@allowed_types(METADATA_ALLOWED_TYPES)
def metadata_tsv(context, request):
    return metadata_report_factory(context, request)


from advent_of_code import BaseSolution
from typing import List
from .instruction import Instruction
from .sample import Sample
from .device import Device
from .opcode_map import OpcodeMap

class Solution(BaseSolution):
    samples: List[Sample]
    instructions: List[Instruction]

    def setup(self):
        (input_file, ) = self.parameters()
        lines: List[str] = list(self.read_input(input_file))

        self.samples = []
        self.instructions = []

        while len(lines) > 0:
            section = lines.pop(0)
            if section.startswith('Before'):
                before_parts = section.split(': ')[1][1:-1].split(', ')
                before = Device([int(x) for x in before_parts])
                ins = Instruction(*[int(x) for x in lines.pop(0).split(' ')])
                after_parts = lines.pop(0).split(':  ')[1][1:-1].split(', ')
                after = Device([int(x) for x in after_parts])
                self.samples.append(Sample(before, after, ins))
                continue

            ins = Instruction(*[int(x) for x in section.split(' ')])
            self.instructions.append(ins)

    def part1(self):
        samples_with_more_than_two_matches: List[Sample] = []
        for s in self.samples:
            if s.test() > 2:
                samples_with_more_than_two_matches.append(s)
        return len(samples_with_more_than_two_matches)

    def part2(self):
        for s in self.samples:
            s.train()

        final_device = Device()
        for ins in self.instructions:
            OpcodeMap.execute(final_device, ins)

        return final_device.registers[0]


import sys
from typing import Generic, List, MutableMapping, TypeVar, cast

if sys.version_info >= (3, 8):
    from typing import Final
else:
    from typing_extensions import Final

from .term import NonNumTermElem, Term

T = TypeVar("T", bound=NonNumTermElem)

class DefinedItemRegistry(Generic[T]):

    """Registers items by definition."""

    def __init__(self, unique_items: bool = True) -> None:
        self._unique_items: Final = unique_items
        self._item_def_map: MutableMapping[Term[T], int] = {}
        self._item_list: List[List[T]] = []

    def register_item(self, item: T) -> int:
        """Register `item` by its normalized definition.

        Args:
            item (T): defined item to be registered

        Returns:
            int: index of registered item

        Raises:
            ValueError: item with same or equivalent definition already
                registered
        """
        item_norm_def = cast(Term[T], item.normalized_definition)
        try:
            idx = self._item_def_map[item_norm_def]
        except KeyError:
            item_list = self._item_list
            idx = len(item_list)
            item_list.append([item])
            self._item_def_map[item_norm_def] = idx
            return idx
        else:
            reg_item = self._item_list[idx][0]
            if reg_item == item:
                return idx
            elif self._unique_items:
                raise ValueError("Item with same or equivalent definition "
                                 f"already registered: '{reg_item}'.")
            else:
                self._item_list[idx].append(item)
                return idx

    def __getitem__(self, item_def: Term[T]) -> T:
        """Get item by definition.

        Args:
            item_def (Term[T]): definition of item to be looked-up

        Returns:
            T: item registered with a definition equivalent
                to `item_def`

        Raises:
            KeyError: no item registered with definition equivalent to
                `item_def`
        """
        norm_item_def = item_def.normalized()
        idx = self._item_def_map[norm_item_def]
        return self._item_list[idx][0]

    def __len__(self) -> int:
        """len(self)"""
        return len(self._item_list)


#! /usr/bin/python3

import pygame
import random
import gpiozero
import RPi.GPIO as GPIO
import functions
import time

button3 = 22
GPIO.setup(button3, GPIO.IN, pull_up_down = GPIO.PUD_DOWN)

def define_image(image):
	global electronic_images
	image1 = image + '.png'
	image1 = pygame.image.load(image1).convert_alpha()
	image1 = pygame.transform.scale(image1,(100,100))

	electronic_images.append(image1)

def buttonColors():
	global button1Color, button2Color, button3Color
	button1Color = (0,191,255)
	button2Color = (30,144,255)
	button3Color = (65,105,225)

class Button:
    def __init__(self, text, width, height, pos, color):
        self.top_rect = pygame.Rect(pos,(width,height))
        self.top_color = color
        self.text_surf = gui_font.render(text, 0,(210,255,100))
        self.text_rect = self.text_surf.get_rect(center = self.top_rect.center)

    def draw(self):
        action = False

        mouse_pos = pygame.mouse.get_pos()

        if self.top_rect.collidepoint(mouse_pos):
            if pygame.mouse.get_pressed()[0] == 1:
                action = True

        pygame.draw.rect(screen,self.top_color, self.top_rect, 16)
        screen.blit(self.text_surf, self.text_rect)

        return action

pygame.init()
#screen = pygame.display.set_mode((800,480))
screen = pygame.display.set_mode((800,480), pygame.FULLSCREEN)
clock = pygame.time.Clock()

x = 0
picked = 0
question = 0

white = (255, 255, 255)
red = (255, 51, 58)
button1Color = (0,191,255)
button2Color = (30,144,255)
button3Color = (65,105,225)

buttonNumber = 0

electronic_images = []

global end, ts
end = 0
ts = 0

font = pygame.font.Font('freesansbold.ttf', 100)
gui_font = pygame.font.Font(None,30)

electronics = ['Andorra',
				'Argentina',
				'Armenia',
				'Afghanistan',
				'Australia',
				'Bolivia',
				'Brazil',
				'Belgium',
				'Canada',
				'Cambodia',
				'China',
				'Congo',
				'Cyprus',
				'Denmark',
				'Djibouti',
				'Equador',
				'Egypt',
				'Finland',
				'Figi',
				'France',
				'Georgia',
				'Ghana',
				'Greece',
				'Haiti',
				'Hungary',
				'Honduras',
				'Iceland',
				'Iran',
				'Ireland',
				'Italy',
				'India',
				'Japan',
				'Jordan',
				'Jamaica',
				'Kenya',
				'Kazakhstan',
				'Laos',
				'Liberia',
				'Madagascar',
				'Malta',
				'Mexico',
				'Malawi',
				'Nigeria',
				'Nepal',
				'Oman',
				'Pakistan',
				'Portugal',
				'Panama',
				'Qatar',
				'Romania',
				'Rwanda',
				'Scotland',
				'Spain',
				'Syria',
				'Serbia',
				'Thailand',
				'Tanzania',
				'Uganda',
				'Uruguay',
				'Ukraine',
				'Venezuela',
				'Vietnam',
				'Vanuatu',
				'Yemen',
				'Zambia',
				'Zimbabwe'
				]

for element in electronics:
	image = element.lower()
	define_image(image)

while True:

	buttonOne = Button('', 250, 480, (15,0), button1Color)
	buttonTwo = Button('', 245, 480, (280,0), button2Color)
	buttonThree = Button('',250, 480, (540, 0), button3Color)

	for event in pygame.event.get():
	    if event.type == pygame.QUIT:
	        pygame.quit()
	        sys.exit()
	    if event.type == pygame.KEYDOWN:
	        if event.key == pygame.K_ESCAPE:
	            pygame.quit()
	            sys.exit()

	if GPIO.input(button3) == 1:
	    pygame.quit()
	    sys.exit()

	if question == 0:
		countries = len(electronics)-1
		picked = random.randint(0,countries)
		numbers = [*range(0, countries+1, 1)]

		numbers.remove(picked)
		second = random.choice(numbers)
		numbers.remove(second)
		third = random.choice(numbers)
		question += 1
		all_choice = [picked, second, third]
		random.shuffle(all_choice)
		index = 0
		for element in all_choice:
			if element == picked:
				buttonNumber = index
			index += 1

	textSurface = font.render(electronics[picked], True, (32,178,170),(173, 216, 230))
	textRect = textSurface.get_rect()
	textRect.center = (400, 100)

	screen.fill((173, 216, 230))

	x = 100
	for element in all_choice:
		screen.blit(electronic_images[element], (0+x,250))
		x += 250

	if buttonOne.draw():
		ts = time.time()
		if (ts - end) > 1:
			if buttonNumber == 0:
				question = 0
				buttonColors()
				end = time.time()
			else:
				button1Color = (255,0,0)

	if buttonTwo.draw():
		ts = time.time()
		if (ts - end) > 1:
		    if buttonNumber == 1:
		        question = 0
		        buttonColors()
		        end = time.time()
		    else:
		    	button2Color = (255,0,0)

	if buttonThree.draw():
		ts = time.time()
		if (ts - end) > 1:
		    if buttonNumber == 2:
		        question = 0
		        buttonColors()
		        end = time.time()
		    else:
		    	button3Color = (255,0,0)

	screen.blit(textSurface, textRect)
	pygame.display.update()
	clock.tick(10)


from django.urls import path

from . import views

app_name = 'reviewapp'
urlpatterns = [

    # ex: /reviewapp/
    path('', views.home, name='home'),

    # ex: /reviewapp/resto/5
    path('resto/<int:restaurant_id>/', views.details, name='details'),

    # ex: /reviewapp/resto/5/add
    path('resto/<int:restaurant_id>/add/', views.add, name='add'),

    # ex: /reviewapp/resto/5/reviewed
    path('resto/<int:restaurant_id>/reviewed/', views.reviewed, name='reviewed'),

    # ex: /reviewapp/api/comment/add/
    path('api/comment/add/', views.CommentAdd.as_view(), name="comment"),

    # ex: /reviewapp/api/reply/add/
    path('api/reply/add/', views.ReplyAdd.as_view(), name="reply"),

    # ex: /reviewapp/api/comment/add/
    path('api/like/add/', views.LikeAdd.as_view(), name="like"),

    # ex: /reviewapp/api/restaurants/list/
    path('api/restaurants/list/', views.GetRestaurantsByCategory.as_view(), name="restaurants_by_category"),

]


import numpy as np
from matplotlib import pyplot as plt

indices = np.zeros((256,256,3))
values = np.zeros((256,256,3))

for i in range(3):
    p1 = np.random.random((1,256,256,2))
    indices_p1 = np.argmax(p1, axis=-1)
    indices_p1[indices_p1 != 0] += i
    values_p1 = np.max(p1, axis=-1)
    indices_p1 = indices_p1[0,:,:]
    values_p1 = values_p1[0,:,:]
    indices[:,:,i] = indices_p1
    values[:,:,i] = values_p1

prediction = np.zeros((256,256))

for i in range(256):
    for j in range(256):
        if np.sum(indices[i,j]) == 0:
            prediction[i,j] = 0
        else:
            max_class = 0
            max_value = 0
            for k in range(3):
                if values[i,j,k] > max_value and indices[i,j,k] != 0:
                    max_class = indices[i,j,k]
                    max_value = values[i,j,k]
            prediction[i,j] = max_class

plt.imshow(prediction)
plt.show()


# -*- coding: utf-8 -*-
# @Time    : 2021/12/31 16:51
# @Author  : Yaojie Shen
# @Project : AutoTransition
# @File    : checkpoint.py

import os
import torch
import logging
from typing import *
from collections import OrderedDict

logger = logging.getLogger(__name__)

def auto_resume(ckpt_folder):
    try:
        ckpt_files = [ckpt for ckpt in os.listdir(ckpt_folder) if ckpt.endswith(".pth")]
    except FileNotFoundError:
        ckpt_files = []
    if len(ckpt_files) > 0:
        return max([os.path.join(ckpt_folder, file) for file in ckpt_files], key=os.path.getmtime)
    else:
        return None

def save_checkpoint(ckpt_folder, epoch, model, optimizer, scheduler, config, prefix="checkpoint"):
    stat_dict = {
        "epoch": epoch,
        "model": model.state_dict(),
        "optimizer": optimizer.state_dict(),
        "scheduler": scheduler.state_dict() if scheduler is not None else "",
        "config": config
    }
    ckpt_path = os.path.join(ckpt_folder, f"{prefix}_{epoch}.pth")
    os.makedirs(ckpt_folder, exist_ok=True)
    torch.save(stat_dict, ckpt_path)
    return ckpt_path

def load_checkpoint(ckpt_file, model: torch.nn.Module, optimizer: Union[torch.optim.Optimizer, None], scheduler: Any,
                    restart_train=False, rewrite: Tuple[str, str] = None):
    state_dict = torch.load(ckpt_file, map_location="cpu")
    if rewrite is not None:
        logger.info("rewrite model checkpoint prefix: %s->%s", *rewrite)
        state_dict["model"] = {k.replace(*rewrite) if k.startswith(rewrite[0]) else k: v
                               for k, v in state_dict["model"].items()}
    try:
        missing = model.load_state_dict(state_dict["model"], strict=False)
        logger.debug(f"checkpoint key missing: {missing}")
    except RuntimeError:
        print("fail to directly recover from checkpoint, try to match each layers...")
        net_dict = model.state_dict()
        print("find %s layers", len(state_dict["model"].items()))
        missing_keys = [k for k, v in state_dict["model"].items() if k not in net_dict or net_dict[k].shape != v.shape]
        print("missing key: %s", missing_keys)
        state_dict["model"] = {k: v for k, v in state_dict["model"].items() if
                               (k in net_dict and net_dict[k].shape == v.shape)}
        print("resume %s layers from checkpoint", len(state_dict["model"].items()))
        net_dict.update(state_dict["model"])
        model.load_state_dict(OrderedDict(net_dict))

    if not restart_train:
        if optimizer is not None and state_dict["optimizer"]:
            optimizer.load_state_dict(state_dict["optimizer"])
        if scheduler is not None and state_dict["scheduler"]:
            scheduler.load_state_dict(state_dict["scheduler"])
        epoch = state_dict["epoch"]
    else:
        logger.info("restart train, optimizer and scheduler will not be resumed")
        epoch = 0

    del state_dict
    torch.cuda.empty_cache()
    return epoch  # start epoch


# Deep Learning
#
# Assignment 3
#
# Previously in 2_fullyconnected.ipynb, you trained a logistic regression and a neural network model.
#
# The goal of this assignment is to explore regularization techniques.

# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
from copy import copy

# First reload the data we generated in `1_notmnist.ipynb`.

pickle_file = 'notMNIST.pickle'

with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    train_dataset = save['train_dataset']
    train_labels = save['train_labels']
    valid_dataset = save['valid_dataset']
    valid_labels = save['valid_labels']
    test_dataset = save['test_dataset']
    test_labels = save['test_labels']
    del save  # hint to help gc free up memory
    print('Training set', train_dataset.shape, train_labels.shape)
    print('Validation set', valid_dataset.shape, valid_labels.shape)
    print('Test set', test_dataset.shape, test_labels.shape)

# Reformat into a shape that's more adapted to the models we're going to train:
# - data as a flat matrix,
# - labels as float 1-hot encodings.

image_size = 28
num_labels = 10

def reformat(dataset, labels):
    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
    # Map 1 to [0.0, 1.0, 0.0 ...], 2 to [0.0, 0.0, 1.0 ...]
    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)
    return dataset, labels

train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)

def accuracy(predictions, labels):
    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
            / predictions.shape[0])

# ---
# Problem 1
# ---------
#
# Introduce and tune L2 regularization for both logistic and neural network models. Remember that L2 amounts to adding
# a penalty on the norm of the weights to the loss. In TensorFlow, you can compute the L2 loss for a tensor `t` using
# `nn.l2_loss(t)`. The right amount of regularization should improve your validation / test accuracy.
#
# ---

batch_size = 128
num_steps = 5000
hidden_layers = 1028
learning_rate = 0.5
l2_constant = 0.001

# graph = tf.Graph()
# with graph.as_default():
#     train = tf.placeholder(tf.float32, [batch_size, image_size * image_size])
#     labels = tf.placeholder(tf.float32, [batch_size, num_labels])
#     valid = tf.constant(valid_dataset)
#     test = tf.constant(test_dataset)
#
#     W = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels], mean=0, stddev=0.5))
#     bias = tf.Variable(tf.zeros([num_labels, ]))
#
#     logits = tf.matmul(train, W) + bias
#     loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) +\
#         l2_constant * tf.nn.l2_loss(W)
#
#     optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
#
#     prediction = tf.nn.softmax(logits)
#     valid_prediction = tf.nn.softmax(tf.matmul(valid, W) + bias)
#     test_prediction = tf.nn.softmax(tf.matmul(test, W) + bias)

graph = tf.Graph()
with graph.as_default():
    train = tf.placeholder(tf.float32, [batch_size, image_size * image_size])
    labels = tf.placeholder(tf.float32, [batch_size, num_labels])
    valid = tf.constant(valid_dataset)
    test = tf.constant(test_dataset)

    W1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_layers], mean=0, stddev=0.5))
    bias1 = tf.Variable(tf.zeros([hidden_layers, ]))
    W2 = tf.Variable(tf.truncated_normal([hidden_layers, num_labels], mean=0, stddev=0.5))
    bias2 = tf.Variable(tf.zeros([num_labels, ]))

    layer1_out = tf.nn.relu(tf.matmul(train, W1) + bias1)
    logits = tf.matmul(layer1_out, W2) + bias2
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) +\
        l2_constant * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2))

    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

    prediction = tf.nn.softmax(logits)
    valid_hidden = tf.nn.relu(tf.matmul(valid, W1) + bias1)
    valid_prediction = tf.nn.softmax(tf.matmul(valid_hidden, W2) + bias2)
    test_hidden = tf.nn.relu(tf.matmul(test, W1) + bias1)
    test_prediction = tf.nn.softmax(tf.matmul(test_hidden, W2) + bias2)

# with tf.Session(graph=graph) as session:
#     tf.global_variables_initializer().run()
#     for step in range(num_steps):
#         offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
#
#         batch_data = train_dataset[offset:(offset + batch_size), :]
#         batch_labels = train_labels[offset:(offset + batch_size), :]
#
#         _, l, preds = session.run([optimizer, loss, prediction], feed_dict={train: batch_data, labels: batch_labels})
#         # _, loss = session.run([optimizer, loss], feed_dict={train: batch_data, labels: batch_labels})
#         if step % (num_steps/20) == 0:
#             print('Minibatch loss: {}'.format(l))
#             print('Minibatch valid accuracy: {}'.format(accuracy(valid_prediction.eval(), valid_labels)))
#     print('Test accuracy: {}'.format(accuracy(test_prediction.eval(), test_labels)))

# ---
# Problem 2
# ---------
# Let's demonstrate an extreme case of overfitting. Restrict your training data to just a few batches. What happens?
#
# ---

batch_size = 128
num_steps = 500
hidden_layers = 1028
learning_rate = 0.5
l2_constant = 0.001

graph = tf.Graph()
with graph.as_default():
    train = tf.placeholder(tf.float32, [batch_size, image_size * image_size])
    labels = tf.placeholder(tf.float32, [batch_size, num_labels])
    valid = tf.constant(valid_dataset)
    test = tf.constant(test_dataset)

    W1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_layers], mean=0, stddev=0.5))
    bias1 = tf.Variable(tf.zeros([hidden_layers, ]))
    W2 = tf.Variable(tf.truncated_normal([hidden_layers, num_labels], mean=0, stddev=0.5))
    bias2 = tf.Variable(tf.zeros([num_labels, ]))

    layer1_out = tf.nn.relu(tf.matmul(train, W1) + bias1)
    logits = tf.matmul(layer1_out, W2) + bias2
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) +\
        l2_constant * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2))

    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

    prediction = tf.nn.softmax(logits)
    valid_hidden = tf.nn.relu(tf.matmul(valid, W1) + bias1)
    valid_prediction = tf.nn.softmax(tf.matmul(valid_hidden, W2) + bias2)
    test_hidden = tf.nn.relu(tf.matmul(test, W1) + bias1)
    test_prediction = tf.nn.softmax(tf.matmul(test_hidden, W2) + bias2)

# with tf.Session(graph=graph) as session:
#     tf.global_variables_initializer().run()
#     for step in range(num_steps):
#         offset = 128 * (num_steps % 4)
#
#         batch_data = train_dataset[offset:(offset + batch_size), :]
#         batch_labels = train_labels[offset:(offset + batch_size), :]
#
#         _, l, preds = session.run([optimizer, loss, prediction], feed_dict={train: batch_data, labels: batch_labels})
#         # _, loss = session.run([optimizer, loss], feed_dict={train: batch_data, labels: batch_labels})
#         if step % (num_steps/20) == 0:
#             print('Minibatch loss: {}'.format(l))
#             print('Minibatch accuracy: {}'.format(accuracy(preds, batch_labels)))
#             print('Minibatch valid accuracy: {}'.format(accuracy(valid_prediction.eval(), valid_labels)))
#     print('Test accuracy: {}'.format(accuracy(test_prediction.eval(), test_labels)))

# ---
# Problem 3
# ---------
# Introduce Dropout on the hidden layer of the neural network. Remember: Dropout should only be introduced during
# training, not evaluation, otherwise your evaluation results would be stochastic as well. TensorFlow provides
# `nn.dropout()` for that, but you have to make sure it's only inserted during training.
#
# What happens to our extreme overfitting case?
#
# ---

batch_size = 128
num_steps = 5000
hidden_layers = 1028
learning_rate = 0.5
l2_constant = 0.001
drop_prob = 0.9

import time
start = time.time()

graph = tf.Graph()
with graph.as_default():
    train = tf.placeholder(tf.float32, [batch_size, image_size * image_size])
    labels = tf.placeholder(tf.float32, [batch_size, num_labels])
    valid = tf.constant(valid_dataset)
    test = tf.constant(test_dataset)

    W1 = tf.Variable(tf.truncated_normal([image_size * image_size, hidden_layers], mean=0, stddev=0.5))
    bias1 = tf.Variable(tf.zeros([hidden_layers, ]))
    W2 = tf.Variable(tf.truncated_normal([hidden_layers, num_labels], mean=0, stddev=0.5))
    bias2 = tf.Variable(tf.zeros([num_labels, ]))

    layer1_out = tf.nn.relu(tf.nn.dropout(tf.matmul(train, W1), drop_prob) + bias1)
    logits = tf.nn.dropout(tf.matmul(layer1_out, W2), drop_prob) + bias2
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)) +\
        l2_constant * (tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2))

    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

    batch_hidden = tf.nn.relu(tf.matmul(train, W1) + bias1)
    batch_prediction = tf.nn.softmax(tf.matmul(batch_hidden, W2) + bias2)
    valid_hidden = tf.nn.relu(tf.matmul(valid, W1) + bias1)
    valid_prediction = tf.nn.softmax(tf.matmul(valid_hidden, W2) + bias2)
    test_hidden = tf.nn.relu(tf.matmul(test, W1) + bias1)
    test_prediction = tf.nn.softmax(tf.matmul(test_hidden, W2) + bias2)

with tf.Session(graph=graph) as session:
    tf.global_variables_initializer().run()
    for step in range(num_steps):
        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)

        batch_data = train_dataset[offset:(offset + batch_size), :]
        batch_labels = train_labels[offset:(offset + batch_size), :]

        _, l = session.run([optimizer, loss], feed_dict={train: batch_data, labels: batch_labels})
        if step % (num_steps/10) == 0:
            print('Minibatch loss: {}'.format(l))
            print('Validation accuracy: {}'.format(accuracy(valid_prediction.eval(), valid_labels)))
    print('Test accuracy: {}'.format(accuracy(test_prediction.eval(), test_labels)))

print('Processed in {} minutes'.format(round((time.time() - start)/60, 2)))

# 94.05 test:
batch_size = 128
num_steps = 5000
hidden_layers = 1028
learning_rate = 0.5
l2_constant = 0.001
drop_prob = 0.9

# ---
# Problem 4
# ---------
#
# Try to get the best performance you can using a multi-layer model! The best reported test accuracy using a deep
# network is [97.1%]
# (http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html?showComment=1391023266211#c8758720086795711595).
#
# One avenue you can explore is to add multiple layers.
#
# Another one is to use learning rate decay:
#
#     global_step = tf.Variable(0)  # count the number of steps taken.
#     learning_rate = tf.train.exponential_decay(0.5, global_step, ...)
#     optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
#
#  ---

batch_size = 128
num_steps = 5000
hidden_layers = [256]
learning_rate = 0.5
l2_constant = 0.001
drop_prob = 0.9

def evaluate_network(hidden_layers, batch_size=128, num_steps=5000, learning_rate=0.5, l2_constant=0.001, drop_prob=0.9,
                     store_results=20):
    batch_error = []
    valid_error = []
    graph = tf.Graph()
    with graph.as_default():
        train = tf.placeholder(tf.float32, [batch_size, image_size * image_size])
        train_layers = [train]
        labels = tf.placeholder(tf.float32, [batch_size, num_labels])
        valid = tf.constant(valid_dataset)
        test = tf.constant(test_dataset)

        weight_sizes = [image_size * image_size] + hidden_layers + [num_labels]
        W = [tf.Variable(tf.truncated_normal([size_in, size_out], mean=0, stddev=0.5))
             for size_in, size_out in zip(weight_sizes[:-1], weight_sizes[1:])]
        bias_sizes = hidden_layers + [num_labels]
        bias = [tf.Variable(tf.zeros([size, ])) for size in bias_sizes]

        valid_layer = valid
        test_layer = test
        for i in range(len(W)):
            if i < (len(W) - 1):
                train_layer = tf.nn.relu(tf.nn.dropout(tf.matmul(train_layers[i], W[i]), drop_prob) + bias[i])
                train_layers = train_layers + [train_layer]
                valid_layer = tf.nn.relu(tf.matmul(valid_layer, W[i]) + bias[i])
                test_layer = tf.nn.relu(tf.matmul(test_layer, W[i]) + bias[i])
            else:
                train_layer = tf.nn.dropout(tf.matmul(train_layers[i], W[i]), drop_prob) + bias[i]
                train_layers = train_layers + [train_layer]
                valid_layer = tf.matmul(valid_layer, W[i]) + bias[i]
                test_layer = tf.matmul(test_layer, W[i]) + bias[i]
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=train_layers[-1])) +\
            l2_constant * sum(tf.nn.l2_loss(W_i) for W_i in W)

        optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

        batch_prediction = tf.nn.softmax(train_layers[-1])
        valid_prediction = tf.nn.softmax(valid_layer)
        test_prediction = tf.nn.softmax(test_layer)

    with tf.Session(graph=graph) as session:
        tf.global_variables_initializer().run()
        for step in range(num_steps):
            offset = (step * batch_size) % (train_labels.shape[0] - batch_size)

            batch_data = train_dataset[offset:(offset + batch_size), :]
            batch_labels = train_labels[offset:(offset + batch_size), :]

            _, l, preds = session.run([optimizer, loss, batch_prediction],
                                      feed_dict={train: batch_data, labels: batch_labels})
            if step % (num_steps/store_results) == 0:
                # print('Minibatch loss: {}'.format(l))
                # print('Validation accuracy: {}'.format(accuracy(valid_prediction.eval(), valid_labels)))
                batch_error += [accuracy(preds, batch_labels)]
                valid_error += [accuracy(valid_prediction.eval(), valid_labels)]
        test_accuracy = accuracy(test_prediction.eval(), test_labels)
        print('Test accuracy: {}'.format(test_accuracy))

    return batch_error, valid_error, test_accuracy

# Test Loss: 94.4
batch_size = 128; num_steps = 5000; hidden_layers = [1028]; learning_rate = 0.5; l2_constant = 0.001; drop_prob = 0.9

layers = [
    [128], [256], [512], [1024], [2048],
    [128, 128], [128*2, 128], [128*3, 128],
    [256, 256], [256*2, 256], [256*3, 256],
    [512, 512], [512 * 2, 512], [512 * 3, 512],
    [2048, 1024, 512, 256]
]

params = [{'hidden_layers': l, 'learning_rate': 0.5 / (2**(len(l) - 1))} for l in layers]
smaller_lr = [{'hidden_layers': x['hidden_layers'], 'learning_rate': x['learning_rate'] / 5} for x in params]
params = params + smaller_lr

test_error = map(lambda x: evaluate_network(**x), params)

[x[2] for x in test_error]


import os
import urllib

import webapp2
import jinja2

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

class MainPage(webapp2.RequestHandler):

    def get(self):
        template = JINJA_ENVIRONMENT.get_template('index.html')
        self.response.write(template.render())

class UserInterfacePage(webapp2.RequestHandler):

    def get(self):
        template = JINJA_ENVIRONMENT.get_template('userInterface.html')
        self.response.write(template.render())

class FAQPage(webapp2.RequestHandler):

    def get(self):
        template = JINJA_ENVIRONMENT.get_template('FAQ.html')
        self.response.write(template.render())

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/UserInterface', UserInterfacePage),
    ('/FAQ', FAQPage),
], debug=True)


# -*- coding:utf-8 -*-

from setuptools import find_packages
from setuptools import setup

version = '1.0b4.dev0'
long_description = (
    open('README.rst').read() + '\n' +
    open('CONTRIBUTORS.rst').read() + '\n' +
    open('CHANGES.rst').read()
)

setup(name='collective.portlet.calendar',
      version=version,
      description="Extended Calendar Portlet: A configurable implementation of a Calendar Portlet for Plone.",
      long_description=long_description,
      classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Framework :: Plone",
        "Framework :: Plone :: 4.2",
        "Framework :: Plone :: 4.3",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: GNU General Public License (GPL)",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Topic :: Internet :: WWW/HTTP",
        "Topic :: Software Development :: Libraries :: Python Modules",
      ],
      keywords='plone portlet calendar',
      author='Simples Consultoria',
      author_email='products@simplesconsultoria.com.br',
      url='https://github.com/collective/collective.portlet.calendar',
      license='GPL',
      packages=find_packages(exclude=['ez_setup']),
      namespace_packages=['collective', 'collective.portlet'],
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'plone.app.collection',
          'plone.app.portlets',
          'plone.memoize',
          'plone.portlets',
          'Products.CMFCore',
          'Products.GenericSetup',
          'setuptools',
          'zope.component',
          'zope.interface',
          'zope.schema',
      ],
      extras_require={
          'test': [
              'plone.app.testing',
              'plone.browserlayer',
              'plone.testing',
              'unittest2',
          ],
      },
      entry_points="""
      [z3c.autoinclude.plugin]
      target = plone
      """,
      )


from django.core.management.base import BaseCommand
from easy_thumbnails.files import get_thumbnailer

from api.models import Thumbnail

class Command(BaseCommand):
    help = 'Destroys thumbnails when expired'

    def handle(self, *args, **kwargs):
        expired = Thumbnail.objects.expired()
        thumbnails = [item.pk for item in expired]
        for item in expired:
            thumbmanager = get_thumbnailer(item.image)
            thumbmanager.delete_thumbnails()
        expired.delete()
        self.stdout.write('Succesfully deleted {}'.format(thumbnails))


import sqlite3 as sq

f = open("./DATA/Log_data.txt", encoding="utf-8")
data = []

for i in f: data.append(i.split("\n")[0].split("\x02")[1].split("\x03"))

equi = []
sens = []
wind = []
temp = []
stat = []
date = []

cnt = 0

for i in data:

    if cnt == 0:

        equi.append(i[0][0:6])
        sens.append(int(i[0][10:14]))
        wind.append(int(i[0][14:18].strip()))
        temp.append(int(i[0][18:22].strip()))
        stat.append(int(i[0][22]))
        date.append(i[1])

        cnt += 1

    else: cnt = 0

f.close()

f = open("./DB/P_23_1_DB.db", "w")
f.close()

con = sq.connect("./DB/P_23_1_DB.db", isolation_level= None)
cmd = con.cursor()

cmd.execute('''
CREATE TABLE "LOG_DATA" (
	"EQUIP_CODE"	TEXT NOT NULL,
	"SENSOR_CODE"	INTEGER,
	"WIND"	INTEGER,
	"TEMP"	INTEGER,
	"STATUS_CODE"	INTEGER,
	"DATE"  TEXT
);
''')
cmd.fetchall()

for i in range(len(equi)):

    cmd.execute(f"INSERT INTO LOG_DATA VALUES ('{equi[i]}', {sens[i]}, {wind[i]}, {temp[i]}, {stat[i]}, '{date[i]}')")
    cmd.fetchall()

print("저장 완료되었습니다.")


import argparse
import json
import numpy as np
import html
import spacy
import pickle

parser = argparse.ArgumentParser(description='Preprocess amazon reviews dataset using Spacy')
parser.add_argument('--input_file', default='', required=True, metavar='PATH')
parser.add_argument('--output_file', default='', required=True, metavar='PATH')
args = parser.parse_args()

def load_data(datafile):
    samples = [json.loads(line) for line in open(datafile).readlines()]
    data = {}
    data['review'] = [html.unescape(sample['reviewText']) for sample in samples]
    data['summary'] = [html.unescape(sample['summary']) for sample in samples]
    data['rating'] = np.array([sample['overall'] for sample in samples])
    return data

def get_clean_review(review, summ, rating):
    sample = {}
    # remove stop-words and whitespace tokens split paragraphs into sentences
    review_valid = [[tok for tok in sent if not tok.is_stop and tok.text.strip() != ''] for sent in review.sents]
    # remove empty sentences
    review_valid = [sent for sent in review_valid if not len(sent) == 0]
    sample['review'] = [[tok.text.lower() for tok in sent] for sent in review_valid]
    # remove stop-words and whitespace tokens
    summary_valid = [tok for tok in summ if not tok.is_stop and tok.text.strip() != '']
    sample['summary'] = [tok.text.lower() for tok in summary_valid]
    sample['rating'] = int(rating)
    return sample

def dump_dataset(raw_data, outfile, summary=True):
    with open(outfile, 'w') as outf:
        nlp = spacy.load('en_core_web_sm')
        review_docs = nlp.pipe(raw_data['review'])
        summ_docs = nlp.pipe(raw_data['summary'])
        n = len(raw_data['rating'])
        for review, summ, rating in zip(review_docs, summ_docs, raw_data['rating']):
            sample = get_clean_review(review, summ, rating)
            outf.write(json.dumps(sample) + '\n')

if __name__ == '__main__':
    print("Loading raw data from {}".format(args.input_file))
    data = load_data(args.input_file)
    print("Preprocessing data and writing to {}".format(args.output_file))
    dump_dataset(data, args.output_file)


"""
https://leetcode.com/problems/single-threaded-cpu/

You are given n tasks labeled from 0 to n - 1 represented by a 2D integer array
tasks, where tasks[i] = [enqueueTimei, processingTimei] means that the ith task
will be available to process at enqueueTimei and will take processingTimei to
finish processing.

You have a single-threaded CPU that can process at most one task at a time and
will act in the following way:

  If the CPU is idle and there are no available tasks to process, the CPU
  remains idle.

  If the CPU is idle and there are available tasks, the CPU will choose the one
  with the shortest processing time. If multiple tasks have the same shortest
  processing time, it will choose the task with the smallest index.

  Once a task is started, the CPU will process the entire task without stopping.

  The CPU can finish a task then start a new one instantly.

Return the order in which the CPU will process the tasks.
"""

from typing import List
from bisect import bisect

def get_order(tasks: List[List[int]]) -> List[int]:
  """
  O() O(n)
  """
  tasks = sorted((t[0], (t[1], i)) for i, t in enumerate(tasks))
  len_tasks = len(tasks)
  t = tasks[0][0]
  i = j = 0
  while True:
    while tasks[j][0] <= t:
      k = bisect(tasks, tasks[j][1], lo=i, hi=j, key=lambda item: item[1])
      if k != j: tasks.insert(k, tasks.pop(j))
      j += 1
      if j == len_tasks:
        return [task[1][1] for task in tasks]
    if tasks[i][0] > t:
      t = tasks[i][0]
    else:
      t += tasks[i][1][0]
      i += 1


import os
import cv2
import tqdm
import attparsenet_segments

import numpy as np
import pandas as pd

def get_attribute_points(image_name, annotations, landmarks, attribute='mouth'):

    ''' Gets the points that define a given attribute'''

    attribute_functions = {
        '5_o_Clock_Shadow': attparsenet_segments.five_oclock_shadow,
        'Arched_Eyebrows': attparsenet_segments.arched_eyebrows,
        'Attractive': attparsenet_segments.attractive,
        'Bags_Under_Eyes': attparsenet_segments.bags_under_eyes,
        'Bald': attparsenet_segments.bald,
        'Bangs': attparsenet_segments.bangs,
        'Big_Lips': attparsenet_segments.big_lips,
        'Big_Nose': attparsenet_segments.big_nose,
        'Black_Hair': attparsenet_segments.black_hair,
        'Blond_Hair': attparsenet_segments.blond_hair,
        'Blurry': attparsenet_segments.blurry,
        'Brown_Hair': attparsenet_segments.brown_hair,
        'Bushy_Eyebrows': attparsenet_segments.bushy_eyebrows,
        'Chubby': attparsenet_segments.chubby,
        'Double_Chin': attparsenet_segments.double_chin,
        'Eyeglasses': attparsenet_segments.eyeglasses,
        'Goatee': attparsenet_segments.goatee,
        'Gray_Hair': attparsenet_segments.grey_hair,
        'Heavy_Makeup': attparsenet_segments.heavy_makeup,
        'High_Cheekbones': attparsenet_segments.high_cheeckbones,
        'Male': attparsenet_segments.male,
        'Mouth_Slightly_Open': attparsenet_segments.mouth_slightly_open,
        'Mustache': attparsenet_segments.mustache,
        'Narrow_Eyes': attparsenet_segments.narrow_eyes,
        'No_Beard': attparsenet_segments.no_beard,
        'Oval_Face': attparsenet_segments.oval_face,
        'Pale_Skin': attparsenet_segments.pale_skin,
        'Pointy_Nose': attparsenet_segments.pointy_nose,
        'Receding_Hairline': attparsenet_segments.receding_hairline,
        'Rosy_Cheeks': attparsenet_segments.rosy_cheeks,
        'Sideburns': attparsenet_segments.side_burns,
        'Smiling': attparsenet_segments.smiling,
        'Straight_Hair': attparsenet_segments.straight_hair,
        'Wavy_Hair': attparsenet_segments.wavy_hair,
        'Wearing_Earrings': attparsenet_segments.wearing_earrings,
        'Wearing_Hat': attparsenet_segments.wearing_hat,
        'Wearing_Lipstick': attparsenet_segments.wearing_lipstick,
        'Wearing_Necklace': attparsenet_segments.wearing_necklace,
        'Wearing_Necktie': attparsenet_segments.wearing_necktie,
        'Young': attparsenet_segments.young
    }

    attribute_points = attribute_functions[attribute](image_name, annotations, landmarks, attribute)
    return attribute_points

def create_attribute_mask(mask_image, attribute_points):
    '''Creates an attribute mask for the given image for a given attribute  '''
    for region in attribute_points:
        cv2.fillPoly(img=mask_image, pts=region, color=(255))
    return mask_image

def create_attribute_masks(input_directory='OpenFace_not_detected_originals',
                           output_directory_1='OpenFace_not_detected_masks',
                           output_directory_2='OpenFace_not_detected_masks_resized',
                           landmarks_path='OpenFace_not_detected_landmarks.csv', annotations_path='list_attr_celeba.txt',
                           path_to_crop_points='OpenFace_not_detected_crop_points.csv'):
    ''' Creates attribute masks for the images in the given directory and saves them to
    the output directory. A subdirectory is created in the output directory for every attribute
    in the attribute list'''

    # Get the name of each file in the input directory
    image_names = sorted(os.listdir(input_directory))

    # Open the attribute label, landmark, and crop point csv files
    annotations_df = pd.read_csv(annotations_path, sep=',', header=0, index_col="image_name")
    landmarks_df = pd.read_csv(landmarks_path, index_col='image_name')
    crop_points_df = pd.read_csv(path_to_crop_points, index_col=0)

    # Ensure that the output directories exist. If they do not exist, create them
    if not os.access(output_directory_1, os.F_OK):
        os.mkdir(output_directory_1)
    if not os.access(output_directory_2, os.F_OK):
        os.mkdir(output_directory_2)

    # Iterate through each image in the input directory
    for image_name in tqdm.tqdm(image_names):

        # store the complete path to the image and open it
        image_path = input_directory + '/' + image_name
        image = cv2.imread(image_path)

        # crop the image according to the crop points csv
        x_min, x_max, y_min, y_max = crop_points_df.loc[image_name]
        cropped_image = image[y_min:y_max, x_min:x_max]

        # Ensure that the resulting image is of the correct size
        #      The image will have minimum dimensions of 179x219
        if (cropped_image.shape[0] <= 178):
            scale_factor = (179 / cropped_image.shape[0])
            resized_dim = (round(cropped_image.shape[1] * scale_factor), round(cropped_image.shape[0] * scale_factor))
            cropped_image = cv2.resize(cropped_image, resized_dim, interpolation=cv2.INTER_AREA)
        if (cropped_image.shape[1] <= 218):
            scale_factor = (219 / cropped_image.shape[1])
            resized_dim = (round(cropped_image.shape[1] * scale_factor), round(cropped_image.shape[0] * scale_factor))
            cropped_image = cv2.resize(cropped_image, resized_dim, interpolation=cv2.INTER_AREA)
        # Store the complete output path for the resized image
        image_resized_path = os.path.join(output_directory_2, image_name)

        # For each attribute
        for attribute in annotations_df:
            # Generate a black image with same dimensions as the resized input image
            mask_image = np.zeros((image.shape[0], image.shape[1], 1), np.uint8)

            # Get the associated landmark points for the current attribute
            attribute_points = get_attribute_points(image_name, annotations_df, landmarks_df, attribute)

            # Fill in the regions of the mask image, where the current attribute occurs, with white pixels
            create_attribute_mask(mask_image, attribute_points)

            # Crop the mask image
            cropped_mask = mask_image[y_min:y_max, x_min:x_max]

            if (cropped_mask.shape[0] <= 178):
                scale_factor = (179 / cropped_mask.shape[0])
                resized_dim = (round(cropped_mask.shape[1] * scale_factor), round(cropped_mask.shape[0] * scale_factor))
                cropped_mask = cv2.resize(cropped_mask, resized_dim, interpolation=cv2.INTER_NEAREST)
            if (cropped_mask.shape[1] <= 218):
                scale_factor = (219 / cropped_mask.shape[1])
                resized_dim = (round(cropped_mask.shape[1] * scale_factor), round(cropped_mask.shape[0] * scale_factor))
                cropped_mask = cv2.resize(cropped_mask, resized_dim, interpolation=cv2.INTER_NEAREST)

            # Store the full path to the mask output directory
            mask_name = attribute + '_mask_' + image_name
            mask_path = os.path.join(output_directory_1, mask_name)

            # Save the mask for the current attribute
            if cv2.imwrite(mask_path, cropped_mask) == False:
                print(f'Mask {mask_name} not saved successfully!')

        # Save the resized image
        if cv2.imwrite(image_resized_path, cropped_image) == False:
            print(f'Image {image_resized_path} not saved successfully!')

        # Uncomment the four lines below and comment the lines above (the ones that save the images) to see the masks
        #   and resized images

        # cv2.imshow(image_name + " Resized", cropped_image)
        # cv2.imshow(image_name + " " + attribute, cropped_mask)
        # cv2.waitKey(0)
        # cv2.destroyAllWindows()

path_to_original_images = "/home/nthom/Documents/datasets/CelebA/Img/img_celeba"
path_to_output_segment_labels = "/home/nthom/Documents/datasets/CelebA/Img/segment_labels_178x218/"
path_to_output_resized_images = "/home/nthom/Documents/datasets/CelebA/Img/resized_images_178x218/"

path_to_landmark_csv = "/home/nthom/Documents/datasets/UNR_Facial_Attribute_Parsing_Dataset/landmarks.csv"
path_to_crop_points_csv = "/home/nthom/Documents/datasets/UNR_Facial_Attribute_Parsing_Dataset/crop_points.csv"
path_to_image_annotations = "/home/nthom/Documents/datasets/UNR_Facial_Attribute_Parsing_Dataset/list_attr_celeba_attparsenet.csv"

create_attribute_masks(path_to_original_images, path_to_output_segment_labels, path_to_output_resized_images,
                       path_to_landmark_csv,path_to_image_annotations, path_to_crop_points_csv)


from google.appengine.ext.webapp.template import render, register_template_library
from google.appengine.ext import ndb
from google.appengine.api import users, memcache
from googleapiclient.errors import HttpError
from urlparse import urlparse
import logging
import webapp2
import config

from third_party import xsrfutil

register_template_library('third_party.xsrfutil')

class Link(ndb.Model):
  url = ndb.StringProperty()
  owner_id = ndb.StringProperty()
  owner_name = ndb.StringProperty()
  viewcount = ndb.IntegerProperty()

def errorPage(response, code, message):
  context = {'code': code, 'message': message, 'corpname': config.CORP_NAME}
  response.write(render('template/error.html', context))
  response.set_status(code)

def check_redirect(func):

  def decorate(self, *args, **kwargs):
    if config.ALWAYS_REDIRECT_TO_FQDN and self.request.host != config.GOLINKS_FQDN:
      url = self.request.url.replace(self.request.host, config.GOLINKS_FQDN, 1)
      return self.redirect(url)
    return func(self, *args, **kwargs)

  return decorate

def isValidUrl(url):
  o = urlparse(url)
  return o.scheme in config.URL_ALLOWED_SCHEMAS

class ShowLinks(webapp2.RequestHandler):

  @check_redirect
  def get(self, param):
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url(self.request.path))
      return
    if param == "all":
      links = Link.query().fetch()
    else:
      links = Link.query(Link.owner_id == user.user_id()).fetch()
    context = {
        "links": links,
        "fqdn": config.GOLINKS_FQDN,
        "hostname": config.GOLINKS_HOSTNAME
    }
    self.response.write(render("template/list.html", context))

class DeleteLink(webapp2.RequestHandler):

  @check_redirect
  @xsrfutil.xsrf_protect
  def post(self, link):
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url(self.request.path))
      return
    key = link.rstrip("/")
    l = Link.get_by_id(key)
    if l.owner_id:
      if l.owner_id != user.user_id() and not users.is_current_user_admin():
        logging.info("%s tried to delete /%s but doesn't have permission" %
                     (user.email(), key))
        errorPage(self.response, 403, "Access denied")
        return
    l.key.delete()
    logging.info("%s deleted /%s" % (user.email(), key))
    self.redirect("/links/my")

class EditLink(webapp2.RequestHandler):

  @check_redirect
  @xsrfutil.xsrf_protect
  def post(self, link):
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url(self.request.path))
      return
    key = self.request.get("key", "").rstrip("/")
    url = self.request.get("url", None)
    if not key:
      errorPage(self.response, 400, "Shortened URL required")
      return
    blacklist = ["edit", "links", "delete"]
    for word in blacklist:
      if key.startswith(word + '/') or key == word:
        logging.info("%s tried to add forbidden URL /%s" % (user.email(), key))
        errorPage(self.response, 400, "Shortened URL forbidden")
        return
    if link:
      if key != link:
        logging.info(
            "%s tried to change /%s to %s but such request is forbidden" %
            (user.email(), link, key))
        errorPage(self.response, 400, "Cannot change shortened URL")
        return
    if not isValidUrl(url):
      logging.info("%s tried to set /%s to illegal URL: %s" %
                   (user.email(), key, url))
      errorPage(self.response, 400, "URL Illegal")
      return
    l = Link.get_or_insert(key)
    if l.owner_id:
      if not link:
        logging.info("%s tried to overwrite /%s" % (user.email(), key))
        errorPage(
            self.response, 500,
            "Link already exists... Please update existing one or change url.")
        return
      if l.owner_id != user.user_id() and not users.is_current_user_admin():
        logging.info("%s tried to modify /%s but doesn't have permission" %
                     (user.email(), key))
        errorPage(self.response, 403, "Access denied")
        return
    else:
      l.owner_id = user.user_id()
      l.owner_name = user.nickname()
    l.url = url
    if not l.viewcount:
      l.viewcount = 0
    l.put()
    logging.info("%s created or updated /%s to %s" % (user.email(), key, url))
    self.redirect("/edit/" + key)

  @check_redirect
  def get(self, link):
    user = users.get_current_user()
    if not user:
      self.redirect(users.create_login_url(self.request.path))
      return
    is_admin = users.is_current_user_admin()
    context = {
        'hostname': config.GOLINKS_HOSTNAME
    }
    if link:
      link = link.rstrip("/")
      context.update({'key': link})
      l = Link.get_by_id(link)
      if l:
        if l.owner_id:
          if l.owner_id != user.user_id() and not is_admin:
            logging.info(
                "%s tried to check details page of /%s but doesn't have permission"
                % (user.email(), link))
            errorPage(self.response, 403, "Access denied")
            return
        context.update({
            'url': l.url,
            'viewcount': l.viewcount,
            'can_delete': 1,
            'owner': l.owner_name
        })
    logging.info("%s checked details page of /%s" % (user.email(), link))
    self.response.write(render("template/edit.html", context))

class RedirectLink(webapp2.RequestHandler):

  @check_redirect
  def get(self, link):
    user = users.get_current_user()
    if link:
      link = link.rstrip("/")
      l = Link.get_by_id(link)
      if l:
        if not user:
          self.redirect(users.create_login_url(self.request.path))
          return
        username = user.email()
        l.viewcount += 1
        l.put()
        logging.info("%s accessed /%s and redirected to %s" %
                     (username, link, str(l.url)))
        self.redirect(str(l.url))
        return
    if not user:  # we don't want external to know if url exists
      self.redirect(users.create_login_url(self.request.path))
      return
    if not link:
      self.redirect('/links/my')
      return

    # Link doesn't exist, let's make it!
    self.redirect('/edit/%s' % link.rstrip('/'))

app = webapp2.WSGIApplication([
    ('/edit/([-\/\w]*)', EditLink),
    ('/delete/([-\/\w]+)', DeleteLink),
    ('/links/(\w*)', ShowLinks),
    ('/([-\/\w]*)', RedirectLink),
],
                              debug=True)


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from Othello import *

"""
黑白棋，DQN,
各种超参数
"""
BOARD_SIZE = 8
N_STATE = pow(BOARD_SIZE, 2)
N_ACTION = pow(BOARD_SIZE, 2) + 1

LR = 0.001
EPISODE = 10000
BATCH_SIZE = 32
GAMMA = 0.9
ALPHA = 0.8
TRANSITIONS_CAPACITY = 200
UPDATE_DELAY = 10

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class NET(nn.Module):
    """定义网络结构

    Returns:
        x [tensor] -- (batch, N_ACTION)，每一行表示各个action的分数
    """

    def __init__(self):
        super(NET, self).__init__()

        self.linear1 = nn.Sequential(
            nn.Linear(N_STATE, 128),
            nn.LeakyReLU()
        )
        # self.linear1.weight.data.normal_(0, 0.1)

        self.conv1 = nn.Sequential(
            nn.Conv1d(1, 4, 3, 1, 1),
            nn.LeakyReLU(inplace=True)
        )

        self.conv2 = nn.Sequential(
            nn.Conv1d(4, 8, 3, 1, 1),
            nn.LeakyReLU()
        )

        self.linear2 = nn.Sequential(
            nn.Linear(8 * 128, N_ACTION)
        )

    def forward(self, x):
        x = self.linear1(x)
        x = x.view(x.shape[0], 1, -1)
        x = self.conv1(x)
        x = self.conv2(x)
        # x = x.flatten()
        x = x.view(x.shape[0], -1)
        x = self.linear2(x)
        return x

class DQN(object):
    def __init__(self, color):
        """
        color: 1表示先手；-1表示后手

        transitions : 存储状态的空间，格式为(state, action, reward, state_), state_为后继状态
        transitions_index : 记录当前使用存储空间的索引
        learn_iter : 当到达UPDATE_ITERS时，就更新预测网络 Q_ ，把Q的参数复制给它
        """
        self.transitions = np.zeros((TRANSITIONS_CAPACITY, 2 * N_STATE + 2))
        self.transitions_index = 0
        self.learn_iter = 0

        self.Q, self.Q_ = NET(), NET()
        # if color == 1:
        #     self.Q.load_state_dict(torch.load('model_offensive.pth'))
        # elif color == -1:
        #     self.Q.load_state_dict(torch.load('model_defensive.pth'))

        self.optimizer = torch.optim.Adam(self.Q.parameters(), lr=LR)
        self.criteria = nn.MSELoss()

    def Choose_Action_EpsilonGreedy(self, x, game_state, color, Epsilon=0.1):
        """ε-greedy算法选择下一个action。以ε概率随机选择一个action，否则就选择Q值最大的action

        Arguments:
            x [tensor] -- NET网络的输入值，即当前状态，在Q-Learning中，选择下一个动作应该是查表得到的，
                            在DQN中没有这个表，所以要先经过Q网络得到一个状态的Q值，然后选择这向量里概率最大的action
            game_state [class] -- 当前的游戏状态
            color int -- 1表示黑棋，-1表示白棋

        Returns:
            action [int] -- 0~64中的一个数，表示下棋的位置；64表示跳过
        """

        if color == 1:
            avaliable_pos = game_state.Get_Valid_Pos(game_state.black_chess, game_state.white_chess)
        elif color == -1:
            avaliable_pos = game_state.Get_Valid_Pos(game_state.white_chess, game_state.black_chess)

        avaliable_pos = list(map(lambda a: game_state.board_size * a[0] + a[1], avaliable_pos))  # 列表,表明合法位置
        if len(avaliable_pos) == 0:
            return 64  # 表示这一步只能跳过

        if np.random.uniform() < Epsilon:  # random choose an action
            action = np.random.choice(avaliable_pos, 1)[0]
        else:  # choose the max Q-value action
            x = torch.tensor(x, dtype=torch.float)
            x = x.view(1, -1)
            actions_values = self.Q(x)[0]  # 65维tensor，各个action在各个位置的值

            ava_actions = torch.tensor(actions_values[avaliable_pos])

            _, action_ind = torch.max(ava_actions, 0)
            action = avaliable_pos[action_ind]
        return action

    def Store_transition(self, s, a, r, s_):
        """把一组转移属性存储到transitions中

        Arguments:
            s {[type]} -- 当前状态
            a {[type]} -- 选择的动作
            r {[type]} -- reward值
            s_ {[type]} -- 后继状态
        """
        transition = np.hstack((s, a, r, s_))
        self.transitions[self.transitions_index % TRANSITIONS_CAPACITY] = transition
        self.transitions_index += 1

    def Learn(self, oppo_Q_):
        for step in range(10):
            if self.learn_iter % UPDATE_DELAY == 0:  # update parameters of Q_ 每隔一段时间将Q的参数直接给到Q_
                self.Q_.load_state_dict(self.Q.state_dict())
            self.learn_iter += 1

            sample_index = np.random.choice(TRANSITIONS_CAPACITY,
                                            BATCH_SIZE)  # randomly choose BATCH_SIZE samples to learn 从经验池中随机选取进行训练
            batch_tran = self.transitions[sample_index, :]
            batch_s = batch_tran[:, :N_STATE]
            batch_a = batch_tran[:, N_STATE: N_STATE + 1]
            batch_r = batch_tran[:, N_STATE + 1: N_STATE + 2]
            batch_s_ = batch_tran[:, N_STATE + 2:]

            batch_s = torch.tensor(batch_s, dtype=torch.float)
            batch_s_ = torch.tensor(batch_s_, dtype=torch.float)
            batch_a = torch.tensor(batch_a, dtype=int)
            batch_r = torch.tensor(batch_r, dtype=torch.float)

            batch_y = self.Q(batch_s).gather(1,
                                             batch_a)  # gather figure out which action actually is chosen 相当于从第一维取第batch_a位置的值
            batch_y_ = oppo_Q_(
                batch_s_).detach()  # detach return a new Variable which do not have gradient detach就是禁止梯度更新，这些图变量包含了梯度，在计算loss的时候会更新，因为Q_不用更新，因此禁止梯度。
            batch_y_ = batch_r - GAMMA * torch.max(batch_y_, 1)[0].view(-1,
                                                                        1)  # max(1) return (value,index) for each row

            loss = self.criteria(batch_y, batch_y_)
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

if __name__ == "__main__":
    offensive = DQN(1)
    defensive = DQN(-1)

    for episode in range(EPISODE * 50):
        game_state = Game()
        round_ = 0
        while True:
            # 先手
            # print(round_)
            round_ += 1
            # game_state.Display()    # 输出棋盘
            s = game_state.Get_State()
            a = offensive.Choose_Action_EpsilonGreedy(s, game_state, 1)
            game_state.Add(1, a)
            r = game_state.Gameover() * 100.0
            s_ = game_state.Get_State()

            offensive.Store_transition(s, a, r, s_)   # 先后手的经验池分开存
            # defensive.Store_transition(s, a, -r, s_)

            if r != 0 or round_ > 100:  # 当这局游戏结束或双方下够了100次。经验池已经有很多样本，此时可以开始训练
                offensive.Learn(defensive.Q_) # 用对手的Q_网络来计算下一个状态
                # print("END~~~~~")
                # game_state.Display()
                # print("==================================")
                print('Episode:{} | Reward:{}'.format(episode, r))
                break

            # 后手
            # game_state.Display()
            s = game_state.Get_State()
            a = defensive.Choose_Action_EpsilonGreedy(s, game_state, -1)
            game_state.Add(-1, a)
            r = game_state.Gameover() * 100.0
            s_ = game_state.Get_State()

            # offensive.Store_transition(s, a, r, s_)
            defensive.Store_transition(s, a, -r, s_) # 先后手的经验池分开存

            if r != 0:
                defensive.Learn(offensive.Q_) # 用对手的Q_网络来计算下一个状态
                # print("END~~~~~")
                # game_state.Display()
                # print("==================================")
                print('Episode:{} | Reward:{}'.format(episode, r))
                break

        if (episode + 1) % 100 == 0:
            torch.save(offensive.Q.state_dict(), 'model_offensive.pth')
            torch.save(defensive.Q.state_dict(), 'model_defensive.pth')


from collections import deque

import numpy as np

from ..agents.sample_efficient import Dyna_Q_Agent

class Trial:

    def __init__(self, env, agent, episodes, scores : deque, lrews : list):

        # Just to store the long-term-reward of the last 100 experiments
        self.scores = scores
        self.lrews = lrews

        self.env = env
        self.agent = agent
        self.episodes = episodes

    def interact(self):

        for episode in range(1, self.episodes+1):
            state = self.env.reset()
            R = 0
            done = False
            R, reward = 0,0

            for step in range(self.env.spec.max_episode_steps):
                action = self.agent.select_action(state)

                next_state, reward, done, info = self.env.step(action)

                self.agent.update(state, action, reward, next_state)

                if isinstance(self.agent, Dyna_Q_Agent):
                    self.agent.update_model(state, action, reward, next_state)

                state = next_state
                R += reward

                if done:
                    break

            self.scores.append(R)
            self.lrews.append(np.mean(self.scores))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('dashboard', '0043_auto_20150802_1151'),
    ]

    operations = [
        migrations.CreateModel(
            name='AppointmentNote',
            fields=[
                ('note_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='dashboard.Note')),
                ('date', models.DateField(verbose_name=b'Appointment date')),
                ('time', models.TimeField(verbose_name=b'Appointment time')),
                ('reason_for_visit', models.CharField(max_length=200)),
                ('clinic', models.ManyToManyField(related_name='appointments', to='dashboard.Clinic')),
                ('doctor', models.ManyToManyField(related_name='appointments', to=settings.AUTH_USER_MODEL)),
            ],
            bases=('dashboard.note',),
        ),
    ]


# -*- coding: utf-8 -*-
import requests
import bs4
from bs4 import BeautifulSoup as bs
import json
import lxml
import re

def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()    # 产生异常信息
        r.encoding = r.apparent_encoding
        print(r.json())
        return r.text
    except:
        return "网页获取错误"
    return ""

def fillUnivList(ulist, html):
    soup = bs(html, 'html.parser')
    for tr in soup.find('tbody').children:
        if isinstance(tr, bs4.element.Tag):
            tds = tr('td')  # tds = tr.find_all('td')
            ulist.append([tds[0].string, tds[1].string, tds[2].string, tds[3].string])

def printUnivList(ulist, num):
    tplt = "{0:^10}\t{1:{4}^10}\t{2:{4}^10}\t{3:^10}"
    print(tplt.format("排名", "学校名称", "省市", "分数", chr(12288)))
    for i in range(num):
        u = ulist[i]
        print(tplt.format(u[0], u[1], u[2], u[3], chr(12288)))

def main():
    uinfo = []
    url = "https://vjudge.net/comment/thread?path=error&t=1524644989968.5"
    html = getHTMLText(url)
    print(html)
    # fillUnivList(uinfo, html)
    # printUnivList(uinfo, 100)

main()


from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Set

CWD = Path(__file__).parent

class Food:
    ingredients: Set[str]
    allergens: Set[str]

    def __init__(self, raw: str):
        ings, allergs = raw.split(" (", maxsplit=1)
        self.ingredients = set(ings.split(" "))
        self.allergens = set(allergs.strip()[9:-1].split(", "))

def read_input(filename: str="input.txt") -> List[Food]:
    input_file = CWD.joinpath(filename)
    with open(input_file, "r") as reader:
        return [Food(l) for l in reader.readlines()]

def get_allergen_sources(foods: List[Food]) -> Dict[str, str]:
    allergens_list = set()
    for f in foods:
        allergens_list = allergens_list.union(f.allergens)
    found_allergens = defaultdict(str)
    while len(found_allergens) != len(allergens_list):
        for i, f in enumerate(foods):
            unfound_allergens = f.allergens - set(found_allergens.keys())
            unfound_ingredients = f.ingredients - set(found_allergens.values())
            if len(unfound_allergens) == len(unfound_ingredients) == 1:
                found_allergens[unfound_allergens.pop()] = unfound_ingredients.pop()
                continue
            for a in unfound_allergens:
                common_ings = unfound_ingredients
                for j, g in enumerate(foods):
                    if i == j or a not in g.allergens:
                        continue
                    common_ings = common_ings.intersection(g.ingredients)
                    if len(common_ings) == 1:
                        found_allergens[a] = common_ings.pop()
                        break
    return found_allergens

def first(foods: List[Food], allergen_sources: Dict[str, str]) -> int:
    ingredients_list = set()
    for f in foods:
        ingredients_list = ingredients_list.union(f.ingredients)
    safe_ingredients = ingredients_list - set(allergen_sources.values())
    return len([i for f in foods for i in f.ingredients if i in safe_ingredients])

def second(foods: List[Food],allergen_sources: Dict[str, str]) -> str:
    allergens = list(allergen_sources.keys())
    allergens.sort()
    return ','.join([allergen_sources[a] for a in allergens])

def test_example() -> None:
    foods = read_input("example.txt")
    allergen_sources = get_allergen_sources(foods)
    assert first(foods, allergen_sources) == 5
    assert second(foods, allergen_sources) == "mxmxvkd,sqjhc,fvjkl"

def run() -> None:
    test_example()
    foods = read_input()
    allergen_sources = get_allergen_sources(foods)
    print("First step:")
    print(first(foods, allergen_sources)) # 2380
    print("\nSecond step:")
    print(second(foods, allergen_sources)) # ktpbgdn,pnpfjb,ndfb,rdhljms,xzfj,bfgcms,fkcmf,hdqkqhh


import torch
import torch.nn as nn
from collections import OrderedDict

class RobertaWithClfHead(nn.Module):
    def __init__(self, roberta, config):
        super(RobertaWithClfHead, self).__init__()

        self.roberta = roberta
        self.classifier = nn.Sequential(OrderedDict([
            ('dropout_1', nn.Dropout(config['dropout_prob'])),
            ('dense', nn.Linear(in_features=config['linear_1_in_features']*config["no_of_pooling_layers"], out_features=config['linear_1_out_features'])),
            ('batchnorm', nn.BatchNorm1d(num_features=config['linear_1_out_features'])) if config["normalization"]=="batchnorm" else ('layernorm', nn.LayerNorm(config['linear_1_out_features'])),
            ('relu', nn.ReLU()) if config['activation']=='relu' else ('gelu', nn.GELU()),
            ('dropout_2', nn.Dropout(config['dropout_prob'])),
            ('out_proj', nn.Linear(in_features=config['linear_1_out_features'], out_features=config['num_labels']))
        ]))
        self.classifier.apply(self.init_weights)

    def forward(self, no_of_pooling_layers=1, **kwargs):
        inputs = kwargs

        if no_of_pooling_layers == 1:
            outputs = self.roberta(**inputs, output_hidden_states=True)
            hidden_states = outputs.last_hidden_state
            pooler = hidden_states[:, 0, :]
        else:
            outputs = self.roberta(**inputs, output_hidden_states=True)
            hidden_states = outputs.hidden_states
            pooling_layers = [l for l in range(-no_of_pooling_layers, 0)]
            pooler = torch.cat(tuple([hidden_states[i] for i in pooling_layers]), dim=-1)[:, 0, :]

        logits = self.classifier(pooler)
        return logits

    def init_weights(self,m):
        if type(m) == nn.Linear:
            torch.nn.init.xavier_uniform_(m.weight)
            torch.nn.init.zeros_(m.bias)