import os
from datetime import datetime
from dateutil.parser import parse as date_parser

from django.conf import settings
from django.urls import reverse

from sources.extraction.base import SingleResponseExtractProcessor
from sources.extraction.hanze.research_themes import FOCUS_AREA_TO_RESEARCH_THEME
from sources.extraction.pure import PureAPIMixin

class HanzePersonsExtractProcessor(SingleResponseExtractProcessor, PureAPIMixin):
    @classmethod
    def get_name(cls, node):
        return f"{node['name']['firstName']} {node['name']['lastName']}"

    @classmethod
    def get_isni(cls, node):
        isni_identifier = next(
            (identifier for identifier in node.get("identifiers", [])
             if "isni" in identifier.get("type", {}).get("uri", "")),
            None
        )
        return isni_identifier["id"] if isni_identifier else None

    @classmethod
    def get_is_employed(cls, node):
        today = datetime.today()
        for association in node.get("staffOrganizationAssociations", []):
            end_date = association["period"].get("endDate", None)
            if not end_date or date_parser(end_date) > today:
                break
        else:
            return False
        return True

    @classmethod
    def get_photo_url(cls, node):
        photo_list = node.get("profilePhotos", None)
        if not photo_list:
            return
        photo_url = photo_list[0].get("url", None)
        if not photo_url:
            return
        file_path_segment = "/nppo/persons/"
        if file_path_segment not in photo_url:
            return photo_url  # not dealing with a url we recognize as a file url
        start = photo_url.index(file_path_segment)
        file_path = photo_url[start + len(file_path_segment):]
        proxy_photo_url = reverse("v1:files", kwargs={
            "source": "hanze",
            "file_path": file_path
        })
        return f"https://{settings.DOMAIN}{proxy_photo_url}"

    @classmethod
    def get_job_title(cls, node):
        today = datetime.today()
        for association in node.get("staffOrganizationAssociations", []):
            end_date = association.get("period", None).get("endDate", None)
            if not end_date or date_parser(end_date, ignoretz=True) > today:
                break
        else:
            return
        job_title_object = association.get("jobTitle", None)
        if not job_title_object:
            return
        return next(iter(job_title_object["term"].values()), None)

HanzePersonsExtractProcessor.OBJECTIVE = {
    "external_id": "$.uuid",
    "name": HanzePersonsExtractProcessor.get_name,
    "first_name": "$.name.firstName",
    "last_name": "$.name.lastName",
    "prefix": lambda node: None,
    "initials": lambda node: None,
    "title": lambda node: None,
    "email": "$.user.email",
    "phone": lambda node: None,
    "skills": lambda node: [],
    "themes": lambda node: [],
    "description": lambda node: None,
    "parties": lambda node: [],
    "photo_url": HanzePersonsExtractProcessor.get_photo_url,
    "isni": HanzePersonsExtractProcessor.get_isni,
    "dai": lambda node: None,
    "orcid": "$.orcid",
    "is_employed": HanzePersonsExtractProcessor.get_is_employed,
    "job_title": HanzePersonsExtractProcessor.get_job_title,
    "research_themes": lambda node: [],
}

class HanzeProjectExtractProcessor(SingleResponseExtractProcessor, PureAPIMixin):

    @classmethod
    def get_status(cls, node):
        today = datetime.today()
        end_date = node.get("period", {}).get("endDate")
        return "ongoing" if not end_date or today <= date_parser(end_date) else "finished"

    @classmethod
    def get_title(cls, node):
        title = node["title"]
        return list(title.values())[0]

    @classmethod
    def get_description(cls, node):
        # Gather all possible descriptions
        language_code = None
        descriptions = {}
        for description in node["descriptions"]:
            if "value" not in description:
                continue
            if language_code is None:
                language_code = list(description["value"].keys())[0]
            description_type = os.path.split(description["type"]["uri"])[1]
            description_text = description["value"].get(language_code, None)
            if description_text:
                descriptions[description_type] = description_text
        # Concatenate different descriptions to be a singular text
        description = ""
        if "laymansdescription" in descriptions:
            description += descriptions["laymansdescription"]

        if "keyfindings" in descriptions:
            if description:
                description += "

"
            description += descriptions["keyfindings"]
        if "projectdescription" in descriptions:
            if description:
                description += "

"
            description += descriptions["projectdescription"]
        return description or None

    @classmethod
    def get_keywords(cls, node):
        keyword_groups = [
            keyword_group for keyword_group in node.get("keywordGroups", [])
            if keyword_group.get("logicalName", None) == "keywordContainers"
        ]
        if not keyword_groups:
            return []
        keywords = []
        for keyword_group in keyword_groups:
            keywords += keyword_group["keywords"][0]["freeKeywords"]
        return keywords

    @classmethod
    def get_products(cls, node):
        return [product["researchOutput"]["uuid"] for product in node.get("researchOutputs", [])]

    @classmethod
    def get_persons(cls, node):
        persons = []
        for participant in node.get("participants", []):
            if "person" in participant:
                person = participant["person"]
            elif "externalPerson" in participant:
                person = participant["externalPerson"]
            else:
                continue
            persons.append({
                "external_id": person["uuid"],
                "email": None,
                "name": f"{participant['name']['firstName']} {participant['name']['lastName']}"
            })
        return persons

    @classmethod
    def get_owners(cls, node):
        persons = cls.get_persons(node)
        return [persons[0]] if persons else []

    @classmethod
    def get_research_themes(cls, node):
        research_themes = []
        for keywords in node.get("keywordGroups", []):
            if keywords["logicalName"] == "research_focus_areas":
                for classification in keywords["classifications"]:
                    if classification["uri"] in FOCUS_AREA_TO_RESEARCH_THEME.keys():
                        research_themes.append(FOCUS_AREA_TO_RESEARCH_THEME[classification["uri"]])
        return research_themes

HanzeProjectExtractProcessor.OBJECTIVE = {
    "external_id": "$.uuid",
    "title": HanzeProjectExtractProcessor.get_title,
    "status": HanzeProjectExtractProcessor.get_status,
    "started_at": "$.period.startDate",
    "ended_at": "$.period.endDate",
    "coordinates": lambda node: [],
    "goal": lambda node: None,
    "description": HanzeProjectExtractProcessor.get_description,
    "contacts": HanzeProjectExtractProcessor.get_owners,
    "owners": HanzeProjectExtractProcessor.get_owners,
    "persons": HanzeProjectExtractProcessor.get_persons,
    "keywords": HanzeProjectExtractProcessor.get_keywords,
    "parties": lambda node: [],
    "products": HanzeProjectExtractProcessor.get_products,
    "research_themes": HanzeProjectExtractProcessor.get_research_themes,
}


import cv2
import matplotlib.pyplot as plt

# resmi içe aktar
img = cv2.imread("sudoku.jpg", 0)
plt.figure(), plt.imshow(img, cmap = "gray"), plt.axis("off"), plt.title("Orijinal Img")

# x gradyan
sobelx = cv2.Sobel(img, ddepth = cv2.CV_16S, dx = 1, dy = 0, ksize = 5)
plt.figure(), plt.imshow(sobelx, cmap = "gray"), plt.axis("off"), plt.title("Sobel X")

# y gradyan
sobely = cv2.Sobel(img, ddepth = cv2.CV_16S, dx = 0, dy =1, ksize = 5)
plt.figure(), plt.imshow(sobely, cmap = "gray"), plt.axis("off"), plt.title("Sobel Y")

# laplacian gradyan
laplacian = cv2.Laplacian(img, ddepth = cv2.CV_16S)
plt.figure(), plt.imshow(laplacian, cmap = "gray" ),plt.axis("off"), plt.title("Laplacian")


import aiotools
import asyncio
import faust
from neo4j import GraphDatabase
from models import Question
from utils import read_env
#from aiokafkadaemon import worker, daemon

app = faust.App('ship-app', broker=read_env('KAFKA_LISTENER'))
questions_topic = app.topic('ship-topic', value_type=Question)

@app.agent(questions_topic)
async def answer(questions):
    print("TOTAL QUESTION "+len(questions))
    async for question in questions:
        print(f"HI THERE SEE ===========> {question.slack_id} {question.text}")

def create_new(tx, name):
    tx.run("CREATE (p:Person {name:$name}) RETURN p", name=name)

def add_liked(tx, name, liked_name):
    tx.run("MERGE (a:Person {name: $name}) "
           "MERGE (a)-[:LIKES]->(liked:Person {name: $liked_name})",

           name=name, liked_name=liked_name)

def add_person(name):
    try:
        uri = "bolt://neo4j:7687"
        driver = GraphDatabase.driver(uri, auth=("neo4j", "graph"))

        with driver.session() as session:
            session.write_transaction(create_new, name)
    except Exception as e:
        print(f"Failed connecting to bolt - {e}")

def add_liked_person(name, liked):
    try:
        uri = "bolt://neo4j:7687"
        driver = GraphDatabase.driver(uri, auth=("neo4j", "graph"))

        with driver.session() as session:
            session.write_transaction(create_new, name)
            session.write_transaction(add_liked, name, liked)
    except Exception as e:
        print(f"Failed connecting to bolt - {e}")

@app.agent(questions_topic)
async def read_stream(loop):
    print("======> LET ME TRY AND CHECK IT ")

async def echo(reader, writer):
   data = await reader.read(100)
   writer.write(data)
   await writer.drain()
   writer.close()

@aiotools.server
async def worker(loop, pidx, args):
   app.main()
   server = await asyncio.start_server(answer, '0.0.0.0', 8888, reuse_port=True)
   loop.run_forever(read_stream(loop))
   print(f'[{pidx}] started')
   yield  # wait until terminated
   server.close()
   await server.wait_closed()
   print(f'[{pidx}] terminated')

def main():
    try:
        uri = "bolt://neo4j:7687"
        driver = GraphDatabase.driver(uri, auth=("neo4j", "graph"))

        with driver.session() as session:
            session.write_transaction(create_new, "Alice")

        with driver.session() as session:
            session.write_transaction(create_new, "Paul")

        with driver.session() as session:
            session.write_transaction(create_new, "Rob")

        with driver.session() as session:
            session.write_transaction(create_new, "Dmitry")

        with driver.session() as session:
            session.write_transaction(add_liked, "Alice", "Neo4j")
        with driver.session() as session:
            session.write_transaction(add_liked, "Rob", "Neo4j")
        driver.close()
    except Exception as e:
        print(f"Failed connecting to bolt - {e}")

def execute():
    print("Started...")

async def echo(reader, writer):
   data = await reader.read(100)
   writer.write(data)
   await writer.drain()
   writer.close()

if __name__ == '__main__':
   # Run the above server using 4 worker processes.
   aiotools.start_server(worker, num_workers=4)


import time
import random
####just copy and paste the below given code to your shell

# KERAS

import numpy as np
import os
import cv2
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from Definitions import *
import gc
from imutils import paths
from keras.utils import to_categorical

def get_data(path_to_images, remove_ratio):
    listing = os.listdir(path_to_images)

    imlist = os.listdir(path_to_images)
    farrowinglist = []
    nofarrowinglist = []
    for f in imlist:
        if not f.endswith(".jpg") and not f.endswith(".txt") and not f.endswith(".dat") and not f.endswith(".py"):
            f2 = os.listdir(path_to_images+f)
            for f3 in f2:
                if not f3.endswith(".jpg") and not f3.endswith(".txt") and not f3.endswith(".dat") and not f3.endswith(".py"):
               #     print(str(os.listdir(path_to_images+f+"\\"+f3)))
                   # print(path_to_images+f+"\\"+f3)
                    if f3 == "farrow_full":
                        farrowings = os.listdir(path_to_images+f+"\\"+f3)
                        tofarrowinglist = [path_to_images+f+"\\"+f3+"\\"+farrow for farrow in farrowings]
                        farrowinglist.extend(tofarrowinglist)
                    elif f3 == "nofarrow_full":
                        nofarrowings = os.listdir(path_to_images+f+"\\"+f3)
                        tonofarrowinglist = [path_to_images+f+"\\"+f3+"\\"+nofarrow for nofarrow in nofarrowings]
                        nofarrowinglist.extend(tonofarrowinglist)

    imlist = []
    #The line of code below reduces the number of no farrowings to match the number of farrowings
    nofarrowinglist = random.choices(nofarrowinglist, k=int(len(nofarrowinglist) / remove_ratio))

    imlist.extend(nofarrowinglist)
    imlist.extend(farrowinglist)
    num_samples = len(imlist)
    nb_class0 = len(nofarrowinglist)
    nb_class1 = len(farrowinglist)
    del farrowinglist, nofarrowinglist
    print("Number of farrowings: " + str(nb_class1) + "\n Number of no farrowings: " + str(nb_class0))
  #  print(imlist)
    im1 = cv2.imread(imlist[0], cv2.IMREAD_COLOR)
    img_rows = np.size(im1, 0)
    img_cols = np.size(im1, 1)  #  get the size of the images

    # create matrix to store all flattened images

    immatrix = np.array([np.array(cv2.imread(im2, cv2.IMREAD_COLOR)).flatten()
                      for im2 in imlist], 'f')

    del imlist, farrowings, nofarrowings, im1
    # immatrix = []
   # arr = []
   # for im2 in imlist:
   #     arr.append(np.array(cv2.imread(im2, cv2.IMREAD_COLOR)).flatten())
   # import sys
   # print(sys.getsizeof(arr))
   # immatrixtemp = np.array(arr[:int(len(arr)/2)], "f")
   # immatrixtemp2 = np.array(arr[int(len(arr)/2):], "f")
   # immatrix.extend(immatrixtemp)
   # immatrix.extend(immatrixtemp2)
    label = np.ones((num_samples,), dtype=int)
    label[0:(nb_class0-1)] = 0
    label[nb_class0:(nb_class0+nb_class1-1)] = 1
    data, Label = shuffle(immatrix, label, random_state=2)
    del immatrix
    train_data = [data, Label]
    del data, Label
    # STEP 1: split X and y into training and testing sets
    x_train, x_test, y_train, y_test = train_test_split(train_data[0], train_data[1], test_size=0.25, random_state=4)
    #x_train, x_val, y_train, y_val = train_test_split(train_data[0], train_data[1], test_size=0.25, random_state=4)
    del train_data
    # Return the sets to their original shapes
    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, num_channels)
    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, num_channels)

    # Scale the values from 0 to 1 instead of 0 to 255
    gc.collect()
    x_train2 = x_train.astype('float32')
    x_test2 = x_test.astype('float32')
    x_train2 /= 255
    x_test2 /= 255

    nofarrowingclass = []
    farrowingclass = []

    return (x_train2, y_train), (x_test2, y_test)


import colander
import deform
from zope.interface import implementer, invariant
from pyramid.threadlocal import get_current_request

from substanced.content import content
from substanced.schema import NameSchemaNode
from substanced.util import renamer, get_oid

from dace.objectofcollaboration.entity import Entity
from dace.descriptors import (
    SharedMultipleProperty, SharedUniqueProperty,
    CompositeUniqueProperty)
from dace.util import get_obj
from pontus.file import File, ObjectData
from pontus.schema import Schema, omit, select
from pontus.core import VisualisableElement, VisualisableElementSchema
from pontus.widget import (
    Select2Widget,
    SimpleMappingWidget,
    AjaxSelect2Widget,
    SequenceWidget)
from deform_treepy.widget import (
    DictSchemaType)

from .interface import ISmartFolder
from novaideo import _, VIEW_TYPES, log, AVAILABLE_LANGUAGES, LANGUAGES_TITLES
from novaideo.widget import (
    CssWidget,
    TextInputWidget,
    BootstrapIconInputWidget
    )
from novaideo.views.filter import FilterSchema
from novaideo.content import get_file_widget

DEFAULT_FOLDER_COLORS = {'usual_color': 'white, #2d6ca2',
                         'hover_color': 'white, #2d6ca2'}

def context_is_a_smartfolder(context, request):
    return request.registry.content.istype(context, 'smartfolder')

@colander.deferred
def view_type_widget(node, kw):
    values = list(VIEW_TYPES.items())
    values.insert(0, ('', _('- Select -')))
    return Select2Widget(css_class="viewtype-field",
                         values=values)

class CssSchema(Schema):
    usual_color = colander.SchemaNode(
        colander.String(),
        widget=CssWidget(),
        title=_('Usual color'),
        # description=('Choisir la couleur du texte et du fond de la section de menu.'),
        description=_('Choose the text and background color of the menu section.'),
        )

    hover_color = colander.SchemaNode(
        colander.String(),
        widget=CssWidget(),
        title=_('Hover color'),
        # description=('Choisir la couleur du texte et du fond de la section de menu au survol de la souris.'),
        description=_('Choose the text and background color of the menu section on mouse-over.')
        )

@colander.deferred
def relatedcontents_choice(node, kw):
    request = node.bindings['request']
    context = node.bindings['context']
    root = request.root
    values = []
    if isinstance(context, SmartFolder):
        values = [(get_oid(t), t.title) for
                  t in context.contents]

    def title_getter(id):
        try:
            obj = get_obj(int(id), None)
            if obj:
                return obj.title
            else:
                return id
        except Exception as e:
            log.warning(e)
            return id

    ajax_url = request.resource_url(
        root, '@@novaideoapi',
        query={'op': 'find_smart_folder_contents'})
    return AjaxSelect2Widget(
        values=values,
        ajax_url=ajax_url,
        ajax_item_template="related_item_template",
        title_getter=title_getter,
        multiple=True,
        page_limit=50,)

@colander.deferred
def locale_widget(node, kw):
    locales = [(l, LANGUAGES_TITLES.get(l, l)) for l in AVAILABLE_LANGUAGES]
    sorted_locales = sorted(locales)
    sorted_locales.insert(0, ('', _('- Select -')))
    return Select2Widget(values=sorted_locales)

class SmartFolderSchema(VisualisableElementSchema):
    """Schema for keyword"""

    name = NameSchemaNode(
        editing=context_is_a_smartfolder,
        )

    title = colander.SchemaNode(
        colander.String(),
        widget=TextInputWidget(css_class="smartfolder-title-field"),
        title=_('Title'),
        )

    description = colander.SchemaNode(
        colander.String(),
        widget=deform.widget.TextAreaWidget(rows=4, cols=60),
        title=_("Description"),
        )

    locale = colander.SchemaNode(
        colander.String(),
        title=_('Locale'),
        description=_('The language for which the folder will be displayed'),
        widget=locale_widget,
        missing=''
    )

    cover_picture = colander.SchemaNode(
        ObjectData(File),
        widget=get_file_widget(file_extensions=['png', 'jpg', 'svg']),
        title=_('Cover picture'),
        missing=None,
        description=_("Only PNG and SVG files are supported."),
        )

    filters = colander.SchemaNode(
        colander.Sequence(),
        omit(select(FilterSchema(
                    name='filter',
                    title=_('Filter'),
                    widget=SimpleMappingWidget(
                        css_class='object-well default-well')),
             ['metadata_filter',
              'temporal_filter', 'contribution_filter',
              'text_filter', 'other_filter']),
             ["_csrf_token_"]),
        widget=SequenceWidget(
            add_subitem_text_template=_('Add a new filter')),
        title=_('Filters'),
        description=_('Applied filters'),
        missing=[]
        )

    contents = colander.SchemaNode(
        colander.Set(),
        widget=relatedcontents_choice,
        title=_('Associated contents'),
        description=_('Choose the contents to be associated'),
        missing=[],
        default=[],
        )

    view_type = colander.SchemaNode(
        colander.String(),
        widget=view_type_widget,
        title=_("View type"),
        description=_('How to display contents'),
        default='default'
        )

    icon_data = colander.SchemaNode(
        DictSchemaType(),
        widget=BootstrapIconInputWidget(),
        title=_('Icon'),
        default={'icon': 'glyphicon-folder-open',
                 'icon_class': 'glyphicon'},
        description=_('Select an icon.')
        )

    style = omit(CssSchema(widget=SimpleMappingWidget()), ["_csrf_token_"])

    @invariant
    def contact_invariant(self, appstruct):
        contents = appstruct.get('contents', [])
        filters = appstruct.get('filters', [])
        if not contents and not filters:
            raise colander.Invalid(
                self, _('Filters or associated contents must be specified.'))

@content(
    'smartfolder',
    icon='glyphicon glyphicon-align-left',
    )
@implementer(ISmartFolder)
class SmartFolder(VisualisableElement, Entity):
    """SmartFolder class"""

    default_icon = 'glyphicon glyphicon-briefcase'
    templates = {
        'default': 'novaideo:views/templates/folder_result.pt',
        'bloc': 'novaideo:views/templates/folder_result.pt',
        'header': 'novaideo:views/templates/folder_header.pt',
    }
    name = renamer()
    children = SharedMultipleProperty('children', 'parents')
    parents = SharedMultipleProperty('parents', 'children')
    author = SharedUniqueProperty('author', 'folders')
    contents = SharedMultipleProperty('contents')
    cover_picture = CompositeUniqueProperty('cover_picture')

    def __init__(self, **kwargs):
        super(SmartFolder, self).__init__(**kwargs)
        self.set_data(kwargs)
        self.folder_order = None

    @property
    def parent(self):
        parents = self.parents
        return parents[0] if parents else None

    @property
    def root(self):
        parent = self.parent
        return parent.root if parent else self

    @property
    def folder_lineage(self):
        result = [self]
        parent = self.parent
        if parent:
            parent_result = parent.folder_lineage
            parent_result.extend(result)
            result = parent_result

        return result

    @property
    def icon(self):
        icon = getattr(self, 'icon_data', {})
        if icon:
            return icon.get('icon_class') + ' ' + icon.get('icon')
        else:
            return self.default_icon

    @property
    def url(self,):
        request = get_current_request()
        return self.get_url(request)

    def contains(self, folder):
        if folder is None:
            return False

        if folder is self:
            return True

        return any(c.contains(folder) for c in self.children)

    def all_sub_folders(self, state=None):
        if state:
            result = [f for f in self.children if state in f.state]
        else:
            result = list(self.children)

        for sub_f in list(result):
            result.extend(sub_f.all_sub_folders(state))

        return list(set(result))

    def get_order(self):
        folder_order = getattr(self, 'folder_order', None)
        if folder_order is None:
            root = self.__parent__
            folders = root.smart_folders
            if self in folders:
                self.set_order(folders.index(self))

        return getattr(self, 'folder_order', None)

    def set_order(self, order):
        self.folder_order = order

    def get_url(self, request):
        return request.resource_url(
            request.root, 'open', query={'folderid': get_oid(self)})


'''
KNN
'''

import numpy as np
import pandas as pd
from sqlalchemy import create_engine
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier

def knn(data):
    '''
    K-Nearest Neighbors
    '''
    sql_engine = create_engine('mysql+pymysql://root:root@localhost/heart')

    db_connection = sql_engine.connect()

    # dataset = pd.read_sql("select * from datasets", db_connection)
    dataset = pd.read_sql("select * from heart.datasets", db_connection)

    pd.set_option('display.expand_frame_repr', False)

    # Lori: 1:-1 remove id from db and jump to age and next
    train_feature = dataset.iloc[:, 1:-1].values
    target = dataset.iloc[:, 14].values

    # Scaling and it uses Z-Score
    scaler = StandardScaler()
    scaler.fit(train_feature)
    # Scaling the dataset feature
    train_scaled = scaler.transform(train_feature)
    # user input
    test_input = np.array(data).reshape(1, -1)
    # Scaling the dataset feature
    test_input_scaled = scaler.transform(test_input)

    # PCA WORKS
    # Make an instance of the Model
    pca = PCA(2)
    # FIT the train set only
    pca.fit(train_scaled)
    # Transform pca to train and test
    train_pca = pca.transform(train_scaled)
    test_input_pca = pca.transform(test_input_scaled)

    # Set the KNN and the K with PCA
    classifier = KNeighborsClassifier(n_neighbors=7)
    classifier.fit(train_pca, target)
    # Scoring
    # result=classifier.score(train_pca, target)
    # Prediction
    result = classifier.predict(test_input_pca)

    # # Show result

    return result[0]


from flask import Flask
from flask_mysqldb import MySQL
from flask import jsonify

back = Flask(__name__)

# Configuration of db
back.config["MYSQL_HOST"] = "localhost"
back.config["MYSQL_USER"] = "mdso"
back.config["MYSQL_PASSWORD"] = "mdso1234"
back.config["MYSQL_DB"] = "mdso"

# Extra configs, optional:
back.config["MYSQL_CURSORCLASS"] = "DictCursor"
back.config["MYSQL_CUSTOM_OPTIONS"] = {"ssl": {"ca": "/path/to/ca-file"}}  # https://mysqlclient.readthedocs.io/user_guide.html#functions-and-attributes

mysql = MySQL(back)

@back.route("/")
def home():
    return "<h1 style='text-align: center;'>Welcome to your back app </h1>"

@back.route("/db/")
def db():
    cur = mysql.connection.cursor()
    cur.execute(""" SELECT * FROM mdsoMember""")
    data = cur.fetchall()
    cur.close()
    return jsonify(data)

@back.route('/identify/<name>/<mail>/')
def identify(name, mail):
    cur = mysql.connection.cursor()
    cur.execute(""" SELECT * FROM mdsoMember WHERE name=%s AND mail=%s """,(name, mail))
    data = cur.fetchall()
    cur.close()
    if( data == [ ] ):
        return jsonify("[{status: False}]")
    else:
        return jsonify(data)

if __name__ == "__main__":
    back.run(debug=True)


# Creates masks for cameras in the active chunk, based on user defined color and tolerance.
#
# This is python script for Metashape Pro. Scripts repository: https://github.com/agisoft-llc/metashape-scripts

import Metashape
from PySide2 import QtGui, QtCore, QtWidgets

# Checking compatibility
compatible_major_version = "1.6"
found_major_version = ".".join(Metashape.app.version.split('.')[:2])
if found_major_version != compatible_major_version:
    raise Exception("Incompatible Metashape version: {} != {}".format(found_major_version, compatible_major_version))

class MaskByColor(QtWidgets.QDialog):

    def __init__(self, parent):
        QtWidgets.QDialog.__init__(self, parent)

        self.color = QtGui.QColor(0, 0, 0)
        red, green, blue = self.color.red(), self.color.green(), self.color.blue()

        self.setWindowTitle("Masking by color:")

        self.btnQuit = QtWidgets.QPushButton("Quit")
        self.btnQuit.setFixedSize(100, 50)

        self.btnP1 = QtWidgets.QPushButton("Mask")
        self.btnP1.setFixedSize(100, 50)

        self.pBar = QtWidgets.QProgressBar()
        self.pBar.setTextVisible(False)
        self.pBar.setFixedSize(130, 50)

        self.selTxt = QtWidgets.QLabel()
        self.selTxt.setText("Apply to:")
        self.selTxt.setFixedSize(100, 25)

        self.radioBtn_all = QtWidgets.QRadioButton("all cameras")
        self.radioBtn_sel = QtWidgets.QRadioButton("selected cameras")
        self.radioBtn_all.setChecked(True)
        self.radioBtn_sel.setChecked(False)

        self.colTxt = QtWidgets.QLabel()
        self.colTxt.setText("Select color:")
        self.colTxt.setFixedSize(100, 25)

        strColor = "{:0>2d}{:0>2d}{:0>2d}".format(int(hex(red)[2:]), int(hex(green)[2:]), int(hex(blue)[2:]))
        self.btnCol = QtWidgets.QPushButton(strColor)
        self.btnCol.setFixedSize(80, 25)
        pix = QtGui.QPixmap(10, 10)
        pix.fill(self.color)
        icon = QtGui.QIcon()
        icon.addPixmap(pix)
        self.btnCol.setIcon(icon)
        palette = QtGui.QPalette()
        palette.setColor(QtGui.QPalette.Button, self.color)
        self.btnCol.setPalette(palette)
        self.btnCol.setAutoFillBackground(True)

        self.txtTol = QtWidgets.QLabel()
        self.txtTol.setText("Tolerance:")
        self.txtTol.setFixedSize(100, 25)

        self.sldTol = QtWidgets.QSlider()
        self.sldTol.setOrientation(QtCore.Qt.Orientation.Horizontal)
        self.sldTol.setMinimum(0)
        self.sldTol.setMaximum(99)

        hbox = QtWidgets.QHBoxLayout()
        hbox.addStretch(1)
        hbox.addWidget(self.pBar)
        hbox.addWidget(self.btnP1)
        hbox.addWidget(self.btnQuit)

        layout = QtWidgets.QGridLayout()
        layout.setSpacing(5)
        layout.addWidget(self.selTxt, 0, 0)
        layout.addWidget(self.radioBtn_all, 1, 0)
        layout.addWidget(self.radioBtn_sel, 2, 0)
        layout.addWidget(self.colTxt, 0, 1)
        layout.addWidget(self.btnCol, 1, 1)
        layout.addWidget(self.txtTol, 0, 2)
        layout.addWidget(self.sldTol, 1, 2)
        layout.addLayout(hbox, 3, 0, 5, 3)
        self.setLayout(layout)

        proc_mask = lambda: self.maskColor()
        proc_color = lambda: self.changeColor()

        QtCore.QObject.connect(self.btnP1, QtCore.SIGNAL("clicked()"), proc_mask)
        QtCore.QObject.connect(self.btnCol, QtCore.SIGNAL("clicked()"), proc_color)
        QtCore.QObject.connect(self.btnQuit, QtCore.SIGNAL("clicked()"), self, QtCore.SLOT("reject()"))

        self.exec()

    def changeColor(self):

        color = QtWidgets.QColorDialog.getColor()

        self.color = color
        red, green, blue = color.red(), color.green(), color.blue()
        if red < 16:
            red = "0" + hex(red)[2:]
        else:
            red = hex(red)[2:]
        if green < 16:
            green = "0" + hex(green)[2:]
        else:
            green = hex(green)[2:]
        if blue < 16:
            blue = "0" + hex(blue)[2:]
        else:
            blue = hex(blue)[2:]

        strColor = red + green + blue
        self.btnCol.setText(strColor)

        pix = QtGui.QPixmap(10, 10)
        pix.fill(self.color)
        icon = QtGui.QIcon()
        icon.addPixmap(pix)
        self.btnCol.setIcon(icon)

        palette = self.btnCol.palette()
        palette.setColor(QtGui.QPalette.Button, self.color)
        self.btnCol.setPalette(palette)
        self.btnCol.setAutoFillBackground(True)

        return True

    def maskColor(self):
        print("Masking...")

        tolerance = 10
        tolerance = self.sldTol.value()

        self.sldTol.setDisabled(True)
        self.btnCol.setDisabled(True)
        self.btnP1.setDisabled(True)
        self.btnQuit.setDisabled(True)

        chunk = Metashape.app.document.chunk
        mask_list = list()
        if self.radioBtn_sel.isChecked():
            for camera in chunk.cameras:
                if camera.selected and camera.type == Metashape.Camera.Type.Regular:
                    mask_list.append(camera)
        elif self.radioBtn_all.isChecked():
            mask_list = [camera for camera in chunk.cameras if camera.type == Metashape.Camera.Type.Regular]

        if not len(mask_list):
            Metashape.app.messageBox("Nothing to mask!")
            return False

        color = self.color
        red, green, blue = color.red(), color.green(), color.blue()

        processed = 0
        for camera in mask_list:

            for frame in camera.frames:
                print(frame)
                app.processEvents()
                mask = Metashape.utils.createDifferenceMask(frame.photo.image(), (red, green, blue), tolerance, False)
                m = Metashape.Mask()
                m.setImage(mask)
                frame.mask = m
                processed += 1
                self.pBar.setValue(int(processed / len(mask_list) / len(chunk.frames) * 100))

        print("Masking finished. " + str(processed) + " images masked.")

        self.sldTol.setDisabled(False)
        self.btnCol.setDisabled(False)
        self.btnP1.setDisabled(False)
        self.btnQuit.setDisabled(False)

        return True

def mask_by_color():
    global doc
    doc = Metashape.app.document

    global app
    app = QtWidgets.QApplication.instance()
    parent = app.activeWindow()

    dlg = MaskByColor(parent)

label = "Custom menu/Masking by color"
Metashape.app.addMenuItem(label, mask_by_color)
print("To execute this script press {}".format(label))


from bs4 import BeautifulSoup
import requests
import urllib.request
import time

'''
被远程主机强制关闭了
'''

pic_link = []
header = {
    'Cookie':'vglnk.Agent.p=b83680e409308d1c7eb0504459135028',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36'}
def get_one_page(url):
    time.sleep(4)
    web_data = requests.get(url, params= header)
    soup = BeautifulSoup(web_data.content, 'html5lib')
    pic_hrefs = soup.select('img.entry-thumbnail')

    for pic_href in pic_hrefs:
        pics = pic_href.get('src')
        pic_link.append(pics)
    print(pic_link)
    i = 1
    for pic in pic_link:
        urllib.request.urlretrieve(pic,'C:\\Users\\ZhangLi\\Desktop\\123\\1.jpg')
        i+=1
        print(i)

def get_all_pic(start, end):
    urls = ['http://weheartit.com/inspirations/taylorswift?scrolling=true&page={}&b'.format(str(i) for i in range(start,end))]
    for url in urls:
        get_one_page(url)

get_all_pic(1,2)

获取异步加载网页数据_python_spider/异步加载数据网页爬取.py

异步加载数据网页爬取.py


# coding:utf-8
# Created by yang
import os

from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = "mysql+pymysql://root:@127.0.0.1:3306/movie"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
app.config["SECRET_KEY"] = "9cc7674436ef46d9bdd9b3d97ac15805"
app.config["UP_DIR"] = os.path.join(os.path.abspath(os.path.dirname(__file__)), "static/uploads/")
app.config["FC_DIR"] = os.path.join(os.path.abspath(os.path.dirname(__file__)), "static/uploads/users/")
app.debug=True
db = SQLAlchemy(app)

from app.home import home as home_blueprint
from app.admin import admin as admin_blueprint

app.register_blueprint(home_blueprint)
app.register_blueprint(admin_blueprint,url_prefix = "/admin")

@app.errorhandler(404)
def page_not_found(error):
    return render_template("home/404.html")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jan 21 14:25:02 2021

@author: miyawaki
"""
#%%
import h5py
from box import Box
import os
import numpy
#%%
class matHandler:
    __filePath=''
    name=Box({})
    def __init__(self,filePath):
        self.__filePath=os.path.expanduser(filePath)
        with h5py.File(self.__filePath,'r') as f:
            self.name=self.__getName(f,'','');

    def getNum(self,path):
        with h5py.File(self.__filePath,'r') as f:
            res=f[path][()]
            if len(res)==1:
                res=res[0]
            if len(res.T)==1:
                res=res.T[0]
        return res

    def getStr(self,path):
        res=[]
        with h5py.File(self.__filePath,'r') as f:
            if f[path][()].dtype=='u2':
                row=(''.join(map(chr, f[path])))
                res.append(row)
            else:
                for col in f[path][()]:
                    row=[]
                    for rIdx in range(len(col)):
                        row.append(''.join(map(chr, map(int,f[col[rIdx]][:]))))
                    res.append(row)
            res=numpy.array(res)
            if len(res)==1:
                res=res[0]
            if len(res.T)==1:
                res=res.T[0]
        return res

    def __getName(self,h5f,keys,path):
        if keys=='':
            # print('key is empty')
            res=self.__getName(h5f,h5f.keys(),'')
            return Box(res)

        res={}
        for k in keys:
            # print('process ' + k)
            if k[0]=='#':
                # print('  key start with #')
                continue

            if isinstance(h5f[k],h5py.Dataset):
                # print('  ' + k + ' is dataset')
                res[k]=path+'/'+k
                # print(k)

            elif isinstance(h5f[k],h5py.Group):
                # print('  ' + k + ' is group')
                res[k]=self.__getName(h5f[k],h5f[k].keys(),path+'/'+k)
            # else:
                # print('  ' + k + ' is something else')

        return Box(res)


from kivy.app import App
from kivy.uix.image import Image, AsyncImage

class MainApp(App):
    def build(self):
        img = AsyncImage(source = 'jeff.png',
                        size_hint = (1, .5),
                        pos_hint = { 'center_x': .5, 'center_y': .5 })
        return img

if __name__ == '__main__':
    app = MainApp()
    app.run()

# from kivy.uix.image import AsyncImage
# class MainApp(App):
#     def build(self):
#         label = Label(text = 'Leonardo Oliveira Santos',
#                       size_hint = (5, .5),
#                       pos_hint = { 'center_x': .5, 'center_y': .5 })
#         return label


import json
import re
from datetime import date, datetime

import six
import validictory
from syncano import PUSH_ENV, logger
from syncano.exceptions import SyncanoFieldError, SyncanoValueError
from syncano.utils import force_text

from .geo import Distance, GeoPoint
from .manager import SchemaManager
from .registry import registry
from .relations import RelationManager, RelationValidatorMixin

class JSONToPythonMixin(object):

    def to_python(self, value):
        if value is None:
            return

        if isinstance(value, six.string_types):
            try:
                value = json.loads(value)
            except (ValueError, TypeError):
                raise SyncanoValueError('Invalid value: can not be parsed')
        return value

class Field(object):
    """Base class for all field types."""

    required = False
    read_only = True
    blank = True
    default = None
    primary_key = False

    has_data = True
    has_endpoint_data = False

    query_allowed = True
    allow_increment = False

    creation_counter = 0
    field_lookups = []

    def __init__(self, name=None, **kwargs):
        self.name = name
        self.model = None
        self.default = kwargs.pop('default', self.default)
        self.required = kwargs.pop('required', self.required)
        self.read_only = kwargs.pop('read_only', self.read_only)
        self.blank = kwargs.pop('blank', self.blank)
        self.label = kwargs.pop('label', None)
        self.mapping = kwargs.pop('mapping', None)
        self.max_length = kwargs.pop('max_length', None)
        self.min_length = kwargs.pop('min_length', None)
        self.query_allowed = kwargs.pop('query_allowed', self.query_allowed)
        self.has_data = kwargs.pop('has_data', self.has_data)
        self.has_endpoint_data = kwargs.pop('has_endpoint_data', self.has_endpoint_data)
        self.primary_key = kwargs.pop('primary_key', self.primary_key)

        # Adjust the appropriate creation counter, and save our local copy.
        self.creation_counter = Field.creation_counter
        Field.creation_counter += 1

    def __repr__(self):
        """Displays current instance class name and field name."""
        return '<{0}: {1}>'.format(self.__class__.__name__, self.name)

    def __eq__(self, other):
        if isinstance(other, Field):
            return self.creation_counter == other.creation_counter
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, Field):
            return self.creation_counter < other.creation_counter
        if isinstance(other, int):
            return self.creation_counter < other
        return NotImplemented

    def __hash__(self):  # pragma: no cover
        return hash(self.creation_counter)

    def __str__(self):
        """Wrapper around ```repr`` method."""
        return repr(self)

    def __unicode__(self):
        """Wrapper around ```repr`` method with proper encoding."""
        return six.u(repr(self))

    def __get__(self, instance, owner):
        if instance is not None:
            return instance._raw_data.get(self.name, self.default)

    def __set__(self, instance, value):
        if self.read_only and value and instance._raw_data.get(self.name):
            logger.debug('Field "{0}"" is read only, '
                         'your changes will not be saved.'.format(self.name))

        instance._raw_data[self.name] = self.to_python(value)

    def __delete__(self, instance):
        if self.name in instance._raw_data:
            del instance._raw_data[self.name]

    def validate(self, value, model_instance):
        """
        Validates the current field instance.

        :raises: SyncanoFieldError
        """
        if self.required and not value:
            raise self.ValidationError('This field is required.')

        if isinstance(value, six.string_types):
            if self.max_length and len(value) > self.max_length:
                raise self.ValidationError('Max length reached.')

            if self.min_length and len(value) < self.min_length:
                raise self.ValidationError('Min length reached.')

    def to_python(self, value):
        """
        Returns field's value prepared for usage in Python.
        """
        if isinstance(value, dict) and 'type' in value and 'value' in value:
            return value['value']

        return value

    def to_native(self, value):
        """
        Returns field's value prepared for serialization into JSON.
        """
        return value

    def to_query(self, value, lookup_type, **kwargs):
        """
        Returns field's value prepared for usage in HTTP request query.
        """
        if not self.query_allowed:
            raise self.ValidationError('Query on this field is not supported.')

        return self.to_native(value)

    def contribute_to_class(self, cls, name):
        if name in cls._meta.endpoint_fields:
            self.has_endpoint_data = True

        if not self.name:
            self.name = name

        if not self.label:
            self.label = self.name.replace('_', ' ').capitalize()

        if self.primary_key:
            if cls._meta.pk:
                raise SyncanoValueError('Multiple pk fields detected.')

            cls._meta.pk = self
            setattr(cls, 'pk', self)

        self.model = cls
        cls._meta.add_field(self)
        setattr(cls, name, self)

        error_class = type(
            '{0}ValidationError'.format(self.__class__.__name__),
            (SyncanoFieldError, ),
            {'field_name': name}
        )

        setattr(self, 'ValidationError', error_class)

class RelatedManagerField(Field):

    def __init__(self, model_name, endpoint='list', *args, **kwargs):
        super(RelatedManagerField, self).__init__(*args, **kwargs)
        self.model_name = model_name
        self.endpoint = endpoint

    def __get__(self, instance, owner=None):
        if instance is None:
            raise AttributeError("RelatedManager is accessible only via {0} instances.".format(owner.__name__))

        Model = registry.get_model_by_name(self.model_name)
        method = getattr(Model.please, self.endpoint, Model.please.all)
        properties = instance._meta.get_endpoint_properties('detail')
        properties = [getattr(instance, prop) for prop in properties]
        return method(*properties)

    def contribute_to_class(self, cls, name):
        setattr(cls, name, self)

class PrimaryKeyField(Field):
    primary_key = True

class WritableField(Field):
    required = True
    read_only = False

class EndpointField(WritableField):
    has_data = False
    has_endpoint_data = True

class StringField(WritableField):

    field_lookups = [
        'startswith',
        'endswith',
        'contains',
        'istartswith',
        'iendswith',
        'icontains',
        'ieq',
    ]

    def to_python(self, value):
        value = super(StringField, self).to_python(value)

        if isinstance(value, six.string_types) or value is None:
            return value
        return force_text(value)

class IntegerField(WritableField):
    allow_increment = True

    def to_python(self, value):
        value = super(IntegerField, self).to_python(value)

        if value is None:
            return
        try:
            return int(value)
        except (TypeError, ValueError):
            raise self.ValidationError('Invalid value. Value should be an integer.')

class ReferenceField(IntegerField):

    def to_python(self, value):
        if isinstance(value, int):
            return value

        if hasattr(value, 'pk') and isinstance(value.pk, int):
            value = value.pk

        return super(ReferenceField, self).to_python(value)

class FloatField(WritableField):
    allow_increment = True

    def to_python(self, value):
        value = super(FloatField, self).to_python(value)

        if value is None:
            return
        try:
            return float(value)
        except (TypeError, ValueError):
            raise self.ValidationError('Invalid value. Value should be a float.')

class BooleanField(WritableField):

    def to_python(self, value):
        value = super(BooleanField, self).to_python(value)

        if value is None:
            return

        if value in (True, 't', 'true', 'True', '1'):
            return True

        if value in (False, 'f', 'false', 'False', '0'):
            return False

        raise self.ValidationError('Invalid value. Value should be a boolean.')

class SlugField(StringField):
    regex = re.compile(r'^[-a-zA-Z0-9_]+$')

    def validate(self, value, model_instance):
        super(SlugField, self).validate(value, model_instance)

        if not isinstance(value, six.string_types):
            raise self.ValidationError('Invalid value. Value should be a string.')

        if not bool(self.regex.search(value)):
            raise self.ValidationError('Invalid value.')
        return value

class EmailField(StringField):
    regex = re.compile(r'(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)')

    def validate(self, value, model_instance):
        super(EmailField, self).validate(value, model_instance)

        if not isinstance(value, six.string_types):
            raise self.ValidationError('Invalid value. Value should be a string.')

        if not value or '@' not in value:
            raise self.ValidationError('Enter a valid email address.')

        if not bool(self.regex.match(value)):
            raise self.ValidationError('Enter a valid email address.')

class ChoiceField(WritableField):

    def __init__(self, *args, **kwargs):
        self.choices = kwargs.pop('choices', [])
        self.allowed_values = [choice['value'] for choice in self.choices]
        super(ChoiceField, self).__init__(*args, **kwargs)

    def validate(self, value, model_instance):
        super(ChoiceField, self).validate(value, model_instance)
        if self.choices and value is not None and value not in self.allowed_values:
            raise self.ValidationError("Value '{0}' is not a valid choice.".format(value))

class DateField(WritableField):
    date_regex = re = re.compile(
        r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})$'
    )

    def to_python(self, value):
        value = super(DateField, self).to_python(value)

        if value is None:
            return

        if isinstance(value, datetime):
            return value.date()

        if isinstance(value, date):
            return value

        if isinstance(value, (int, float)):
            dt = datetime.fromtimestamp(value)
            return dt.date()

        try:
            parsed = self.parse_date(value)
            if parsed is not None:
                return parsed
        except (ValueError, TypeError):
            pass

        raise self.ValidationError("'{0}' value has an invalid date format. It must be "
                                   "in YYYY-MM-DD format.".format(value))

    def parse_date(self, value):
        match = self.date_regex.match(value)
        if match:
            kw = {k: int(v) for k, v in six.iteritems(match.groupdict())}
            return date(**kw)

    def to_native(self, value):
        if isinstance(value, datetime):
            value = value.date()
        return value.isoformat()

class DateTimeField(DateField):
    FORMAT = '%Y-%m-%dT%H:%M:%S.%f'

    def to_python(self, value):
        if value is None:
            return

        if isinstance(value, dict) and 'type' in value and 'value' in value:
            value = value['value']

        if isinstance(value, datetime):
            return value

        if isinstance(value, date):
            return datetime(value.year, value.month, value.day)

        if isinstance(value, (int, float)):
            return datetime.fromtimestamp(value)

        if isinstance(value, six.string_types):
            value = value.split('Z')[0]

        parsers = [
            self.parse_from_string,
            self.parse_from_date,
        ]

        for parser in parsers:
            try:
                value = parser(value)
            except (ValueError, TypeError):
                pass
            else:
                return value

        raise self.ValidationError("'{0}' value has an invalid format. It must be in "
                                   "YYYY-MM-DD HH:MM[:ss[.uuuuuu]] format.".format(value))

    def parse_from_string(self, value):
        return datetime.strptime(value, self.FORMAT)

    def parse_from_date(self, value):
        parsed = self.parse_date(value)

        if not parsed:
            raise ValueError

        return datetime(parsed.year, parsed.month, parsed.day)

    def to_native(self, value):
        if value is None:
            return
        ret = value.strftime(self.FORMAT)
        if ret.endswith('+00:00'):
            ret = ret[:-6] + 'Z'

        if not ret.endswith('Z'):
            ret = ret + 'Z'

        return ret

class LinksWrapper(object):

    def __init__(self, links_dict, ignored_links):
        self.links_dict = links_dict
        self.ignored_links = ignored_links

    def __getattribute__(self, item):
        try:
            return super(LinksWrapper, self).__getattribute__(item)
        except AttributeError:
            value = self.links_dict.get(item)
            if not value:
                item = item.replace('_', '-')
                value = self.links_dict.get(item)

            if not value:
                raise

            return value

    def to_native(self):
        return self.links_dict

class LinksField(Field):
    query_allowed = False
    IGNORED_LINKS = ('self', )

    def __init__(self, *args, **kwargs):
        super(LinksField, self).__init__(*args, **kwargs)

    def to_python(self, value):
        return LinksWrapper(value, self.IGNORED_LINKS)

    def to_native(self, value):
        return value.to_native()

class ModelField(Field):

    def __init__(self, rel, *args, **kwargs):
        self.rel = rel
        self.just_pk = kwargs.pop('just_pk', True)
        self.is_data_object_mixin = kwargs.pop('is_data_object_mixin', False)
        super(ModelField, self).__init__(*args, **kwargs)

    def contribute_to_class(self, cls, name):
        super(ModelField, self).contribute_to_class(cls, name)

        if isinstance(self.rel, six.string_types):

            def lazy_relation(cls, field):
                if isinstance(field.rel, six.string_types):
                    field.rel = registry.get_model_by_name(field.rel)

            try:
                self.rel = registry.get_model_by_name(self.rel)
            except LookupError:
                value = (lazy_relation, (cls, self), {})
                registry._pending_lookups.setdefault(self.rel, []).append(value)
            else:
                lazy_relation(cls, self)

    def validate(self, value, model_instance):
        super(ModelField, self).validate(value, model_instance)

        if not isinstance(value, (self.rel, dict)):
            if not isinstance(value, (self.rel, dict)) and not self.is_data_object_mixin:
                raise self.ValidationError('Value needs to be a {0} instance.'.format(self.rel.__name__))

        if (self.required and isinstance(value, self.rel)) or \
           (self.is_data_object_mixin and hasattr(value, 'validate')):
            value.validate()

    def to_python(self, value):

        if value is None:
            return

        if isinstance(value, self.rel):
            return value

        if isinstance(value, dict):
            return self.rel(**value)

        raise self.ValidationError("'{0}' has unsupported format.".format(value))

    def to_native(self, value):
        if value is None:
            return

        if isinstance(value, self.rel):
            if not self.just_pk:
                return value.to_native()

            pk_field = value._meta.pk
            pk_value = getattr(value, pk_field.name)
            return pk_field.to_native(pk_value)

        if self.is_data_object_mixin and not self.just_pk and hasattr(value, 'to_native'):
            return value.to_native()

        return value

class FileField(WritableField):
    param_name = 'files'

    def to_native(self, value):
        if isinstance(value, six.string_types):
            return None
        return {self.name: value}

class JSONField(JSONToPythonMixin, WritableField):
    query_allowed = False
    schema = None

    def __init__(self, *args, **kwargs):
        self.schema = kwargs.pop('schema', None) or self.schema
        super(JSONField, self).__init__(*args, **kwargs)

    def validate(self, value, model_instance):
        super(JSONField, self).validate(value, model_instance)
        if self.schema:
            try:
                validictory.validate(value, self.schema)
            except ValueError as e:
                raise self.ValidationError(e)

    def to_native(self, value):
        if value is None:
            return

        if not isinstance(value, six.string_types):
            value = json.dumps(value)
        return value

class ArrayField(JSONToPythonMixin, WritableField):

    def validate(self, value, model_instance):
        super(ArrayField, self).validate(value, model_instance)

        if not self.required and not value:
            return

        if isinstance(value, six.string_types):
            try:
                value = json.loads(value)
            except (ValueError, TypeError):
                raise SyncanoValueError('Expected an array')

        if isinstance(value, dict):
            if len(value) != 1 or len(set(value.keys()).intersection(['_add', '_remove', '_addunique'])) != 1:
                raise SyncanoValueError('Wrong value: one operation at the time.')

        elif not isinstance(value, list):
            raise SyncanoValueError('Expected an array')

        value_to_check = value if isinstance(value, list) else value.values()[0]

        for element in value_to_check:
            if not isinstance(element, six.string_types + (bool, int, float)):
                raise SyncanoValueError(
                    'Currently supported types for array items are: string types, bool, float and int')

class ObjectField(JSONToPythonMixin, WritableField):

    def validate(self, value, model_instance):
        super(ObjectField, self).validate(value, model_instance)

        if not self.required and not value:
            return

        if isinstance(value, six.string_types):
            try:
                value = json.loads(value)
            except (ValueError, TypeError):
                raise SyncanoValueError('Expected an object')

        if not isinstance(value, dict):
            raise SyncanoValueError('Expected an object')

class SchemaField(JSONField):
    required = False
    query_allowed = False
    not_indexable_types = ['text', 'file']
    schema = {
        'type': 'array',
        'items': {
            'type': 'object',
            'properties': {
                'name': {
                    'type': 'string',
                    'required': True,
                },
                'type': {
                    'type': 'string',
                    'required': True,
                    'enum': [
                        'string',
                        'text',
                        'integer',
                        'float',
                        'boolean',
                        'datetime',
                        'file',
                        'reference',
                        'relation',
                        'array',
                        'object',
                        'geopoint',
                    ],
                },
                'order_index': {
                    'type': 'boolean',
                    'required': False,
                },
                'filter_index': {
                    'type': 'boolean',
                    'required': False,
                },
                'target': {
                    'type': 'string',
                    'required': False,
                }
            }
        }
    }

    def validate(self, value, model_instance):
        if value is None:
            return

        if isinstance(value, SchemaManager):
            value = value.schema

        super(SchemaField, self).validate(value, model_instance)

        fields = [f['name'] for f in value]
        if len(fields) != len(set(fields)):
            raise self.ValidationError('Field names must be unique.')

        for field in value:
            is_not_indexable = field['type'] in self.not_indexable_types
            has_index = ('order_index' in field or 'filter_index' in field)
            if is_not_indexable and has_index:
                raise self.ValidationError('"{0}" type is not indexable.'.format(field['type']))

    def to_python(self, value):
        if isinstance(value, SchemaManager):
            return value

        value = super(SchemaField, self).to_python(value)
        return SchemaManager(value)

    def to_native(self, value):
        if isinstance(value, SchemaManager):
            value = value.schema

        return super(SchemaField, self).to_native(value)

class PushJSONField(JSONField):
    def to_native(self, value):
        if value is None:
            return

        if not isinstance(value, six.string_types):
            if 'environment' not in value:
                value.update({
                    'environment': PUSH_ENV,
                })
            value = json.dumps(value)
        return value

class ListField(WritableField):

    def validate(self, value, model_instance):
        if value is None:
            return

        if not isinstance(value, list):
            raise self.ValidationError('List expected.')

class GeoPointField(Field):

    field_lookups = ['near', 'exists']

    def validate(self, value, model_instance):
        super(GeoPointField, self).validate(value, model_instance)

        if not self.required and not value:
            return

        if isinstance(value, six.string_types):
            try:
                value = json.loads(value)
            except (ValueError, TypeError):
                raise SyncanoValueError('Expected an object')

        if not isinstance(value, GeoPoint):
            raise SyncanoValueError('Expected a GeoPoint')

    def to_native(self, value):
        if value is None:
            return

        if isinstance(value, bool):
            return value  # exists lookup

        if isinstance(value, dict):
            value = GeoPoint(latitude=value['latitude'], longitude=value['longitude'])

        if isinstance(value, tuple):
            geo_struct = value[0].to_native()
        else:
            geo_struct = value.to_native()

        geo_struct = json.dumps(geo_struct)

        return geo_struct

    def to_query(self, value, lookup_type, **kwargs):
        """
        Returns field's value prepared for usage in HTTP request query.
        """
        super(GeoPointField, self).to_query(value, lookup_type, **kwargs)

        if lookup_type not in self.field_lookups:
            raise SyncanoValueError('Lookup {} not supported for geopoint field'.format(lookup_type))

        if lookup_type in ['exists']:
            if isinstance(value, bool):
                return value
            else:
                raise SyncanoValueError('Bool expected in {} lookup.'.format(lookup_type))

        if isinstance(value, dict):
            value = (
                GeoPoint(latitude=value.pop('latitude'), longitude=value.pop('longitude')),
                Distance(**value)
            )

        if len(value) != 2 or not isinstance(value[0], GeoPoint) or not isinstance(value[1], Distance):
            raise SyncanoValueError('This lookup should be a tuple with GeoPoint and Distance: '
                                    '<field_name>__near=(GeoPoint(52.12, 22.12), Distance(kilometers=100))')

        query_dict = value[0].to_native()
        query_dict.update(value[1].to_native())

        return query_dict

    def to_python(self, value):
        if value is None:
            return

        value = self._process_string_types(value)

        if isinstance(value, GeoPoint):
            return value

        latitude, longitude = self._process_value(value)

        if not latitude or not longitude:
            raise SyncanoValueError('Expected the `longitude` and `latitude` fields.')

        return GeoPoint(latitude=latitude, longitude=longitude)

    @classmethod
    def _process_string_types(cls, value):
        if isinstance(value, six.string_types):
            try:
                return json.loads(value)
            except (ValueError, TypeError):
                raise SyncanoValueError('Invalid value: can not be parsed.')
        return value

    @classmethod
    def _process_value(cls, value):
        longitude = None
        latitude = None

        if isinstance(value, dict):
            latitude = value.get('latitude')
            longitude = value.get('longitude')
        elif isinstance(value, (tuple, list)):
            try:
                latitude = value[0]
                longitude = value[1]
            except IndexError:
                raise SyncanoValueError('Can not parse the geo point.')

        return latitude, longitude

class RelationField(RelationValidatorMixin, WritableField):
    query_allowed = True
    field_lookups = ['contains', 'is']

    def __call__(self, instance, field_name):
        return RelationManager(instance=instance, field_name=field_name)

    def to_python(self, value):
        if not value:
            return None

        if isinstance(value, dict) and 'type' in value and 'value' in value:
            value = value['value']

        if isinstance(value, dict) and ('_add' in value or '_remove' in value):
            return value

        if not isinstance(value, (list, tuple)):
            return [value]

        return value

    def to_query(self, value, lookup_type, related_field_name=None, related_field_lookup=None, **kwargs):

        if not self.query_allowed:
            raise self.ValidationError('Query on this field is not supported.')

        if lookup_type not in self.field_lookups:
            raise SyncanoValueError('Lookup {} not supported for relation field.'.format(lookup_type))

        query_dict = {}

        if lookup_type == 'contains':
            if self._check_relation_value(value):
                value = [obj.id for obj in value]
            query_dict = value

        if lookup_type == 'is':
            query_dict = {related_field_name: {"_{0}".format(related_field_lookup): value}}

        return query_dict

    def to_native(self, value):
        if not value:
            return None

        if isinstance(value, dict) and ('_add' in value or '_remove' in value):
            return value

        if not isinstance(value, (list, tuple)):
            value = [value]

        if self._check_relation_value(value):
            value = [obj.id for obj in value]
        return value

MAPPING = {
    'string': StringField,
    'text': StringField,
    'file': FileField,
    'ref': StringField,
    'reference': ReferenceField,
    'relation': RelationField,
    'integer': IntegerField,
    'float': FloatField,
    'boolean': BooleanField,
    'name': SlugField,
    'email': EmailField,
    'choice': ChoiceField,
    'date': DateField,
    'datetime': DateTimeField,
    'field': Field,
    'writable': WritableField,
    'endpoint': EndpointField,
    'links': LinksField,
    'model': ModelField,
    'json': JSONField,
    'schema': SchemaField,
    'array': ArrayField,
    'object': ObjectField,
    'geopoint': GeoPointField,
}


#!/usr/bin/python3
"""
A node that bridges ROS topics to a physical CANbus interface through the socketcan interface.

Ported to Python from the ros2can_bridge package by philippwuestenberg.
https://github.com/ROS4SPACE/ros2can_bridge

Date created: 05/01/2023
Date modified: 05/05/2023

CHANGELOG:
v1.0.0 - Initial Release
v1.0.1 - Changed some of the parameter calls and variable names
v1.0.2 - Updated docs

TODO:
- Add exception handling for KeyboardInterrupt (close CAN bus)
- Add exception handling for can.exceptions.CanOperationError: Failed to transmit: No buffer space available (stop transmitting until bus opens up)
"""

import can
import rclpy
from rclpy.node import Node
from seacat_msg.msg import CANFrame

__author__ = "Braidan Duffy"
__copyright__ = "Copyright 2023, PANTHER Boat Team"
__credits__ = ["Braidan Duffy"]
__license__ = "MIT"
__version__ = "1.0.2"
__maintainer__ = "Braidan Duffy"
__email__ = "bduffy2018@my.fit.edu"
__status__ = "Production"

class ROS2CANNode(Node):
    """
    Node that bridges software ROS topics to the physical world via a physical CAN bus and the socketcan interface

    Parameters:
        bus_name (str): the CAN bus interface the VESC is connected. Default: 'can0'
    """
    BUS_NAME: str
    bus: can.interface.Bus

    def __init__(self):
        super().__init__('ROS2CANNode')

        self.declare_parameter('bus_name', 'can0')

        # Set up the CAN Interface
        self.BUS_NAME = self.get_parameter('bus_name').get_parameter_value().string_value
        self.bus = can.interface.Bus(channel=self.BUS_NAME, bustype='socketcan')
        self.get_logger().info(f"Using bus: {self.BUS_NAME}")

        if not self.bus: # Check if bus successfully created
            self.get_logger().fatal("Bus already open!")
            raise RuntimeError(f"CAN Bus '{self.BUS_NAME}' is already in use!")

        # Set up topic names
        self.topicname_receive = f"CAN/{self.BUS_NAME}/receive"
        self.topicname_transmit = f"CAN/{self.BUS_NAME}/transmit"

        # Set up the ROS2 publisher/subscribers
        self.publisher_ = self.create_publisher(CANFrame,
                                                self.topicname_receive,
                                                10)
        self.get_logger().info(f"Publishing received CAN frames to: {self.topicname_transmit}")

        self.subscription_ = self.create_subscription(CANFrame,
                                                      self.topicname_transmit,
                                                      self.CAN_send,
                                                      10)
        self.get_logger().info(f"Listening for CAN frames at: {self.topicname_receive}")

        # Create a timer to call the listener function at 10 Hz
        self.timer_listener = self.create_timer(0.1, self.CAN_listener)
        self.get_logger().info("Created a timer to check for new CAN bus messages at 10 Hz")

    # =============================
    # === SUBSCRIPTION HANDLERS ===
    # =============================

    def CAN_send(self, msg: CANFrame):
        """
        Formats a CAN bus message from a ROS CAN frame and sends it to the bus through the socketcan interface

        Args:
            msg (CANFrame): A CAN frame sent from within the ROS network
        """
        self.bus.send(can.Message(arbitration_id=msg.id,
                                  is_extended_id=msg.eff,
                                  is_remote_frame=msg.rtr,
                                  is_error_frame=msg.err,
                                  dlc=msg.dlc,
                                  data=msg.data))

        self.get_logger().info(f"Sent CAN message with ID: {msg.id} and Data: {msg.data}")

    def CAN_listener(self):
        """
        Listens for a message from the CAN bus and then publishes it to a ROS topic
        """
        can_msg = self.bus.recv()	                            # Receive a CAN message
        message = CANFrame(id=can_msg.arbitration_id,	        # Extract the CAN ID from the receieved message
                           dlc=can_msg.dlc,                     # Extract the data length
                           data=can_msg.data,		            # Extract the data from the received message
                           err=can_msg.error_state_indicator,   # Extract if the frame is an ERROR f
                           rtr=can_msg.is_remote_frame,         # Extract if the frame is a REQUEST frame
                           eff=can_msg.is_extended_id)          # Extract if the frame uses an extended id
        self.get_logger().info(f'Received CAN message: ID={message.id}, Data={message.data}')

        self.publisher_.publish(message)                        # Publish the ROS2 message to the "can_data" topic

def main(args=None):
    rclpy.init(args=args)
    node = ROS2CANNode()
    rclpy.spin(node)
    node.destroy_node()
    rclpy.shutdown()

if __name__ == '__main__':
	main()


from selenium import webdriver
from selenium.webdriver.common.keys import Keys # 엔터키 치기 위한 라이브러리
import time
import csv

f = open(r"C:\Users\82107\Desktop\data.csv", 'w', encoding='cp949', newline='')
# 경로, 쓰기모드, 인코딩(한글 깨짐 방지), 자동 줄바꿈 없음
csvwriter = csv.writer(f)

browser = webdriver.Chrome('C:/chromedriver.exe') # 브라우져 생성
browser.get('https://www.naver.com') # 웹 사이트 열기
browser.implicitly_wait(10) # 브라우져를 여는데 시간이 많이 걸리면 이어지는 명령어 수행이 불가하므로 시간 초 여유를 준다.

browser.find_element_by_css_selector('a.nav.shop').click() # 네이버 메인에서 쇼핑 클릭
time.sleep(2) # 2초간의 시간 여유를 준다.

search = browser.find_element_by_css_selector('input._searchInput_search_input_QXUFf') # 검색창
search.click()

search.send_keys('아이폰 13')
search.send_keys(Keys.ENTER)

# 스크롤 전 높이(자바스크립트 명령어를 사용할 수 있게 도와줌 execute_script)
before_h = browser.execute_script("return window.scrollY")
# 자바 스크립트로 return window.scrollY를 해준다는 말이다.

# 무한(반복문) 스크롤
while True:
    # 맨 아래로 스크롤을 내린다.
    browser.find_element_by_css_selector("body").send_keys(Keys.END) # END 키를 눌러주기
    time.sleep(2) # 스크롤 사이 페이지 로딩 시간
    after_h = browser.execute_script("return window.scrollY")

    if after_h == before_h:
        break # 더 이상 내릴 수 없을 때까지 내린다는 의미
    before_h = after_h

# 상품 정보 div
items = browser.find_elements_by_css_selector(".basicList_info_area__17Xyo")
for item in items:
    name = item.find_element_by_css_selector(".basicList_title__3P9Q7").text # 검색했을 때 나오는 개수가 같아야 한다.
    try:
        price = item.find_element_by_css_selector("span.price_num__2WUXn").text
    except:
        price = "판매중단" # 가격 정보가 없으면 판매중단 이라고 처리해준다.
    link = item.find_element_by_css_selector(".basicList_title__3P9Q7 > a").get_attribute('href')
    # 자식클래스 선택 basicList_title__3P9Q7 아래 a태그
    # print(name, price, link)
    csvwriter.writerow([name, price, link])

# 파일 닫기
f.close()

f = open(r"C:\Users\82107\Desktop\data.csv", 'r')


from django.conf.urls import url
from django.urls import path
from . import views

app_name = "accommodation"

urlpatterns = [
    url(r'^$', views.index_view, name='index'),
    url(r'^index/$', views.index_view, name='index'),
    url(r'^reservation/$', views.reservation_view, name='reservation'),
    url(r'^property/$', views.property_view, name='property'),
    url(r'^imageUploader/$', views.imageUploader_view, name='imageUploader'),
    path('propertyDetailed/<int:pk>/', views.propertyDetailed_view, name='propertyDetailed'),
    path('delete/<int:pk>/', views.deleteReservation_view, name='deleteReservation'),
]


#!/usr/bin/env python3
import click
import click_log

# Set up logging:
from .. import get_logger

logger = get_logger(__name__)

from . import table

from ...lib import utils

# Parse the expressions:
import ast
import builtins

# Loading the data:
import pyarrow as pa
import pyarrow.parquet as pq
import h5py
import pandas as pd

# Read the arguments:
@table.command()
@click.argument("column_schema", type=click.Path(exists=True))
@click.argument("output_file", type=click.Path(exists=False))
@click.argument("in_paths", nargs=-1, type=click.Path(exists=True))
@click.option(
    "-i",
    "--in-format",
    help="Type of input.",
    type=click.Choice(["TSV", "CSV", "PARQUET", "HDF5", "AUTO"], case_sensitive=False),
    required=False,
    default="auto",
)
@click.option(
    "-o",
    "--out-format",
    help="Type of output_file. Same as input for 'auto'",
    type=click.Choice(["TSV", "CSV", "PARQUET", "HDF5", "AUTO"], case_sensitive=False),
    required=False,
    default="auto",
)
def evaluate(column_schema, output_file, in_paths, in_format, out_format):
    """Create new columns according to the input expression.
    The result of evaluation will be a vector of type column_format with the number of
    entries equal to the input size of array columns.

    Column schema should be tab-separated and contain three columns:
    column_name, column_format and column_expression.

    **Column_name** is the name of the output column with evaluation result.
    **Column expression** is one-liner that does not contain lambda expressions,
    list comprehensions, and can use only column names from input parquets as variables,
    built-in functions and numpy for their evaluation.
    **Column format** is one of the following: str, int, int8, int16, int32, bool.
    """

    import numpy as np  # Import is within the function to guarantee the visibility in vars()

    prohibited_symbols = [":", ".", "-", "/", "!", "?", "&", "|", "'", "%", "@"]
    match = utils.match # define matcher; this now can be used as "match(col, expr)" in the evaluation scheme
    pick_positions = utils.pick_positions
    pick_smallest = utils.pick_smallest
    pick_largest = utils.pick_largest

    # Guess format if not specified:
    if in_format.upper() == "AUTO":
        in_format = utils.guess_format(in_paths[0])
    if out_format.upper() == "AUTO":
        out_format = in_format

    input_tables = utils.load_tables(in_paths, in_format)

    if out_format.upper() == "HDF5":
        h = h5py.File(output_file, "w")

    loaded_arrays = {}
    n_evaluated = 0
    with open(column_schema, "r") as input_file:
        # Iterate over each expression:
        for line in input_file.readlines():
            column_name, column_format, column_expression = line.strip().split("\t")

            assert np.all(
                [x not in column_name for x in prohibited_symbols]
            ), "Check the column name. It cannot contain " + ",".join(
                prohibited_symbols
            )

            # Load dataset keys as variables:
            loaded_ids = []
            syntax_tree = ast.parse(column_expression)
            for node in ast.walk(syntax_tree):
                if type(node) is ast.Name:
                    # The element is not loaded yet and is not a builtin name:
                    if not (
                        (node.id in list(vars().keys())) or (node.id in dir(builtins))
                    ):
                        # Element is already loaded:
                        if node.id in loaded_arrays.keys():
                            if in_format.upper() == "PARQUET":
                                if loaded_arrays[node.id].type == pa.bool_():
                                    vars()[node.id] = loaded_arrays[node.id].to_numpy(
                                        zero_copy_only=False
                                    )
                                else:
                                    vars()[node.id] = loaded_arrays[node.id].to_numpy()
                            elif in_format.upper() == "HDF5":
                                vars()[node.id] = loaded_arrays[node.id][()]
                            else:
                                vars()[node.id] = loaded_arrays[node.id]

                            loaded_ids.append(str(node.id))

                        # Element is not loaded, check the input tables:
                        else:
                            is_found = False
                            for table in input_tables:
                                if in_format.upper() == "PARQUET":
                                    if node.id in table.column_names:
                                        vars()[node.id] = table[node.id].to_numpy()
                                        is_found = True
                                elif in_format.upper() == "HDF5":
                                    if node.id in table.keys():
                                        vars()[node.id] = table[node.id][()]
                                        is_found = True
                                else:
                                    if node.id in table.columns:
                                        vars()[node.id] = table.loc[:, node.id]
                                        is_found = True
                                if is_found:
                                    loaded_ids.append(str(node.id))
                                    break

                            if not is_found:
                                if in_format.upper() == "PARQUET":
                                    avail_colnames = [
                                        list(table.column_names)
                                        for table in input_tables
                                    ] + [list(loaded_arrays.keys())]
                                elif in_format.upper() == "HDF5":
                                    avail_colnames = [
                                        list(table.keys()) for table in input_tables
                                    ] + [list(loaded_arrays.keys())]
                                else:
                                    avail_colnames = [
                                        list(table.columns) for table in input_tables
                                    ] + [list(loaded_arrays.keys())]
                                raise ValueError(
                                    f"Variable {node.id} is not available from input/created pyarrow file. "
                                    f"List of variables that can be loaded:\n{ str( avail_colnames ) }"
                                )

            # Evaluate expression:
            logger.debug(
                f"Evaluating column: {column_name}, expression: {column_expression} "
            )
            result = eval(column_expression)

            # Remove unused variables:
            for v in loaded_ids:
                del vars()[v]

            loaded_arrays.update(
                utils.dump_columns(result, in_format, column_format, column_name)
            )
            n_evaluated += 1

    if n_evaluated > 0:
        logger.info(
            f"Evaluated {n_evaluated} expressions, including columns: {', '.join(loaded_arrays.keys())}"
        )

        utils.dump_arrays(
            loaded_arrays, out_format, h if in_format.upper() == "HDF5" else output_file
        )

    else:
        logger.info(
            "No evaluated expression. Is the input table with expressions empty?"
        )

    return 0


import json
import os

HERE = os.path.split(__file__)[0]

def build_model_search_space(task_type: str, data_path: str, **kwargs):
    search_space = {
        "task_type": {"_type": "choice", "_value": []},
        "data_path": {"_type": "choice", "_value": [data_path]},
    }
    if data_path.endswith(".csv"):
        assert "y_col" in kwargs, "need to specify y_col for csv"
        search_space["y_col"] = {"_type": "choice", "_value": [kwargs["y_col"]]}
        if "X_cols" in kwargs:
            search_space["X_cols"] = {"_type": "choice", "_value": [kwargs["X_cols"]]}
        elif "drop_cols" in kwargs:
            search_space["drop_cols"] = {
                "_type": "choice",
                "_value": [kwargs["drop_cols"]],
            }
        else:
            raise Exception("need to specify either X_cols or drop_cols")

    hps = json.load(open(f"{HERE}/search/model_search/hp.json"))
    models = json.load(open(f"{HERE}/search/model_search/model_select.json"))

    classifier_choices_with_hps = []
    for classifier in models["classification"]:
        classifier_choices_with_hps.append({"_name": classifier, **hps[classifier]})
    regressor_choices_with_hps = []
    for regressor in models["regression"]:
        regressor_choices_with_hps.append({"_name": regressor, **hps[regressor]})

    if task_type == "classification":
        search_space["task_type"]["_value"].append(
            {
                "_name": "classification",
                "model": {"_type": "choice", "_value": classifier_choices_with_hps},
            }
        )
    elif task_type == "regression":
        search_space["task_type"]["_value"].append(
            {
                "_name": "regression",
                "model": {"_type": "choice", "_value": regressor_choices_with_hps},
            }
        )
    else:
        raise Exception(f"unsupported task type {task_type}")
    return search_space

if __name__ == "__main__":
    print(__file__)


import numpy as np
import scipy.io as sio
import random
import torch
import weighted_sum
import divide
import accuracy

def rdm(X_test):
    num=1
    for i in range( len(X_test) ):
        Y_local = []
        for i in range(20):
            y_ = random.random()
            if y_>0.5:
                Y_local.append(1)
            elif y_<0.5:
                Y_local.append(0)

        if num == 1:
            Y_l = Y_local
        elif num > 1:
            Y_l = np.vstack((Y_l, Y_local))
        num = num + 1
    return  Y_l

t_setx=sio.loadmat('./data/t_setx.mat')['t_setx']
t_setx=np.array(t_setx)
d_set6=sio.loadmat('./data/d_set6.mat')['d_set6']
d_set6=np.array(d_set6)
X_train, t_set = divide.divide(t_setx)
Y_train, d_set = divide.divide(d_set6)

y_local = rdm(t_set)
per = weighted_sum.per_q(t_set, y_local)
p_a= accuracy.per_acu(d_set , y_local)


from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.template.defaultfilters import slugify
import requests
from bs4 import BeautifulSoup
import pandas as pd

@api_view(['GET'])
def country_info(request, pk=None):
    idk=pk

    country= idk.replace(" ", '_')
    wiki_link = "https://en.wikipedia.org/wiki/" + country

    dic = {
        'flag_link': 'https://upload.wikimedia.org/wikipedia/en/4/41/Flag_of_India.svg',
        'capital': 'New Delhi',
        'largest_city': ['Mumbai', 'New Delhi'],
        'official_languages': ['Hindi', 'English'],
        'area_total': 3287263,
        'Population': '1,352,642,280',
        'GDP_nominal': ' $3.050 trillion',

    }

    table_class="infobox ib-country vcard"
    response=requests.get(wiki_link)
    soup=BeautifulSoup(response.text,'html.parser')
    table=soup.find('table', attrs={'class':table_class})
    rows = table.find_all('tr')
    images = soup.find('img', attrs={'class': "thumbborder"})

    dic['flag_link'] = "https:" +images['src']

    for tr in rows:

        th = tr.find_all('th', attrs={'class': "infobox-label"})
        td = tr.find_all('td', attrs={'class': 'infobox-data'})
        for h in th:

            for d in td:

                if h.text == "Largest city":
                    temp = d.find_all('a')
                    templ = []
                    for t in temp:
                        if t.get("title") != None:
                            templ.append(t.get('title'))
                    dic['largest_city'] = templ
                if h.text == "Capitaland largest city":

                    templ = []
                    temp = d.find_all('a')
                    for t in temp:
                        if t.get('title') != None:
                            templ.append(t.get('title'))
                    dic['capital'] = templ
                    dic['largest_city'] = templ
                    break
                elif h.text == "Capital":
                    temp = d.find_all('a')
                    templ = []

                    for t in temp:
                        if t.get('title') != None:
                            templ.append(t.get('title'))
                    dic['capital'] = templ
                    break

    dic2={}
    dic3={}
    for tr in rows:

        th = tr.find_all('th')
        td = tr.find_all('td')
        for j in th:
            # print(j.text)

            for k in td:
                # print(k.text)
                temp = k.find_all('a')
                temp_list = []
                for t in temp:
                    if t.get('title') != None:

                        temp_list.append(t.get('title'))

                dic3[j.text] = temp_list

            for s in td:
                dic2[j.text] = s.text

    try:
        dic['official_languages'] = dic3['Official languageand national language']
    except:
        dic['official_languages']=dic3['Official\xa0languages']

    try:
        dic['GDP']=dic2['•\xa0Total']
    except:
        dic['GDP']=dic['GDP']
    try:
        dic['Population']=dic2['•\xa02022 estimate']
    except:
        dic['Population']="---------NA---"

    return Response(dic)


#_*_ coding:utf-8 _*_
#!/usr/bin/env python

import pg

def operate_postgre_tbl_product():
    try:
        #pgdb_conn = pg.connect(dbname = 'tpc', host = '192.168.103.31', user = 'gpadmin', passwd = '')
        pgdb_conn = pg.connect("host=192.168.103.31 port=5432 dbname=tpc user=gpadmin")

    except Exception as e:
         print(e.args[0])
         return

    sql_desc = "select * from call_center limit 5;"
    for row in pgdb_conn.query(sql_desc).dictresult():
        print(row)

    pgdb_conn.close()

if __name__ == '__main__':
    operate_postgre_tbl_product()


from sklearn.model_selection import ParameterGrid
import subprocess
import os
import random

param_dict = {
        # Traing Setting:
        '--model': ['ABWIM_plus', 'HR_BiLSTM_plus'],      #'HR_BiLSTM_plus'/'ABWIM_plus'
        '--dataset': ['wq'],            # dataset_name
        '--framework': ['UHop'],    #'baseline'/'UHop'
        '--train': [' '],
        #'--train1test2': [True],
        '--earlystop_tolerance': [20],

        '--saved_dir':['saved_model'],
#        '--stop_when_err': [''],       # comment to use weighted loss
#        '--step_every_step' : [''],    # comment to step only once in each question
        '--dynamic' : ['recurrent'],         # none/flatten/recurrent
        '--reduce_method' : ['dense'],  # bilstm or dense, for abwim_plus
#        '--only_one_hop' : [''],       # uncomment for one-hop-only training/testing

        # HyperParameter:
        '--epoch_num': [200],
        '--emb_size': [300],
        '--hidden_size': [150, 100],
        '--dropout_rate': [0, 0.2, 0.4],
        '--learning_rate': [0.001, 0.0001, 0.00001],
        '--optimizer': ['rmsprop', 'adadelta'],
        '--neg_sample': [1024],
        '--l2_norm': [0.0],# 0.00000001, 0.000000001],
        '--margin': [0.1, 0.3, 0.5, 0.7, 1.0],
        #'--train_embedding': [True],
        '--hop_weight': [1, 0.8, 1, 1.25],
        '--task_weight': [1],
        '--acc_weight': [1, 0.25, 0.5, 1],
        #'--pretrained_bert': ['bert-base-uncased'],#'bert-base-uncased' or 'bert-base-multilingual-cased'
        '--q_representation': ['lstm']
        }

os.chdir('../src/')
process_str = 'python3 main.py'

################################ NEED TO CONFIGURE ABOVE ####################################################

possible_param_list = list(ParameterGrid(param_dict))
random.shuffle(possible_param_list)
#print(possible_param_list)
print(f'There will be {len(possible_param_list)} runs')
for i, param in enumerate(possible_param_list[:]):
    print(f'{i}/{len(possible_param_list)} run')
    # param is a dict
    run_str = process_str
    for arg, arg_val in param.items():
        run_str += f' {arg} {arg_val}'
    print(run_str)
    process = subprocess.run(run_str.split(), encoding='UTF-8')
    #print(process.stdout.split('\n')[-3])


import logging
import time

from databases import DatabaseHandler
from ec2 import Ec2Handler
from ecs import EcsHandler
from utils import is_substring_in_string

# The only required setup you need to do
looking_for = {
    'TagName': 'Environment',
    'TagValue': 'staging',
    'ECSClusterName': 'staging-main',
    'ECSServices': [
        'svc1-staging',
        'sv2-staging'
    ]
}

####################################
# DO NOT MODIFY BEYOND THIS POINT! #
####################################
# logger setup
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
# AWS resources handlers
rds = DatabaseHandler(looking_for)
ec2 = Ec2Handler(looking_for)
ecs = EcsHandler(looking_for)

def lambda_handler(event, context):
    """
        This is the Lambda handler function. The code will be executed when an event is generated.
    """
    operation_type = event['operation']

    if operation_type == 'start':
        logging.info('Starting everything...')
        rds.start_rds_docd_clusters()
        ec2.start_ec2_instances()
        ecs.start_ecs_services()
        return_msg = f'Successfully started the resources...'
    elif operation_type == 'stop':
        logging.info('Stopping everything...')
        ecs.stop_ecs_services()
        ec2.stop_ec2_instances()
        rds.stop_rds_docd_clusters()
        return_msg = f'Successfully stopped the resources...'
    else:
        return_msg = f'Operation {operation_type} cannot be handled!'

    return return_msg

def full_local_testing():
    """
        Function to mimic calls to the Lambda handler.
        The function will verify the content of each of the 3 possible responses.
        The 600s waiting time is due to the long time RDS & DocDB clusters take to stop.
        This invocation will affect ALL the resources.
    """
    responses = {}

    # Test a stop event
    event = {
        'operation': 'stop'
    }
    responses['stop'] = lambda_handler(event, None)

    logging.warning('Wait 10 mins for everything to stop')
    time.sleep(600)

    # Test a start event
    event = {
        'operation': 'start'
    }
    responses['start'] = lambda_handler(event, None)

    logging.warning('Wait 10 sec for everything to settle')
    time.sleep(10)

    # Test an invalid event
    event = {
        'operation': 'invalid'
    }
    responses['invalid'] = lambda_handler(event, None)

    # Verify the output content
    assert is_substring_in_string('stopped', responses['stop'])
    assert is_substring_in_string('started', responses['start'])
    assert is_substring_in_string('invalid', responses['invalid'])
    logging.info(f'The testing responses are {responses}')

if __name__ == '__main__':
    """
        This code will NOT run in the context of the Lambda function !!!
        It's for local testing purposes only !!!
    """

    # either run the selective testing
    # selective_local_testing(looking_for)

    # or the full lambda code
    full_local_testing()