from parser_utils import ParserUtils
from urls import Urls
import re
import urllib.parse
import json
import http.client

NEWS_FILE_NAME = 'news.csv'
COMMENTS_FILE_NAME = 'comments.csv'

# List for storing news objects
news_data = []

# List for storing comment objects
comment_data = []

def main():
    """Point of entry."""

    parse_news_list()
    ParserUtils.write_news_to_csv(NEWS_FILE_NAME, news_data)
    ParserUtils.write_comments_to_csv(COMMENTS_FILE_NAME, comment_data)

def parse_news_list():
    """Parses data(publish_date, title, link, comment count) from news list."""

    soup = ParserUtils.get_soup(Urls.NEWS_LIST, use_proxy=True)

    # Finding all divs with news
    news_div_list = soup.findAll('div', {'class': 'cat_news_item'})

    # Getting today's date from first div element
    date = news_div_list[0].text.strip()

    # Iterating each div excepting first
    for news_div in news_div_list[1:]:
        # News object formation
        news_object = {
            'publish_date': f"{date} {news_div.findChild('span', {'class': 'tahoma font12 date n3'}).text.strip()}",
            'title': news_div.findChild('a', {'class': 'tahoma font12'}).text.strip(),
            'link': f"{Urls.DOMAIN}{news_div.findChild('a', {'class': 'tahoma font12'}).get('href')}",
        }

        # Getting comment count if exists
        comment_count = news_div.findChild('span', {'class': 'comm_num'})
        news_object['comment_count'] = comment_count.text.strip() if comment_count else None

        # Storing to list
        news_data.append(news_object)
        print(f"{news_object.get('title')[:50]:<50} {news_object.get('link')}")
        try:
            parse_news_details(news_object.get('link'))
        except BaseException as e:
            print(e)

def parse_news_details(link):
    """Goes to news details and parses data(text)."""

    soup = ParserUtils.get_soup(link, use_proxy=True)

    # Removing style tags
    styles = soup.findAll('style')
    for style in styles:
        style.decompose()

    # Getting main div where text stores
    news_details_block = soup.find('div', {'class': 'fullnews white_block'})

    # if text in the table
    if not news_details_block:
        news_details_block = soup.find('div', {'class': 'full_story'})

    news_paragraphs = news_details_block.findAll('p')

    text = ''

    for paragraph in news_paragraphs:
        text += paragraph.text + ' '

    # Complementing news object
    recent_news_object = news_data[-1]
    recent_news_object['text'] = re.sub(' +', ' ', text.strip().replace('\n', ''))

    if recent_news_object.get('comment_count'):
        parse_comments(recent_news_object.get('link'), recent_news_object)

def parse_comments(link, news_object):
    """Parses comments."""

    # Making POST request to get comments json
    conn = http.client.HTTPSConnection("zcomments.net")

    page_title = urllib.parse.quote(news_object.get('title'))
    page_url = news_object.get('link')
    block_code = f"zakonnewsid{page_url.split('kz/')[1].split('-')[0]}"

    payload = f'page_title={page_title}&page_url={page_url}&block_code={block_code}'
    headers = {'content-type': "application/x-www-form-urlencoded; charset=UTF-8"}
    conn.request("POST", "/service/init/1", payload, headers)
    res = conn.getresponse()
    data = res.read().decode("utf-8")

    json_data = json.loads(data)
    serialize_comments_json(json_data)

def serialize_comments_json(json_data):
    """Retrieving comments list from json."""

    for item in json_data.get('comments').get('items'):
        parse_comment_details(item)

def parse_comment_details(comment):
    """Parsing comment details."""

    comment_object = {
        'user_nick': comment.get('user_nick'),
        'user_country': comment.get('user_country'),
        'message': comment.get('message'),
        'created_at': comment.get('created_at'),
        'likes': comment.get('likes'),
    }
    comment_data.append(comment_object)

    nested_comments = comment.get('children')

    # If this comment has children
    if nested_comments:
        for nested_comment in nested_comments:
            # Parsing children using recursion
            parse_comment_details(nested_comment)

if __name__ == '__main__':
    main()


from core.monitorizer import Monitorizer
from core.arguments import args
from datetime import timedelta
from datetime import datetime
from events.on import Events
from core import flags
from time import sleep
import os

scanners = [
	"subfinder",
	"sublist3r",
	"dnsrecon",
	"dnscan",
	"aiodnsbrute",
	#"amass",
	#"subbrute" - not recommended
]

monitorizer = Monitorizer()
if args.debug == False:
	monitorizer.clear()
events = Events()

if os.path.isfile(args.watch):
	_watch_list = 'set([t.strip() for t in open(args.watch,"r").readlines()])'
	monitorizer.log("Reading targets from file: %s" % args.watch)

else:
	monitorizer.banner()
	monitorizer.error("Couldn't read watch list")
	monitorizer.exit()

monitorizer.set_config(args.config)
events.set_config(args.config)
monitorizer.initialize()
monitorizer.self_check(scanners)
monitorizer.banner()
events.start()

while 1:
	for target in eval(_watch_list):
		flags.status = "running"
		flags.current_target = target

		if not target: continue

		report_name  = str(datetime.now().strftime("%Y%m%d_%s"))
		flags.report_name = report_name
		monitorizer.log("Created new report target=%s name=%s" %(target,report_name))

		current_scan = monitorizer.mutliscan(scanners,target)
		monitorizer.generate_report(target,current_scan,report_name)

		new_domains  = monitorizer.merge_scans(current_scan)
		old_domains  = monitorizer.merge_reports(target,exclude=[report_name])

		if len(old_domains) > 0:
			new_domains = new_domains - old_domains
		else:
			new_domains = []

		for domain in new_domains:
			foundby = []
			for tool,subs in current_scan.items():
				if not domain in subs:
					continue
				foundby.append(tool)
			events.discover(domain,foundby,report_name)

		if args.debug == False:
			monitorizer.clean_temp()

	flags.status = "idle"
	monitorizer.log("Sleeping %i hour(s)" % (flags.sleep_time))
	sleep( 60*60*flags.sleep_time)


import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

## @params: [JOB_NAME]
## @params: [target_bucket]
args = getResolvedOptions(sys.argv,
                          ['JOB_NAME',
                           'target_bucket'])
print ("JOB_NAME     : ", args['JOB_NAME'])
print ("target_bucket: ", args['target_bucket'])

# こちらを参考にさせていただきました
# https://future-architect.github.io/articles/20180828/
# https://dev.classmethod.jp/cloud/aws/20180528-aws-glue-etl-job-with-spark-sql/

sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

bucket = args['target_bucket']

df = glueContext.create_dynamic_frame.from_options(
    connection_type="s3",
    connection_options={
        "paths": ["s3://{0}/glue/csv2parquet/csv/test/".format(bucket)]},
    format="csv",
    format_options={"withHeader": True}  # 1行目をスキーマ名として認識True
    )

## @type: DataSink
## @args: [connection_type = "s3", connection_options = "paths": ["s3://{0}/csv2parquet/parquet/test/".format(bucket)], format = "parquet"]
## @return: datasink
## @inputs: [frame = df]
datasink = glueContext.write_dynamic_frame.from_options(
        frame=df,  # 出力するDynamicFrameを指定
        connection_type="s3",
        connection_options={
            "paths": ["s3://{0}/glue/csv2parquet/parquet/test".format(bucket)],
            "compression": "snappy"
        },
        format="parquet",
        transformation_ctx = "datasink"
    )

job.commit()

# 何故か動かない
# 下記エラーログ
#File "script_2019-06-23-11-40-00.py", line 53, in <module>
#transformation_ctx = "datasink"
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/PyGlue.zip/awsglue/dynamicframe.py", line 585, in from_options
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/PyGlue.zip/awsglue/context.py", line 194, in write_dynamic_frame_from_options
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/PyGlue.zip/awsglue/context.py", line 217, in write_from_options
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/PyGlue.zip/awsglue/data_sink.py", line 32, in write
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/PyGlue.zip/awsglue/data_sink.py", line 28, in writeFrame
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/py4j-0.10.4-src.zip/py4j/java_gateway.py", line 1133, in __call__
#File "/mnt/yarn/usercache/root/appcache/application_1561289672599_0002/container_1561289672599_0002_01_000001/pyspark.zip/pyspark/sql/utils.py", line 79, in deco
#pyspark.sql.utils.IllegalArgumentException: u'Expected exactly one path to be specified, but got: '
#End of LogType:stdout


#! /usr/bin/env python3
#
# Gets and retusrns valuse from a single line of temperature sensor data
#
# Author: Cody Sayer
# Version: 2.0
#
from datetime import datetime

class AbstractReading:
    """Maintains the details of each sensor reading
    """

    DATE_TIME_STR = "Date Time"
    SEQUENCE_NUM_STR = "Sequence Number"
    SENSOR_MODEL_STR = "Sensor Model"
    LOW_TEMP_STR = "Low temp"
    AVERAGE_TEMP_STR = "Average temp"
    HIGH_TEMP_STR = "High temp"
    STATUS_STR = "Status"
    STRPTIME = "%Y-%m-%d %H:%M:%S.%f"
    STRFTIME = "%Y/%m/%d %H:%M"
    DEGREE_SIGN = u'\N{DEGREE SIGN}'
    HIGH_READ_ERROR = "HIGH_READ"
    LOW_READ_ERROR = "LOW_READ"

    def __init__(self, date_time, sequence_num, sensor_model, low_temp, avg_temp, high_temp, status):
        """constructor for the SensorReading class

        Arguments:
            date_time {string} -- the date and time from the CSV
            sequence_num {int} -- the sequence number if the sensor reading
            sensor_model {string} -- the model information of the sensor
            low_temp {float} -- the low temp reading from the sensor
            avg_temp {float} -- the avg temp reading from the sensor
            high_temp {float} -- the high temp reading from the sensor
            status {string} -- the status of the sensor during the reading
        """

        AbstractReading._validate_input(AbstractReading.DATE_TIME_STR, date_time)
        AbstractReading._validate_string(AbstractReading.DATE_TIME_STR, date_time)
        self._date_time = date_time

        AbstractReading._validate_input(AbstractReading.SEQUENCE_NUM_STR, sequence_num)
        AbstractReading._validate_int(AbstractReading.SEQUENCE_NUM_STR, sequence_num)
        self._sequence_num = sequence_num

        AbstractReading._validate_input(AbstractReading.SENSOR_MODEL_STR, sensor_model)
        AbstractReading._validate_string(AbstractReading.SENSOR_MODEL_STR, sensor_model)
        self._sensor_model = sensor_model

        AbstractReading._validate_input(AbstractReading.LOW_TEMP_STR, low_temp)
        AbstractReading._validate_float(AbstractReading.LOW_TEMP_STR, low_temp)
        self._low_read = low_temp

        AbstractReading._validate_input(AbstractReading.AVERAGE_TEMP_STR, avg_temp)
        AbstractReading._validate_float(AbstractReading.AVERAGE_TEMP_STR, avg_temp)
        self._average_read = avg_temp

        AbstractReading._validate_input(AbstractReading.HIGH_TEMP_STR, high_temp)
        AbstractReading._validate_float(AbstractReading.HIGH_TEMP_STR, high_temp)
        self._high_read = high_temp

        AbstractReading._validate_input(AbstractReading.STATUS_STR, status)
        AbstractReading._validate_string(AbstractReading.STATUS_STR, status)
        self._status = status

    def format_datetime_string(self):
        """Retrieves the datetime and formats into Y/M/D H:M

        Returns:
            datetime -- the datetime from a sensor reading
        """

        return datetime.strptime(self._date_time, "%Y-%m-%d %H:%M:%S.%f").strftime("%Y/%m/%d %H:%M")

    def format_output_strings(self):
        """Returns a message if error is detected in log

        Returns:
            string -- error message
        """

        if self._status == AbstractReading.HIGH_READ_ERROR:
            status_display = "High reading Error (100%cC)" % AbstractReading.DEGREE_SIGN
            reading_display_datetime = self.format_datetime_string()
            reading_seq_num = self.get_sequence_num()
            return "%s at %s, Sequence: %d" % (status_display, reading_display_datetime, reading_seq_num)

        elif self._status == AbstractReading.LOW_READ_ERROR:
            status_display = "Low reading Error (-50%cC)" % AbstractReading.DEGREE_SIGN
            reading_display_datetime = self.format_datetime_string()
            reading_seq_num = self.get_sequence_num()
            return "%s at %s, Sequence: %d" % (status_display, reading_display_datetime, reading_seq_num)

    def get_sequence_num(self):
        """retrieves and returns the sequence number from the reading line

        Returns:
            int -- sequence number
        """

        return self._sequence_num

    def get_sensor_model(self):
        """retrieves and returns the sensor model from the reading line

        Returns:
            string -- sensor model
        """

        return self._sensor_model

    def get_low_reading(self):
        """retrieves and returns the lowest reading from the reading line

        Returns:
            float -- low reading
        """

        return self._low_read

    def get_average_reading(self):
        """retrieves and returns the average reading from the reading line

        Returns:
            float -- average reading
        """

        return self._average_read

    def get_high_reading(self):
        """retrieves and returns the highest reading from the reading line

        Returns:
            float -- high reading
        """

        return self._high_read

    def get_reading_range(self):
        """calculates and returns the reading range from the reading line

        Returns:
            float -- reading range
        """

        return self._high_read - self._low_read

    def get_status(self):
        """retrieves and returns the status message of a reading line

        Returns:
            string -- sensor status
        """

        return self._status

    def is_error(self):
        """checks if there is an error

        Raises:
            NotImplementedError -- is_error test not implemented
        """

        raise NotImplementedError()

    def get_error_msg(self):
        """retrieves the error message if present

        Raises:
            NotImplementedError -- get_error_message not implemented
        """

        raise NotImplementedError()

    @staticmethod
    def _validate_string(display_name, input_value):
        """validates a string

        Arguments:
            display_name {value name} -- the actual name assigned to the value
            input_value {input} -- the input to be validated

        Raises:
            ValueError -- if it is not a string
        """

        if input_value != str(input_value):
            raise ValueError(display_name + " must be a string type")

    @staticmethod
    def _validate_int(display_name, input_value):
        """validates the input value to make sure it is of int type

        Arguments:
            display_name {value name} -- the given name for a value
            input_value {input} -- the input to be validated

        Raises:
            ValueError -- if it is not an integer
        """

        if input_value != int(input_value):
            raise ValueError(display_name + " must be a integer type")

    @staticmethod
    def _validate_float(display_name, input_value):
        """validates the input value to make sure it is of float type

        Arguments:
            display_name {value name} -- the given name for a value
            input_value {input} -- the input to be validated

        Raises:
            ValueError -- if it is not a float
        """

        if input_value != float(input_value):
            raise ValueError(display_name, " must be a float type")

    @staticmethod
    def _validate_input(display_name, input_value):
        """validates the input to make sure it is not None or empty

        Arguments:
            display_name {value name} -- the given name for a value
            input_value {input} -- the input to be validated

        Raises:
            ValueError -- if input is undefined
            ValueError -- if input is empty
        """

        if input_value is None:
            raise ValueError(display_name + " cannot be undefined.")

        if input_value == "":
            raise ValueError(display_name + " cannot be empty.")


# Copyright 2012 Rackspace

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import mock

from twisted.trial import unittest
from twisted.python import filepath

import bot

class LogBotTestCase(unittest.TestCase):
    """
    Tests for L{bot.LogBot}
    """

    def setUp(self):
        for logger in ('PyLogger', 'BufferedSearchLogger',
                       'BufferedMultiChannelFileLogger'):
            self.patch(bot.loggers, logger, mock.MagicMock(['log']))

    def _make_mock_logbot(self, channels, log_path=None):
        """
        Initialize a fake logbot (to use as self to functions) and set the
        factory to a fake factory.
        """
        if not log_path:
            log_path = self.mktemp()
            fp = filepath.FilePath(log_path)
            fp.createDirectory()
        self.log_path = log_path

        fake_factory = mock.MagicMock(channels=channels, log_path=log_path)

        fp = filepath.FilePath(self.mktemp())
        fp.createDirectory()

        self.fake_logbot = mock.MagicMock(bot.LogBot,
                                          _user_left_FP=fp,
                                          factory=fake_factory)

    def test_join_all_channels_on_signon(self):
        """
        On signing on to the server, the bot should join all of the channels in
        the channel list
        """
        channels = ['#channel1', '#channel2']
        self._make_mock_logbot(channels)

        # mock calling a LogBot().signedOn(), but with a fake logbot as self
        bot.LogBot.signedOn.im_func(self.fake_logbot)

        expected_calls = [mock.call(name) for name in channels]
        self.assertEqual(expected_calls, self.fake_logbot.join.mock_calls)

    def _run_connection_made(self):
        """
        Fake calls connection made, for the tests to see if a logger is created
        """
        self.channels = ['#channel1', '#channel2']
        self._make_mock_logbot(self.channels)
        # mock calling LogBot().connectionMade()
        bot.LogBot.connectionMade.im_func(self.fake_logbot)

    def test_connection_made_PyLogger(self):
        """
        When a connection is made, a PyLogger should be created
        """
        self._run_connection_made()
        bot.loggers.PyLogger.assert_called_once_with()

    def test_connection_made_BufferedSearchLogger(self):
        """
        When a connection is made, a BufferedSearchLogger should be created
        """
        self._run_connection_made()
        bot.loggers.BufferedSearchLogger.assert_called_once_with()

    def test_connection_made_BufferedMultiChannelFileLogger(self):
        """
        When a connection is made, a BufferedMultiChannelFileLogger should be
        created
        """
        self._run_connection_made()
        bot.loggers.BufferedMultiChannelFileLogger.assert_called_once_with(
            self.log_path, self.channels)

    def test_write_log(self):
        """
        LogBot().writeLog should log the same message to all available loggers
        """
        self.channels = ['#channel1', '#channel2']
        self._make_mock_logbot(self.channels)
        self.fake_logbot.loggers = [
            bot.loggers.PyLogger(),
            bot.loggers.BufferedSearchLogger(),
            bot.loggers.BufferedMultiChannelFileLogger()
        ]
        # mock calling LogBot().writeLog()
        bot.LogBot.writeLog.im_func(
            self.fake_logbot, "message", "user", "channel1")

        calls = None
        for logger in self.fake_logbot.loggers:
            if not calls:
                calls = logger.log.mock_calls
                self.assertEqual(1, len(calls))
            else:
                self.assertEqual(calls, logger.mock_calls)

    def test_user_join_is_logged(self):
        """
        Message should be logged when a user joins the channel
        """
        self._make_mock_logbot(['#channel1'])
        # mock calling LogBot.userJoined()
        bot.LogBot.userJoined.im_func(self.fake_logbot, 'me', '#channel1')
        self.assertEqual(1, self.fake_logbot.writeLog.call_count)
        # mock.call_args[0] = non-keyword arguments ([1] is keyword arguments)
        self.assertEqual((None, '#channel1'),
                         self.fake_logbot.writeLog.call_args[0][1:])

    def test_user_left_is_logged(self):
        """
        Message should be logged when a user leaves the channel
        """
        self._make_mock_logbot(['#channel1'])
        # mock calling LogBot.userLeft()
        bot.LogBot.userLeft.im_func(self.fake_logbot, 'me', '#channel1')
        self.assertEqual(1, self.fake_logbot.writeLog.call_count)
        # mock.call_args[0] = non-keyword arguments ([1] is keyword arguments)
        self.assertEqual((None, '#channel1'),
                         self.fake_logbot.writeLog.call_args[0][1:])

    def test_user_left_channel_time_record(self):
        """
        When the user leaves the channel, the time at which they left, as well
        as their username and the channel name is recorded in such a way that
        this time can be retrieved.
        """
        self._make_mock_logbot(['#channel1', '#channel2'])
        # mock calling LogBot.userLeft()
        bot.LogBot.userLeft.im_func(self.fake_logbot, 'me', '#channel1')
        bot.LogBot.userLeft.im_func(self.fake_logbot, 'me', '#channel2')
        # there should be some way to get the last exit times
        results = bot.LogBot._get_user_last_exit_time(self.fake_logbot, 'me')

        # assert that the correct channels (that the user exited) were recorded
        keys = results.keys()
        keys.sort()
        self.assertEqual(['#channel1', '#channel2'], keys)


from mpl_toolkits import mplot3d
import matplotlib.pyplot as plt
import numpy as np

if __name__ == '__main__':
    fig = plt.figure()
    ax = plt.axes(projection='3d')

    zline = np.linspace(0, 15, 1000)
    xline = np.sin(zline)
    yline = np.cos(zline)

    ax.plot3D(xline, yline, zline)

    plt.show()


def election_donations_analysis(input, output1, output2):
    from pyspark.sql import SparkSession
    import pyspark.sql.functions as F
    spark = SparkSession.builder.getOrCreate()

    reader = spark.read
    reader.option("header",True).option("inferSchema",True)
    df = reader.csv(input)

    cand_info = ['cand_id', 'cand_nm']

    grouped_df = df.groupBy(cand_info)\
        .agg(
            F.count(F.when(F.col('contb_receipt_amt') >= '0', 1)).alias("NumDon"),
        F.sum('contb_receipt_amt').alias("Total_cntrb"),
        F.countDistinct(F.when(F.col('contb_receipt_amt') >= '0', F.array('contbr_nm', 'contbr_zip', 'contbr_employer', 'contbr_occupation'))).alias('UnqCtb'),
        F.mean("contb_receipt_amt").alias("Mean"),
        F.stddev("contb_receipt_amt").alias("SD"),
        F.count(F.when((F.col('contb_receipt_amt') < '50') & (F.col('contb_receipt_amt') >= '0'), 1)).alias("smCtb")
        ).withColumn("%", (F.col("smCtb") /  F.col("NumDon"))*100)

    grouped_df.sort(F.desc('cand_id')).show()
    grouped_df.write.format('csv').mode('overwrite').save(output1)

    required_cand_ids = ["P80001571", "P80000722"]
    df_6 = df.filter(df.cand_id.isin(required_cand_ids)).filter("contb_receipt_amt >= '0'").select('cand_nm','contb_receipt_amt')

    df_6.sort(F.desc('contb_receipt_amt')).show()
    df_6.coalesce(1).write.format('csv').mode('overwrite').save(output2)

def files_from_args():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', metavar= "input")
    parser.add_argument('-o1', '--output1', metavar= "output1")
    parser.add_argument('-o2', '--output2', metavar= "output2")
    args = parser.parse_args()
    return (args.input, args.output1, args.output2)

if __name__ == "__main__":
    inputfile, outputfile1, outputfile2 = files_from_args()
    election_donations_analysis(inputfile, outputfile1, outputfile2)


'''
TODO: 1. Improve plot functions. 2. Create new verify functions.
'''
import os, copy, json, glob, pickle
import numpy as np

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle

class STP:
    def __init__(self, filter=None, keep=None):
        # get STP data from .json files
        self.fns = sorted(glob.glob('microcircuit/stp_new/stp_*.json'))
        # load to dict
        self.stps_default = {}
        for fn in self.fns:
            with open(fn, 'r') as h:
                self.stps_default[os.path.basename(fn)] = json.load(h)
        # duplicate, one for use, one stays default
        self.stps = copy.deepcopy(self.stps_default)
        # filter or keep the one for use
        if isinstance(filter, dict):
            self.filter(filter)
        elif isinstance(keep, dict):
            self.keep(keep)
        # prepare plotting parameters
        self.plot_params()

    def default(self):
        self.stps = copy.deepcopy(self.stps_default)

    def keep(self, keep):
        self.stps = {}
        for set_key, set_value in self.stps_default.items():
            self.stps[set_key] = {}
            for pre, targets in set_value.items():
                for kpre, ktargets in keep.items():
                    # match pre-synaptic
                    if kpre in pre:
                        self.stps[set_key][pre] = {}
                        # copy all
                        if ktargets == 'all':
                            self.stps[set_key][pre] = \
                            copy.deepcopy(self.stps_default[set_key][pre])
                        # copy selected
                        elif isinstance(ktargets, list):
                            for target in targets.keys():
                                for ktarget in ktargets:
                                    # match post-synaptic
                                    if ktarget in target:
                                        self.stps[set_key][pre][target] = \
                                        copy.deepcopy(self.stps_default[set_key][pre][target])

    def filter(self, filter):
        self.stps = copy.deepcopy(self.stps_default)
        for set_key, set_value in self.stps_default.items():
            for pre, targets in set_value.items():
                for fpre, ftargets in filter.items():
                    # match pre-synaptic
                    if fpre in pre:
                        # delete all
                        if ftargets == 'all':
                            del self.stps[set_key][pre]
                        # delete selected
                        elif isinstance(ftargets, list):
                            for target in targets.keys():
                                for ftarget in ftargets:
                                    # match post-synaptic
                                    if ftarget in target:
                                        del self.stps[set_key][pre][target]

    def func1(self, x):
        return 1000.*x

    def func2(self, x):
        return x/1000.

    def plot_params(self):
        self.colors = ['#009988', '#EE7733', '#AA4499',]
        # self.colors = ['#66CCEE', '#CCBB44', '#AA3377',]
        self.legends = ['U', 'F', 'D']
        # self.legends = ['Parameter for release probability (U)', 'Facilitation time constant (F)', 'Depression time constant (D)']
        self.EIset = ['E', 'PV', 'SOM', 'VIP', 'Eth']
        self.EEset = ['L23_E', 'L4_E', 'L5_E', 'L6_E']

    def plot(self, fontsize=40, figsize=(16, 16), rotation=30, w=.2, ncol=3,
        anchor=(0.5, 1.3)):
        matplotlib.rcParams['font.size'] = fontsize
        # loop each STP set
        for idx, (stp_fn, stp) in enumerate(self.stps.items()):
            # print('stp_fn={}'.format(stp_fn))
            EIshift = False
            # E to I
            names_ei, Us_ei, Fs_ei, Ds_ei = [], [], [], []
            for i, pre in enumerate(self.EIset):
                for j, post in enumerate(self.EIset):
                    name, U, F, D = pre + r'$\rightarrow$' + post, 0., 0., 0.
                    try:
                        tmp = stp[pre][post]
                        U, F, D = tmp['U'], tmp['tau_fac']/1000., tmp['tau_rec']/1000.
                    except KeyError:
                        # get rid of E=>E if layer-specific
                        if pre == 'E' and post == 'E':
                            EIshift = True
                        continue
                    names_ei.append(name.replace('_', '').replace('23', '2/3'))
                    Us_ei.append(U)
                    Fs_ei.append(F)
                    Ds_ei.append(D)
            print('names_ei={}'.format(names_ei))

            # E to E
            names_ee, Us_ee, Fs_ee, Ds_ee = [], [], [], []
            for i, pre in enumerate(self.EEset):
                for j, post in enumerate(self.EEset):
                    name, U, F, D = pre + r'$\rightarrow$' + post, 0., 0., 0.
                    try:
                        tmp = stp[pre][post]
                        U, F, D = tmp['U'], tmp['tau_fac']/1000., tmp['tau_rec']/1000.
                    except KeyError:
                        continue
                    names_ee.append(name.replace('_', '').replace('23', '2/3'))
                    Us_ee.append(U)
                    Fs_ee.append(F)
                    Ds_ee.append(D)

            # combine
            if EIshift is True:
                xs = [np.arange(len(names_ei)) + 1, np.arange(len(names_ee))]
            else:
                xs = [np.arange(len(names_ei)), np.arange(len(names_ee))]
            names = [names_ei, names_ee]
            Us, Fs, Ds = [Us_ei, Us_ee], [Fs_ei, Fs_ee], [Ds_ei, Ds_ee]

            # plot
            fig, ax = plt.subplots(2, 1, figsize=figsize, sharey=True)
            for i in range(2):
                # bar
                for j, (dx, params) in enumerate(zip([-w, 0., w], [Us, Fs, Ds])):
                    ax[i].bar(xs[i]+dx, params[i], w, color=self.colors[j])

                # vertical lines
                # for x in [3.5, 7.5, 11.5]:
                #     ax[i].axvline(x, 0, 1, color='k')

                # first y axis
                ax[i].set_ylabel('probability', fontsize=0.9*fontsize)

                # second y axis
                secax = ax[i].secondary_yaxis(
                    'right', functions=(self.func1, self.func2))
                secax.set_ylabel('time (ms)', fontsize=0.9*fontsize)

                # x axis
                ax[i].set_xticks(xs[i])
                ax[i].set_xticklabels(
                    names[i], rotation=rotation, fontsize=0.6*fontsize)
                plt.setp(
                    ax[i].get_xticklabels(), rotation=rotation, ha="right",
                    rotation_mode="anchor")

                # y limits
                ax[i].set_ylim(-0.05, 1.05)

            # legend
            for i in range(3):
                ax[0].scatter(
                    [], [], s=400, marker='s', label=self.legends[i],
                    color=self.colors[i])
            ax[0].legend(
                loc='upper center', ncol=ncol, bbox_to_anchor=anchor,
                frameon=False, fontsize=fontsize)
            # ax[0].set_title(r'E$\rightarrow$Inh, Inh$\rightarrow$E, Inh$\rightarrow$Inh', y=1.15)
            # ax[1].set_title(r'E$\rightarrow$E', y=1.15)

            fig.tight_layout()
            plt.savefig('microcircuit/stp_new/{}.png'.format(stp_fn.replace('.json', '')), bbox_inches='tight')
            plt.close()

if __name__ == '__main__':
    stp = STP()
    stp.plot()


import os, time, re, pickle, signal
from selenium import webdriver
import pytesseract
from PIL import Image
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
import scrapy

# Selenium Part
# Setting driver
driver = webdriver.Chrome('/Users/mya/Downloads/chromedriver')
driver.maximize_window()
screenshot_path =  '/Users/mya/DBDCrawler/scrapy_app/scrapy_app/spiders/temp/screenshot.png'
# The start page (login page)
login_page_url = 'https://datawarehouse.dbd.go.th/login'
# Store cookie
cookie_path = '/Users/mya/DBDCrawler/scrapy_app/scrapy_app/spiders/temp/cookie.json'

# Access login page
driver.get(login_page_url)
print(driver.title)

# get and verify captcha, then access 'https://datawarehouse.dbd.go.th/index' page
def getAndStoreCookieAfterLogin():
    for i in range(10):
        time.sleep(10)
        if 'Home' in driver.title:
            # load cookie
            cookies = driver.get_cookies()
            # find token 'JSESSIONID' and store it into cookie_path
            for i in cookies:
                # print(i)
                if i['name'] == 'JSESSIONID':

                    with open(cookie_path, 'wb') as f:
                        pickle.dump(cookies, f)
                    print(i['value'])
                    break
                else:
                    print('no JSESSIONID in this page!')
            break
        else:
            driver.refresh()

# check whether access 'https://datawarehouse.dbd.go.th/index' page successfully
getAndStoreCookieAfterLogin()

# Change language
driver.find_element_by_xpath('//*[@id="lang"]').click()

# Search 'Opencloud' and access 'https://datawarehouse.dbd.go.th/company/profile/5/0105554123553' page
# English Version
# driver.find_element_by_xpath('//*[@id="textStr"]').send_keys('Opencloud')
# driver.find_element_by_xpath('//*[@id="form"]/div/button').click()
# driver.find_element_by_xpath('//*[@id="fixTable"]/tbody/tr').click()


from datetime import datetime
from os.path import dirname, join

# import pytest
from city_scrapers_core.constants import NOT_CLASSIFIED
from city_scrapers_core.utils import file_response
from freezegun import freeze_time

from city_scrapers.spiders.pitt_city_planning import PittCityPlanningSpider

test_response = file_response(
    join(dirname(__file__), "files", "pitt_city_planning.html"),
    url="http://pittsburghpa.gov/dcp/notices",
)
spider = PittCityPlanningSpider()

freezer = freeze_time("2019-08-14")
freezer.start()

parsed_items = [item for item in spider.parse(test_response)]

freezer.stop()

# def test_tests():
# assert False
"""
Uncomment below
"""

def test_title():
    assert parsed_items[0][
        "title"] == "CMU IMP Amendment for Scaife Hall and Zone Change Petition (parcel53-B-100)"

def test_description():
    assert parsed_items[0]["description"] == ""

def test_start():
    assert parsed_items[0]["start"] == datetime(2019, 7, 30, 14, 0)

# def test_end():
#     assert parsed_items[0]["end"] == datetime(2019, 1, 1, 0, 0)

def test_time_notes():
    assert parsed_items[0]["time_notes"] == ""

# def test_id():
#     assert parsed_items[0]["id"] == "EXPECTED ID"

# def test_status():
#    assert parsed_items[0]["status"] == "PASSED"

def test_location():
    assert parsed_items[0]["location"] == {
        "name": '',
        "address": '200 Ross Street, 1stFloor Conference Room'
    }

# def test_source():
#     assert parsed_items[0]["source"] == "EXPECTED URL"

def test_links():
    url_begin = 'https://apps.pittsburghpa.gov/redtail/'
    assert parsed_items[0]["links"] == [{
        'href': url_begin + 'images/6132_CMU_IMP_amendment_Application.pdf',
        'title': 'Amendment'
    }, {
        'href':
            url_begin +
            'images/6134_Institutional_Master_Plan_with_draft_amendments_highlighted.pdf',
        'title': 'amendments highlighted'
    }, {
        'href': url_begin + 'images/6133_CMU_Zone_Change_Petition_Application.pdf',
        'title': 'Petition'
    }, {
        'href': 'https://apps.pittsburghpa.gov/redtail/images/6135_ZoningCMUMapChange3.pdf',
        'title': 'Map'
    }]

def test_classification():
    assert parsed_items[0]["classification"] == NOT_CLASSIFIED

# @pytest.mark.parametrize("item", parsed_items)
# def test_all_day(item):
#     assert item["all_day"] is False


# -*- coding: utf-8 -*-
"""
__author__ = z0305
__date__ = 08/16/2018
__purpose__ =
__site__ = https://mp.weixin.qq.com/s/dkaV4VZ1QBzcsi9gXvCRdw (Python告诉你上海有哪些高性价比的西餐厅)
           https://github.com/pythonml/restaurant
"""
import re
import json
import pymongo
from lxml import etree
import requests
import pandas as pd
import matplotlib.pyplot as plt
from config import STRING_NUM

DB = "restaurant"
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.8",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "www.dianping.com",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
    "Upgrade-Insecure-Requests": "1",
    "Cookie": "cy=1; cye=shanghai; _lxsdk_cuid=1653dcc99a6c8-04ff4f92706bc4-1c2f1702-2f4f0c-1653dcc99a6bb; _lxsdk=1653dcc99a6c8-04ff4f92706bc4-1c2f1702-2f4f0c-1653dcc99a6bb; _hc.v=c668bd43-a103-53a2-3ccb-2d01ad06f352.1534340144; _lxsdk_s=1653dcc99ae-1-d99-0e5%7C%7C48",
}

def get_districts():
    url = "http://www.dianping.com/shanghai/ch10/g116"
    r = requests.get(url, headers=headers, verify=False)
    content = r.content.decode("utf-8")
    print("====")
    print(content)
    root = etree.HTML(content)
    distr_nodes = root.xpath('.//div[@id="region-nav"]')
    if len(distr_nodes) == 0:
        raise Exception("no districts")
    distr_node = distr_nodes[0]
    result = []
    for node in distr_node:
        name = node.attrib["data-click-title"]
        url = node.attrib["href"]
        result.append({"name": name, "url": url})
    return result

def get_sub_districts():
    districts = get_districts()
    client = pymongo.MongoClient()
    db = client[DB]
    for item in districts:
        distr_name = item["name"]
        distr_url = item["url"]
        r = requests.get(distr_url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        sub_nodes = root.xpath('.//div[@id="region-nav-sub"]/a[@data-cat-id]')
        for node in sub_nodes:
            url = node.attrib["href"]
            name = node.xpath('./span')[0].text
            db.sub.insert_one({"district": distr_name, "sub": name, "url": url})

def get_restaurants_by_district(entry_url, sub_id):
    i = 1
    client = pymongo.MongoClient()
    db = client[DB]
    while True:
        if i > 50:
            print("{} has more than 50 pages".format(url))
            break

        url = "{}p{}".format(entry_url, i)
        print(url)
        r = requests.get(url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        rest_nodes = root.xpath('.//div[@id="shop-all-list"]/ul/li')
        if len(rest_nodes) == 0:
            break
        for node in rest_nodes:
            a_nodes = node.xpath('.//div[@class="tit"]/a')
            if len(a_nodes) == 0:
                continue
            url = a_nodes[0].attrib["href"]
            name = a_nodes[0].attrib["title"]
            db.restaurant_url.insert_one({"name": name, "url": url, "sub_id": sub_id, "is_done": False})
        i += 1

def get_all_restaurants():
    client = pymongo.MongoClient()
    db = client[DB]
    sub_rows = db.sub.find()
    for row in sub_rows:
        district = row["district"]
        url = row["url"]
        sub_id = row["_id"]
        print(district)
        get_restaurants_by_district(url, sub_id)

def get_price(node):
    text = node.text
    matched = re.search(r"人均\D+(\d+)", text)
    price = 0
    if matched:
        price = int(matched.group(1))
    digit_nodes = node.xpath('./span')
    for digit_node in digit_nodes:
        span_class = digit_node.attrib["class"]
        num = STRING_NUM[span_class]
        price = price * 10 + num
        if digit_node.tail:
            matched = re.search(r"(\d+)", digit_node.tail)
            if matched:
                power = len(matched.group(1))
                price = price * 10**power + int(matched.group(1))
    return price

def get_score(node):
    digit_nodes = node.xpath('./span')
    score = 0
    text = node.text
    matched = re.search(r"(\d+)", text)
    if matched:
        score = matched.group(1)

    for digit_node in digit_nodes:
        span_class = digit_node.attrib["class"]
        num = STRING_NUM[span_class]
        score = score * 10 + num
        tail = digit_node.tail
        if tail is not None:
            matched = re.search(r"(\d+)", tail)
            if matched:
                score = score * 10 + int(matched.group(1))
    score = score / 10
    return score

def get_restaurant_details():
    client = pymongo.MongoClient()
    db = client[DB]
    rest_rows = db.restaurant_url.find({"is_done": False})
    done_urls = []
    for row in rest_rows:
        url = row["url"]
        #url = "http://www.dianping.com/shop/5735627"
        #url = "http://www.dianping.com/shop/69208935"
        #url = "http://www.dianping.com/shop/66183142"
        #url = "http://www.dianping.com/shop/13932333"
        if url in done_urls:
            continue
        sub_id = row["sub_id"]
        print(url)
        r = requests.get(url, headers=headers, verify=False)
        content = r.content.decode("utf-8")
        root = etree.HTML(content)
        info_nodes = root.xpath('.//div[@id="basic-info"]')
        if len(info_nodes) == 0:
            print("no info for {}".format(url))
            continue
        name = info_nodes[0].xpath('.//h1[@class="shop-name"]')[0].text
        review_count = 0
        star = 0
        price = 0
        taste_score = 0
        env_score = 0
        service_score = 0
        star_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[contains(@class, "mid-rank-stars")]')
        if len(star_nodes) > 0:
            star_class = star_nodes[0].attrib["class"]
            matched = re.search(r"mid-str(\d+)", star_class)
            if matched:
                star = int(matched.group(1))
        review_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="reviewCount"]')
        if len(review_nodes) > 0:
            review_info = review_nodes[0].text
            matched = re.search(r"(\d+)条评论", review_info)
            if matched:
                review_count = int(matched.group(1))
        price_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="avgPriceTitle"]')
        if len(price_nodes) > 0:
            price = get_price(price_nodes[0])
            print(price)
        comment_nodes = info_nodes[0].xpath('.//div[@class="brief-info"]/span[@id="comment_score"]')
        if len(comment_nodes) > 0:
            score_nodes = comment_nodes[0].xpath('./span')
            for score_node in score_nodes:
                if re.search(r"口味", score_node.text):
                    taste_score = get_score(score_node)
                if re.search(r"环境", score_node.text):
                    env_score = get_score(score_node)
                if re.search(r"服务", score_node.text):
                    service_score = get_score(score_node)
        db.restaurant.insert_one({"name": name,
            "star": star,
            "price": price,
            "review_count": review_count,
            "taste_score": taste_score,
            "env_score": env_score,
            "service_score": service_score,
            "url": url,
            "sub_id": sub_id
        })
        db.restaurant_url.update_many({"url": url}, {"$set": {"is_done": True}})
        done_urls.append(url)

def unique():
    client = pymongo.MongoClient()
    db = client[DB]
    urls = db.restaurant.distinct("url")
    for url in urls:
        row_count = db.restaurant.find({"url": url}).count()
        if row_count == 1:
            continue
        rows = db.restaurant.find({"url": url})
        i = 0
        for row in rows:
            if i == 0:
                i += 1
                continue
            db.restaurant.delete_one({"_id": row["_id"]})
            i += 1

def update_index():
    client = pymongo.MongoClient()
    db = client[DB]
    restaurants = db.restaurant.find({"review_count": {"$gt": 100}})
    for item in restaurants:
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        price = item["price"]
        if taste_score == 0 or env_score == 0 or service_score == 0:
            continue
        index = 3/(1/taste_score+1/env_score+1/service_score) / price * 1000
        avg_score = 3/(1/taste_score+1/env_score+1/service_score)
        db.restaurant.update({"_id": item["_id"]}, {"$set": {"index": index, "avg_score": avg_score}})

def stats():

    client = pymongo.MongoClient("172.24.176.73")
    db = client[DB]

    print("============ most expensive ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 0}}},
        {"$sort": {"price": 1}},
        {"$limit": 10},
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        print("{},{}".format(name, price))

    print("============ highest index ===================")
    restaurants = db.restaurant.aggregate([
        {"$sort": {"index": -1}},
        {"$limit": 10}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        print("{},{},{},{},{},{}".format(name, price, taste_score, env_score, service_score, index))

    print("============ highest index > 100 ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 100}}},
        {"$sort": {"index": -1}},
        {"$limit": 10}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        print(name, price, review_count, taste_score, env_score, service_score, index)

    print("============ highest avg_score ===================")
    restaurants = db.restaurant.aggregate([
        {"$match": {"price": {"$gt": 200}}},
        {"$match": {"price": {"$lt": 300}}},
        {"$sort": {"index": -1}},
        {"$limit": 50}
    ])
    for item in restaurants:
        name = item["name"]
        review_count = item["review_count"]
        star = item["star"]
        price = item["price"]
        index = item["index"]
        taste_score = item["taste_score"]
        env_score = item["env_score"]
        service_score = item["service_score"]
        avg_score = item["avg_score"]
        print("{},{},{}".format(name, price, avg_score))

    #restaurants = db.restaurant.find({})
    #df = pd.DataFrame(columns=["price", "avg_score"])
    #for item in restaurants:
    #    df.loc[len(df)] = [item.get("price"), item.get("avg_score")]
    #print(df)
    #fig, ax = plt.subplots()
    #df.avg_score.hist(ax=ax, bins=20)
    #plt.show()

if __name__ == "__main__":
    stats()


import os
from copy import deepcopy

from flask import Flask, make_response
from flask import session
from flask_cors import CORS
from flask_session import Session

from SodukoUtils import init_board_options, find_next_move
from data.sudopy import Sudoku

app = Flask(__name__)
SESSION_TYPE = 'filesystem'
app.secret_key = b'abjdslgjl'
app.config.from_object(__name__)
CORS(app, supports_credentials=True)
Session(app)
app.config.update(
    SESSION_COOKIE_SECURE=True,
    SESSION_COOKIE_HTTPONLY=True,
    SESSION_COOKIE_SAMESITE='None',
)

'''
why I'm doing so :
1. I want to succeed in being a better SW engineer
2. I want to improve my web development skills
3. I want to be better at sudoku
4. show myself that I can handle this code
5. show myself that I can learn how to develop in web
'''

''' next steps:
FE
1. provide a focus and enter a value and pencil marks
2. provide reason solution
'''

@app.route("/greeting")
def greeting():
    return {"greeting": "Hello from Flask!"}

@app.route('/')
@app.route('/index')
def index():
    return 'hi'

@app.route('/get_board/<int:board_id>', methods=['GET'])
def handle_connect(board_id):
    b = Sudoku(board_id).board
    session['board'] = b
    session['options'] = [[] * 9 for _ in range(9)]
    return {'board': b, 'id': 0}

def jsonify_options(options):
    res = deepcopy(options)
    for i in range(9):
        for j in range(9):
            res[i][j] = list(res[i][j])

    return res

@app.route('/get_pencil_marks', methods=['GET'])
def get_options():
    print('got load options')
    options = init_board_options(session['board'])
    session['options'] = options
    options_with_list_instead_of_sets = [[list(options[i][j]) for j in range(9)]for i in range(9)]
    return {'options': options_with_list_instead_of_sets}

@app.route('/get_next_step', methods=['GET'])
def get_next_step():
    print('got load next step!!!!!')
    board = session['board']
    options = session['options']
    options = [[set(options[i][j]) for j in range(9)]for i in range(9)]
    res = find_next_move(board, options)
    return res

def clean_puzzle(puzzle):
    """
    converts input from request.form to a string format readable by Sudoku
    """
    output = ''
    for val in puzzle.values():
        if val == '':
            output += '.'
        elif int(val) in range(1, 10):
            output += val
    return output


import jellyfish

def readFile(file):
	with open(file, 'r') as f:
		file = f.readlines()
		f.close()
		return file

def normalization(listfile):
	for i in range(len(listfile)):
		listfile[i] = listfile[i].strip()
	return listfile

misspell = readFile('wiki_misspell.txt')
correct = readFile('wiki_correct.txt')
dictionary = readFile('dict.txt')

misspell = normalization(misspell)
correct = normalization(correct)
dictionary = normalization(dictionary)

f = open('result.txt', 'w')

correctnumber = 0
total = 0

i = 0
while(i < len(misspell)):
	mis = misspell[i]
	soundex_mis = jellyfish.soundex(mis)
	resultlist = []
	for word in dictionary:
		distance = jellyfish.levenshtein_distance(mis, word)
		resultlist.append(distance)

	minimum = min(resultlist)
	j = 0
	result = [mis]
	while(j < len(resultlist)):
		if resultlist[j] == minimum:
			soundex_result = jellyfish.soundex(dictionary[j])
			if soundex_mis == soundex_result:
				result.append(dictionary[j])
				total += 1

		j += 1
	if correct[i] in result:
		result.append(True)
		correctnumber += 1
	else:
		result.append(False)

	result.append(correct[i])
	print(result)
	f.write(str(result) + '\n')
	i += 1

f.write(str(correctnumber) + ' ')
f.write(str(total))
f.close()

研一下/COMP90049 Knowledge Technologies/Assignment 1/data/levenshtein_soundex.py


import argparse, os, time, sys, csv, io, time
import numpy as np
import pandas as pd
from PIL import Image, ImageFile

import torch
import torch.nn as nn
import torchvision.models as models
from torchvision import transforms as trn
from torch.nn import functional as F

### ???
ImageFile.LOAD_TRUNCATED_IMAGES = True

### cuda gpu ###
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def feature_extractor():
    resnet50 = models.resnet50(num_classes=1000, pretrained=True)
    modules = list(resnet50.children())[:-1]
    resnet50 = nn.Sequential(*modules)
    return resnet50

def extractor_train(data, tag, folder_dir):
    # load the image transformer
    centre_crop = trn.Compose([
            trn.Resize((256,256)),
            trn.CenterCrop(224),
            trn.ToTensor(),
            trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    resnet50 = feature_extractor().to(device)

    image_feature, total_label, total_hashtag = [], [], []
    st = time.time()
    for idx, (img, label) in enumerate(zip(data, tag), 1):
        print('{:>5d}/{} {}'.format(idx, len(data), img[0]), end=' '*10+'\r' if idx != len(data) else '\n')
        all_hashtag = label[0].strip()
        label = img[0].split('/')[1]
        tmp_name = os.path.join(folder_dir, img[0])
        img = Image.open(tmp_name).convert('RGB')
        input_img = centre_crop(img).unsqueeze(0)
        ### gray image ###
        if input_img.shape[1] == 1:
            print('hh')
            input_img = input_img.detach().numpy()
            fake_img = np.zeros((1, 3, 224, 224))
            for i in range(3):
                fake_img[0, i, :] = input_img[0, 0, :]
            input_img = torch.from_numpy(fake_img).type(torch.FloatTensor)
        input_img = input_img.to(device)
        logit = resnet50.forward(input_img).squeeze().detach().cpu().numpy()

        image_feature.append(logit)
        total_label.append(label)
        total_hashtag.append(all_hashtag)

    np.savez('image_feature.npz',
             image_feature=image_feature[:-5730],
             total_label=total_label[:-5730],
             total_hashtag=total_hashtag[:-5730],
             val_image_feature=image_feature[-5730:],
             val_total_label=total_label[-5730:],
             val_total_hashtag=total_hashtag[-5730:])
    print('total time: ', time.time()-st)

if __name__ == "__main__":
    ### Path of data ###
    folder_dir = "../Hashtag/HARRISON/"
    filename = "tag_list.txt"
    data = pd.read_csv(folder_dir+'data_list.txt', header=None).values
    tag = pd.read_csv(folder_dir+'tag_list.txt', header=None).values
    extractor_train(data, tag, folder_dir)


from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.db import transaction
from .forms import UserForm,ProfileForm
from django.contrib.auth import logout

def home(request):
    return render(request, 'home.html')

@login_required
@transaction.atomic
def update_profile(request):
    if request.method == 'POST':
        user_form = UserForm(request.POST, instance=request.user)
        profile_form = ProfileForm(request.POST, instance=request.user.profile)
        if user_form.is_valid() and profile_form.is_valid():
            user_form.save()
            profile_form.save()
            return HttpResponseRedirect('/')
        #else:
        #    messages.error(request, _('Please correct the error below.'))
    else:
        user_form = UserForm(instance=request.user)
        profile_form = ProfileForm(instance=request.user.profile)
    return render(request, 'profile.html', {
        'user_form': user_form,
        'profile_form': profile_form
    })

def Logout(request):
    logout()
    return HttpResponseRedirect('/')


from flask import Flask, jsonify
import hashlib

app = Flask(__name__)

@app.route("/md5/<string:strname>", methods=['GET'])
def conv_md5(strname):
    hash_obj =hashlib.md5(strname.encode())
    result = {
    "hash": "md5",
    "cleartext": strname,
    "hashedtext": hash_obj.hexdigest()
    }
    result = jsonify(result)
    # Returns JSON
    return result

@app.route("/sha256/<string:strname>", methods=['GET'])
def conv_sha256(strname):
    hash_obj =hashlib.sha256(strname.encode())
    result = {
    "hash": "sha256",
    "cleartext": strname,
    "hashedtext": hash_obj.hexdigest()
    }
    result = jsonify(result)
    # Returns JSON
    return result

# Checks to see if the name of the package is the run as the main package.
if __name__ == "__main__":
    # Runs the Flask application only if the main.py file is being run.
    app.run(host='0.0.0.0')


from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer
import json

from chat.models import *

class ChatConsumer(WebsocketConsumer):
    def connect(self):
        self.room_name = self.scope['url_route']['kwargs']['room_name']
        self.room_group_name = 'chat_%s' % self.room_name

        # Join room group
        async_to_sync(self.channel_layer.group_add)(
            self.room_group_name,
            self.channel_name
        )

        self.accept()

    def disconnect(self, close_code):
        # Leave room group
        async_to_sync(self.channel_layer.group_discard)(
            self.room_group_name,
            self.channel_name
        )

    # Receive message from WebSocket
    def receive(self, text_data):
        text_data_json = json.loads(text_data)
        print(text_data)
        chat=Chat.objects.get(id=text_data_json['chat'])
        user=User.objects.get(id=int(text_data_json['user']))
        message = text_data_json['message']
        create_msg=Message(chat=chat,autor=user,message=message)
        create_msg.save()

        # Send message to room group
        async_to_sync(self.channel_layer.group_send)(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message':text_data_json['message'] ,
                'author':int(text_data_json['user'])
            }
        )

    # Receive message from room group
    def chat_message(self, event):
        message = event['message']
        author_row= event['author']
        author=User.objects.get(id=int(author_row)).username

         # Send message to WebSocket
        self.send(text_data=json.dumps({
            'message': message,
            'author':author,
            'author_id':author_row
        }))


__author__ = 'Farhan'

from random import randint
from bisect import bisect_left

from numpy import *
import matplotlib.pyplot as plt

from process_GTEx import Process
from pylab import *

class Histogram:
    '''
        Create a histogram using rankings from output file. If additional parameters are set,
        create a histogram by randomly selecting individuals
            @input: output_file gives rankings and number of individuals per gene
            tissue_file gives tissue data to generate random histogram
    '''

    def __init__(self, output_file, tissue):
        self.file = output_file
        self.rankings = []
        self.gene_number = []
        self.isHigh = []
        self.randRankings = []

        self.genes_new = tissue.genes
        self.data_new = tissue.data

    # process ranking/gene data
    def processData(self):

        for l in self.file:
            w = l.split()
            if len(w) > 0:
                self.gene_number.append(w[0])
                # append ranking (4th element) to rankings array
                if w[4] > 0:
                    self.rankings.append(w[4])

                # ranking is for k_high
                if w[2] > 0:
                    self.isHigh.append(True)
                # ranking is for k_low
                else:
                    self.isHigh.append(False)
       # print self.rankings
        self.rankings = array(map(float, self.rankings))

    def createHistogram(self):
        plt.hist(self.rankings, bins = 10)
        #savefig('data_hist.png')

        plt.show()

    def createRandomHistogram(self):
        random_rankings = []
        counter = -1
        for gene in self.gene_number:
            counter = counter + 1
            index = nonzero(self.genes_new == gene)
            expr_data_gene = self.data_new[index]
            expr_data_gene = expr_data_gene[0]
            # generate random number between 0 and last index in expression data
            rand_num = randint(0, len(expr_data_gene) - 1)

            # append ranking
            self.randRankings.append(rand_num)

        plt.hist(self.randRankings, bins = 10)
        #savefig('rand_hist.png')
        plt.show()

    def index(self, a, x):
        'Locate the leftmost value exactly equal to x using binary search'
        i = bisect_left(a, x)
        if i != len(a) and a[i] == x:
            return i
        else:
            return -1

    def main(self):
        self.processData()
        self.createHistogram()


"""automatization URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""

from django.contrib import admin
from django.urls import path
# from automatization.data_migration.views import Databases_Generate_File,Backups_File,Long_Range_Move_File,Long_Range_File_Implement,\
#     Long_Range_Check_File

from automatization.data_migration.databases_operation import Ins_Databases_Migration_Task,Del_Databases_Migration_Task,Upd_Databases_Migration_Task,\
    Sel_Databases_Migration_Task,Sel_Implement_Status_Task,Ins_File_Migration_Task,Del_File_Migration_Task,Upd_File_Migration_Task,Sel_File_Migration_Task

from automatization.data_migration.apscheduler_task import Upd_Implement_Status_Task

urlpatterns = [
    # path('admin/', admin.site.urls),
    # path('databases_generate_file',Databases_Generate_File),#生成数据库文件
    # path('backups_file',Backups_File),
    # path('long_range_move_file',Long_Range_Move_File),#文件上传
    # path('long_range_file_implement',Long_Range_File_Implement),#远程数据同步

    path('ins_databases_migration_task',Ins_Databases_Migration_Task), #添加数据库迁移信息
    path('del_databases_migration_task',Del_Databases_Migration_Task),#删除....
    path('upd_databases_migration_task',Upd_Databases_Migration_Task),#修改...
    path('sel_databases_migration_task',Sel_Databases_Migration_Task),#查询...
    path('sel_implement_status_task',Sel_Implement_Status_Task), #查询历史任务信息
    path('upd_implement_status_task',Upd_Implement_Status_Task),#执行任务
    path('ins_file_migration_task',Ins_File_Migration_Task),#添加文件迁移信息
    path('del_file_migration_task',Del_File_Migration_Task),#删除...
    path('upd_file_migration_task',Upd_File_Migration_Task),#修改...
    path('sel_file_migration_task',Sel_File_Migration_Task),#查询...

]


import torch
import torch.nn.functional as F
from torch.nn.modules.batchnorm import _BatchNorm
from Clustering import *

import random

import os

import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as mpimg

def group_norm(input, group, running_mean, running_var, weight=None, bias=None,
               use_input_stats=True, momentum=0.1, eps=1e-5):
    r"""Applies Group Normalization for channels in the same group in each data sample in a
    batch.
    See :class:`~torch.nn.GroupNorm1d`, :class:`~torch.nn.GroupNorm2d`,
    :class:`~torch.nn.GroupNorm3d` for details.
    """
    if not use_input_stats and (running_mean is None or running_var is None):
        raise ValueError('Expected running_mean and running_var to be not None when use_input_stats=False')

    b, c = input.size(0), input.size(1)
    if weight is not None:
        weight = weight.repeat(b)
    if bias is not None:
        bias = bias.repeat(b)

    def _instance_norm(input, group, running_mean=None, running_var=None, weight=None,
                       bias=None, use_input_stats=None, momentum=None, eps=None):
        # Repeat stored stats and affine transform params if necessary
        if running_mean is not None:
            running_mean_orig = running_mean
            running_mean = running_mean_orig.repeat(b)
        if running_var is not None:
            running_var_orig = running_var
            running_var = running_var_orig.repeat(b)
        eps = 1e-5
        G = group
        input_mod = input
        out_final = input
        input_cpu = input.to('cpu')
        x = input_cpu.to('cpu')
        x_np = x.detach().numpy()
        x_np = np.nan_to_num(x_np)
        # At this point we have just stored our input on CPU with Numpy
        print(33333,x_np,x_np.shape)
        Nabs, C, H, W = x_np.shape
        N = Nabs
        # N refers to the __? What the batch size?
        # res_x is the original shape of our input
        res_x = np.zeros((N, C, H, W))
        # this is a looser form where we are just by channel dimensions
        x_np_new = np.zeros((C, N * H * W))
        # we loop through the channels
        for i in range(C):
            # we are converting the raw input data such that there are channel rows of large N*H*W blocks
            # why?
            x_np_new[i, :] = np.reshape(x_np[:, i, :, :], (1, N * H * W))
        # 60 by 10000 trying to cluster in this basis
        # we create a copy of our reformatted data
        image_vector = np.asarray(x_np_new)
        Data = data_preparation(n_cluster=G, data=image_vector[:, :])
        # Up to here we are all correct, as long as we are clustering in the right basis
        count = 0
        Common_mul = 1
        Common_add = 0
        tmp1 = torch.zeros(())
        for val in range(G):
            # this creates all a list of all the indices where our label is valid
            inx = np.argwhere(Data.labels_ == val)
            if len(inx) > 0:
                tmp = torch.zeros((input.shape[0], len(inx), input.shape[2], input.shape[3]))
            for idx, idxx in enumerate(inx):
                if len(inx) == 0:
                    pass
                else:
                    # This is line of code is effectively saying fill into our temp channel equal to
                    # the original index from input for all the matching indices. so we have temp has
                    # number of channels assigned to that cluster
                    tmp[:, idx, :, :] = input[:, idxx[0], :, :]
                    # now we have the correct grouping so we choose to normalize
            # STAT is the function which I think has a bug...
            out_final_tmp = (Stat_torch(tmp)).float().to('cuda')
            for idx, idxx in enumerate(inx):
                if (len(inx) == 0):
                    pass
                else:
                    # here we assign that the the original index of out_final is equal to the temp
                    # index it was given in the normalized group
                    out_final[:, idxx[0], :, :] = out_final_tmp[:, idx, :, :]
        # we reshape it to the original size
        # this is probably fine since code doesnt error...
        # although double check this last line
        return out_final.view(b, c, *input.size()[2:])

    return _instance_norm(input, group, running_mean=running_mean,
                          running_var=running_var, weight=weight, bias=bias,
                          use_input_stats=use_input_stats, momentum=momentum,
                          eps=eps)

class HoogiNorm(_BatchNorm):
    def __init__(self, num_features, num_groups=5, eps=1e-5, momentum=0.1,
                 affine=False, track_running_stats=False):
        self.num_groups = num_groups
        self.track_running_stats = track_running_stats
        super(HoogiNorm, self).__init__(int(num_features / num_groups), eps,
                                        momentum, affine, track_running_stats)

    def _check_input_dim(self, input):
        return NotImplemented

    def forward(self, input):
        self._check_input_dim(input)

        return group_norm(
            input, self.num_groups, self.running_mean, self.running_var, self.weight, self.bias,
            self.training or not self.track_running_stats, self.momentum, self.eps)

# Deprecated Method of Normalization
def Stat(IN):
    tmp = np.zeros((IN.shape[0]))
    tmp2 = 0
    eps = 1e-5
    sigma = np.zeros((IN.shape[0], 1))
    out = IN
    # print('IN is:')
    # print(IN.shape)
    '''for i in range(IN.shape[1]):
        tmp += np.sum(IN[:,i,:,:])'''
    for i in range(IN.shape[0]):
        # for j in range(IN.shape[1]):
        tmp[i] += np.sum(IN[i, :, :, :])

    # tmp = np.sum(IN, where=[False, True, True, True])
    mu = (1 / (IN.shape[1] * IN.shape[2] * IN.shape[3])) * tmp
    for i in range(IN.shape[0]):
        sigma[i] = np.sqrt(
            (1 / (IN.shape[1] * IN.shape[2] * IN.shape[3])) * (np.sum((IN[i, :, :, :] - mu[i]) ** 2) + eps))
    # sigma = np.sqrt((1 / (IN.shape[0] * IN.shape[2] * IN.shape[3])) * tmp2)
    for i in range(IN.shape[0]):
        for j in range(IN.shape[1]):
            out[i, j, :, :] = (1 / sigma[i]) * (IN[i, j, :, :] - mu[i])
    print(out)
    return out

# Current Method of Normalization
def Stat_torch(IN):
    print('I AM WORKING UP TO HERE')
    tmp = torch.zeros((IN.shape[0]))
    tmp2 = 0
    eps = 1e-5
    sigma = torch.zeros((IN.shape[0], 1))
    out = IN
    res = torch.zeros((IN.shape[0], 2, IN.shape[2], IN.shape[3]))
    res[:, 0, :, :], res[:, 1, :, :] = torch.std_mean(IN + eps, dim=1)
    for i in range(IN.shape[0]):
        for j in range(IN.shape[1]):
            out[i, j, :, :] = (1 / res[i, 0]) * (IN[i, j, :, :] - res[i, 1])
    return out


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '0003_auto_20140926_2347'),
    ]

    operations = [
        migrations.CreateModel(
            name='PagedownConfig',
            fields=[
                ('cmsplugin_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='cms.CMSPlugin')),
                ('markdown', models.TextField()),
                ('html', models.TextField()),
                ('plaintext', models.TextField()),
                ('css_id', models.CharField(help_text=b'Add an optional CSS ID to the rendered outputs wrapping element', max_length=50, blank=True)),
                ('css_classes', models.CharField(help_text=b'Add CSS classes to the rendered outputs wrapper element', max_length=50, blank=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('cms.cmsplugin',),
        ),
    ]


from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Optional

import yul.yul_ast as ast
from yul.AstMapper import AstMapper

@dataclass
class BlockEnv:
    split_stmt: Optional[ast.VariableDeclaration] = None
    split_stmt_count: int = 0

    def process_subexpr(self, call: ast.FunctionCall) -> str:
        name = f"__warp_subexpr_{self.split_stmt_count}"
        self.split_stmt = ast.VariableDeclaration(
            variables=[ast.TypedName(name)], value=call
        )
        return name

class ExpressionSplitter(AstMapper):
    def __init__(self):
        super().__init__()
        self.block_env_stack: list[BlockEnv] = []

    def visit_function_call(self, node: ast.FunctionCall):
        if len(self.path) == 1:  # no parent
            return ast.FunctionCall(node.function_name, self.visit_list(node.arguments))
        parent = self.path[-2]
        if isinstance(
            parent, (ast.Assignment, ast.VariableDeclaration, ast.ExpressionStatement)
        ):
            return ast.FunctionCall(node.function_name, self.visit_list(node.arguments))
        var_name = self.block_env_stack[-1].process_subexpr(node)
        return ast.Identifier(var_name)

    def visit_block(self, node: ast.Block):
        with self._new_block_env() as env:
            new_stmts = []
            for stmt in node.statements:
                new_stmt = self.visit(stmt)
                split_stmts = [new_stmt]  # in the reverse order of declaration
                while env.split_stmt:
                    split_stmt = env.split_stmt
                    env.split_stmt = None
                    split_stmts.append(self.visit(split_stmt))
                split_stmts.reverse()
                new_stmts.extend(split_stmts)
            return ast.Block(tuple(new_stmts))

    @contextmanager
    def _new_block_env(self) -> BlockEnv:
        env = BlockEnv()
        self.block_env_stack.append(env)
        try:
            yield env
        finally:
            self.block_env_stack.pop()


"""empty message

Revision ID: 8964ced32de0
Revises: 7409eac83319
Create Date: 2021-08-17 12:36:17.159803

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '8964ced32de0'
down_revision = '7409eac83319'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('answers_to_questions', 'question_id',
               existing_type=sa.INTEGER(),
               nullable=False)
    op.drop_constraint(None, 'answers_to_questions', type_='foreignkey')
    op.create_foreign_key(None, 'answers_to_questions', 'quiz_answers', ['id'], ['id'])
    op.drop_column('answers_to_questions', 'quiz_id')
    op.add_column('possible_answers', sa.Column('text_html', sa.Text(), nullable=True))
    op.add_column('questions', sa.Column('text_html', sa.Text(), nullable=True))
    op.add_column('quizzes', sa.Column('description_html', sa.Text(), nullable=True))
    op.create_unique_constraint(None, 'users', ['username'])
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'users', type_='unique')
    op.drop_column('quizzes', 'description_html')
    op.drop_column('questions', 'text_html')
    op.drop_column('possible_answers', 'text_html')
    op.add_column('answers_to_questions', sa.Column('quiz_id', sa.INTEGER(), nullable=True))
    op.drop_constraint(None, 'answers_to_questions', type_='foreignkey')
    op.create_foreign_key(None, 'answers_to_questions', 'quizzes', ['quiz_id'], ['id'])
    op.alter_column('answers_to_questions', 'question_id',
               existing_type=sa.INTEGER(),
               nullable=True)
    # ### end Alembic commands ###


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse
import matplotlib.transforms as transforms
from scipy.stats import multivariate_normal, pearsonr, spearmanr, multinomial

def Covariance(variances, correlations):
    covariance = [[] for i in range(len(variances))]
    for i in range(len(variances)):
        for k in range(len(variances)):
            covariance[i].append((variances[i] * variances[k]) * correlations[i][k])
    return covariance

def QuadrantCorrelation(x, y):
    centerRVS = [x - np.mean(x), y - np.mean(y)]
    centerRVS = [[centerRVS[0][i], centerRVS[1][i]] for i in range(len(centerRVS[0]))]
    n1 = len([elem for elem in centerRVS if elem[0] >= 0 and elem[1] >= 0])
    n2 = len([elem for elem in centerRVS if elem[0] <= 0 and elem[1] >= 0])
    n3 = len([elem for elem in centerRVS if elem[0] <= 0 and elem[1] <= 0])
    n4 = len([elem for elem in centerRVS if elem[0] >= 0 and elem[1] <= 0])
    return (n1 + n3 - n2 - n4) / float(len(x))

def mean_2(x):
    return np.mean([elem * elem for elem in x])

def MixedNormalDistribution(weights, means, covariances, size):
    cases = multinomial(1, weights).rvs(size)
    rvs = []
    for case in cases:
        k = case.tolist().index(1)
        rvs.append(multivariate_normal.rvs(means[k], covariances[k]))
    rvs = np.asarray(rvs)
    return [rvs[:, k] for k in range(len(rvs[0]))]

def confidence_ellipse(x, y, ax, n_std=3.0, edgecolor='red', linestyle='-'):
    cov = np.cov(x, y)
    pearson = cov[0, 1] / np.sqrt(cov[0, 0] * cov[1, 1])

    radius_x = np.sqrt(1 + pearson)
    radius_y = np.sqrt(1 - pearson)
    ellipse = Ellipse((0, 0), width=radius_x * 2, height=radius_y * 2, facecolor='none',
                      edgecolor=edgecolor, linestyle=linestyle)

    scale_x = np.sqrt(cov[0, 0]) * n_std
    mean_x = np.mean(x)
    scale_y = np.sqrt(cov[1, 1]) * n_std
    mean_y = np.mean(y)

    transf = transforms.Affine2D() \
        .rotate_deg(45) \
        .scale(scale_x, scale_y) \
        .translate(mean_x, mean_y)

    ellipse.set_transform(transf + ax.transData)
    return ax.add_patch(ellipse)

if __name__ == "__main__":
    sizes = [20, 60, 100]
    count = 1000
    means = [0, 0]
    variances_1, variances_2 = [1, 1], [10, 10]
    correlations_1, correlations_2 = [0, 0.5, 0.9], [0.9, -0.9]
    measuares = [['$E(z)$', np.mean], ['$E(z^2)$', mean_2], ['$D(z)$', np.var]]
    empiricalCorrelations = ['r', 'r_S', 'r_Q']
    variateCorrelations = [[[[0 for l in range(len(empiricalCorrelations))]
                             for k in range(len(measuares))]
                            for j in range(len(sizes))]
                           for i in range(len(correlations_1))]
    mixedVariateCorrelations = [[[0 for l in range(len(empiricalCorrelations))]
                                 for k in range(len(measuares))]
                                for j in range(len(sizes))]

    for i in range(len(correlations_1)):
        covariance = Covariance(variances_1,
                                [[1, correlations_1[i]], [correlations_1[i], 1]])
        for j in range(len(sizes)):
            P, S, Q = [], [], []
            for m in range(count):
                rvs = multivariate_normal.rvs(means, covariance, sizes[i])
                x, y = rvs[:, 0], rvs[:, 1]
                p, tmp = pearsonr(x, y)
                s, tmp = spearmanr(x, y)
                P.append(p)
                S.append(s)
                Q.append(QuadrantCorrelation(x, y))
            empCors = [P, S, Q]
            for k in range(len(measuares)):
                for l in range(len(empCors)):
                    variateCorrelations[i][j][k][l] = measuares[k][1](empCors[l])

    mixedCovariances = [Covariance(variances_1, [[1, correlations_2[0]], [correlations_2[0], 1]]),
                        Covariance(variances_2, [[1, correlations_2[1]], [correlations_2[1], 1]])]
    for j in range(len(sizes)):
        P, S, Q = [], [], []
        for m in range(count):
            rvs = MixedNormalDistribution([0.9, 0.1], [means, means], mixedCovariances, sizes[j])
            p, tmp = pearsonr(rvs[0], rvs[1])
            s, tmp = spearmanr(rvs[0], rvs[1])
            P.append(p)
            S.append(s)
            Q.append(QuadrantCorrelation(rvs[0], rvs[1]))
        empCors = [P, S, Q]
        for k in range(len(measuares)):
            for l in range(len(empCors)):
                mixedVariateCorrelations[j][k][l] = measuares[k][1](empCors[l])

    for i in range(len(correlations_1)):
        with open('pho = ' + str(correlations_1[i]) + ".tex", 'w') as file:
            for j in range(len(sizes)):
                file.write(" $n$=" + str(sizes[j]) + "&$r$" + "&$r_S$" + r"&$r_Q$\\")
                file.write("\hline")
                for k in range(len(measuares)):
                    file.write(measuares[k][0])
                    for m in range(len(empiricalCorrelations)):
                        file.write("&" + str(round(variateCorrelations[i][j][k][m], 4)))
                    file.write(r"\\")
                    file.write("\hline")
                file.write(r" & & & \\")
                file.write("\hline ")

    with open('Mixed.tex', 'w') as file:
        for j in range(len(sizes)):
            file.write(" $n$=" + str(sizes[j]) + "&$r$" + "&$r_S$" + r"&$r_Q$\\")
            file.write("\hline")
            for k in range(len(measuares)):
                file.write(measuares[k][0])
                for m in range(len(empiricalCorrelations)):
                    file.write("&" + str(round(mixedVariateCorrelations[j][k][m], 4)))
                file.write(r"\\")
                file.write("\hline")
            file.write(r" & & & \\")
            file.write("\hline ")

    for i in range(len(correlations_1)):
        covariance = Covariance(variances_1,
                                [[1, correlations_1[i]], [correlations_1[i], 1]])
        for j in range(len(sizes)):
            rvs = multivariate_normal.rvs(means, covariance, sizes[j])
            x, y = rvs[:, 0], rvs[:, 1]
            fig, ax_nstd = plt.subplots()
            ax_nstd.axvline(c='grey', lw=1)
            ax_nstd.axhline(c='grey', lw=1)
            ax_nstd.scatter(x, y, s=0.5)
            confidence_ellipse(x, y, ax_nstd)
            ax_nstd.scatter(0, 0, c='red', s=3)
            ax_nstd.set_xlabel('x')
            ax_nstd.set_ylabel('y')
            ax_nstd.set_title('n = ' + str(sizes[j]) + r', $\rho$ =' + str(correlations_1[i]))
            plt.show()
            fig.savefig('Ellipse ' + str(correlations_1[i]) + ' ' + str(sizes[j]) + '.png')


import torch
import math
from torch.optim import Optimizer
class GGDO(Optimizer):
    '''
        Implements the Gaussian Gradient Distruption Optimization
    '''
    def __init__(self, params, lr=1e-2, momentum=0.9, weight_decay=5e-4,eps=1e-6, noise=0.1):
        if not 0.0 <= lr:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if not 0.0 <= weight_decay:
            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
        if not 0.0 <= eps:
            raise ValueError("Invalid eps value: {}".format(eps))
        if not 0.0 <= momentum:
            raise ValueError("Invalid momentum value: {}".format(momentum))
        if not 0.0 <= noise:
            raise ValueError("Invalid noise value: {}".format(noise))

        defaults = dict(lr=lr, momentum=momentum, weight_decay=weight_decay, eps=eps, noise=noise)
        super(GGDO, self).__init__(params, defaults)

    def step(self, closure=None):
        """Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            weight_decay = group['weight_decay']
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                if grad.is_sparse:
                    raise RuntimeError('Gaussian Gradients does not support sparse gradients')
                state = self.state[p]

                #if weight_decay != 0:
                #    grad.add_(weight_decay, p.data)

                # State initialization
                if len(state) == 0:
                    # Intialize mean and variance to zero
                    state['mean'] = torch.zeros_like(p.data)
                    state['variance'] = torch.zeros_like(p.data)
                    state['std'] = torch.zeros_like(p.data)
                    state['step'] = 0

                mean = state['mean'] # Works now
                var = state['variance']
                std = state['std']

                state['step'] += 1

                # Getting mean,std at previous step
                old_mean = mean.clone()
                old_std = std.clone()

                # Calculating gradients
                new_updt = torch.normal(mean=old_mean, std=old_std)
                updt = grad.add(group['noise'],new_updt)
                if weight_decay != 0:
                    updt.add_(weight_decay, p.data)

                # Updating mean
                mean = mean.mul(group['momentum']).add(updt)

                part_var1 = grad.add(-old_mean)
                part_var2 = grad.add(-mean)

                new_std = torch.pow(old_std,2).mul(group['momentum']).addcmul(1,part_var1,part_var2).add(group['eps'])
                new_std = torch.pow(torch.abs_(new_std),1/2)
                std.add_(-1,std).add_(new_std)

                p.data.add_(-group['lr'],updt)

        return loss


import couchdb
from math import radians, cos, sin, asin, sqrt
from flask import Flask, request, jsonify
from collections import Counter
import sys
from nltk.corpus import stopwords
import datetime
import json
import re
from textblob import TextBlob
import package.predict as predict
from flask_cors import CORS, cross_origin
import os

app = Flask(__name__)
CORS(app)

# import stopwords
stopWords = set(stopwords.words('english'))

# public dataset for poi
mel_poi = []
syd_poi = []

# reading DBsettings
def checkSettings():
    try:
        with open(os.path.dirname(__file__) + '/DBsettings.json') as json_file:
            db_settings = json.load(json_file)
            mel_poi_path = db_settings['mel_poi_path']
            syd_poi_path = db_settings['syd_poi_path']
            ins_point_path = db_settings['ins_point']
            twi_point_path = db_settings['twi_point']
        with open(mel_poi_path) as mel_json_file:
            mel_poi = json.load(mel_json_file)['result']
        with open(syd_poi_path) as syd_json_file:
            syd_poi = json.load(syd_json_file)['result']
        with open(ins_point_path) as ins_json_file:
            ins_point = json.load(ins_json_file)
        with open(twi_point_path) as twi_json_file:
            twi_point = json.load(twi_json_file)
    except:
        print('No DBsettings.json, No db connection has been established. Or no POI files have been found.')
        sys.exit(1)
    return db_settings,mel_poi, syd_poi,ins_point,twi_point

def haversine(lon1, lat1, lon2, lat2):
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
    # haversine formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
    c = 2 * asin(sqrt(a))
    km = 6367 * c
    return km

# finding the nearest poi and return
def findTheClosestPOI(lon, lat, loc):
    nearestPOI = -1
    nearestDist = -1
    if loc == 'mel':
        poiDataset = mel_poi
    else:
        poiDataset = syd_poi
    for poidata in poiDataset:
        thisDist = haversine(
            lon, lat, poidata['geometry']['coordinates'][0][0], poidata['geometry']['coordinates'][0][1])
        if nearestDist == -1 or thisDist < nearestDist:
            nearestDist = thisDist
            nearestPOI = poiDataset.index(poidata)
    return nearestPOI, nearestDist

# return the top 10 in each list [word, noun_phrases, hashtags]
def rankContents(databaseResult):
    hashtags = []
    words = []
    nounPhrase = []
    hashtagsWithRanking = {}
    wordsWithRanking = {}
    nounPhraseWithRanking = {}
    hashtagPartten = re.compile('\#\S+')
    for row in databaseResult:
        newRow = re.sub(
            r'(\\\w)|(http\S+)|(\[)|(\])|(\…)|(\@)|(\\)|(\")|(\/)', '', row)
        newRow = newRow.lower()
        hashtags.extend(re.findall(hashtagPartten, newRow))
        newRow = re.sub(hashtagPartten, '', newRow)
        blob = TextBlob(newRow)
        words.extend([n for n, t in blob.tags if t ==
                      'NN' and n not in stopWords])
        nounPhrase.extend([n for n in blob.noun_phrases if n != "i 'm"])
    hashtagsWithRanking = Counter(hashtags)
    wordsWithRanking = Counter(words)
    nounPhraseWithRanking = Counter(nounPhrase)
    return {
        'hashtags': hashtagsWithRanking.most_common(10),
        'words': wordsWithRanking.most_common(10),
        'nounphrase': nounPhraseWithRanking.most_common(10)
    }

# managing route of query for hotspot analysis
@app.route('/query')
def query():
    startYear = int(request.args.get('Sy'))
    startMonth = int(request.args.get('Sm'))
    startDay = int(request.args.get('Sd'))
    startHour = int(request.args.get('Sh'))
    endYear = int(request.args.get('Ey', startYear))
    endMonth = int(request.args.get('Em', startMonth))
    endDay = int(request.args.get('Ed', startDay))
    endHour = int(request.args.get('Eh', startHour))
    lon = request.args.get('lon', "")
    lat = request.args.get('lat', "")
    dist = float(request.args.get('dist', 0.5))
    if dist<=0.1:
        dist = 0.1
    if lon != "" and lat != "":
        lon = float(lon)
        lat = float(lat)
    loc = request.args.get('loc', 'mel')
    if(loc == 'mel'):
        db = meldb
        poiDataset = mel_poi
    else:
        db = syddb
        poiDataset = syd_poi
    print(startYear, startMonth, startDay, startHour,
          endYear, endMonth, endDay, endHour, lat, lon, loc, db)
    result = []
    resultForPredict = []
    viewByText = db.view('chaotest/bytext', startkey=[startYear, startMonth, startDay, startHour,
                                                      0, 0, 0, "AEST"], endkey=[endYear, endMonth, endDay, endHour, 0, 0, 0, "AEST"])
    if lon != "" and lat != "":
        for text in viewByText:
            dis = haversine(
                lon, lat, text.value['coordinates'][1], text.value['coordinates'][0])
            if(dis <= dist):
                result.append(text.value['text'])
                resultForPredict.append(text)

        numOfPostsForAnalyze = len(result)
        result = rankContents(result)
        predict_result = predict.predict_from_tweets(resultForPredict)
        # find the nearest POI
        poiindex,poidist = findTheClosestPOI(lon,lat,loc)
        # return json formate
        return jsonify({
            'numberOfPosts': numOfPostsForAnalyze,
            'result': result,
            'lenght': len(result),
            'nearest_poi': poiDataset[poiindex],
            'min_distance': poidist,
            'predict': predict_result
        })

# managing route of sum for comparison panel
@app.route('/sum')
def sum():
    words = []
    nounphrase = []
    hashtags = []
    hashtagsWithRanking = {}
    wordsWithRanking = {}
    nounPhraseWithRanking = {}
    dataIDs = []
    startYear = int(request.args.get('Sy'))
    startMonth = int(request.args.get('Sm'))
    startDay = int(request.args.get('Sd'))
    endYear = int(request.args.get('Ey'))
    endMonth = int(request.args.get('Em'))
    endDay = int(request.args.get('Ed'))
    loc = request.args.get('loc', 'mel')
    if(loc == 'mel'):
        db = mel_nlp
    else:
        db = syd_nlp

    dataset = db.view(
        'nlp/sum', startkey=[startYear, startMonth, startDay], endkey=[endYear, endMonth, endDay])
    for data in dataset:
        words.extend(data['value']['words'])
        nounphrase.extend(data['value']['nounphrase'])
        hashtags.extend(data['value']['hashtags'])
    hashtagsWithRanking = Counter(hashtags)
    wordsWithRanking = Counter(words)
    nounPhraseWithRanking = Counter(nounphrase)
    return jsonify({
        'result': {
            'hashtags': hashtagsWithRanking.most_common(10),
            'words': wordsWithRanking.most_common(10),
            'nounphrase': nounPhraseWithRanking.most_common(10)
        }
    })

# managing for user behaviour and return the correspoding json
@app.route('/userBehaviour')
def userBehaviour():
    typ = request.args.get('q')
    if typ == 'ins':
        return jsonify(ins_point)
    else:
        return jsonify(twi_point)

if __name__ == '__main__':
    db_settings,mel_poi, syd_poi,ins_point,twi_point= checkSettings()
    couch = couchdb.Server(db_settings['destination_address'])
    mel_nlp = couch['mel_nlp']
    syd_nlp = couch['syd_nlp']
    ori_couch = couchdb.Server(db_settings['original_address'])
    meldb = ori_couch['melbourne']
    syddb = ori_couch['sydney']
    app.run(host='0.0.0.0', port=3000, debug=True, threaded=True)


from django.contrib import admin

from .models import User

class UserAdmin(admin.ModelAdmin):
    list_display = (
        'id', 'reputation', 'creation_date', 'display_name', 'last_access_date',
        'website_url', 'location', 'views', 'up_votes', 'down_votes', 'account_id'
    )
    list_display_links = (
        'id', 'reputation', 'creation_date', 'display_name', 'last_access_date',
        'website_url', 'location', 'views', 'up_votes', 'down_votes', 'account_id'
    )
    search_fields = ('id', 'display_name', 'account_id')

admin.site.register(User, UserAdmin)


import numpy
import random
import matplotlib.pyplot as plt

class SigmoidPerceptron:
    def __init__(self, learning_rate=0.1, dims=2):
        self.weights = [random.uniform(-2, 2) for _ in range(dims)]
        self.bias = random.uniform(-2, 2)

        self.learning_rate = learning_rate

    def output(self, x):
        z = numpy.dot(x, self.weights) + self.bias
        activation = 1 / (1 + numpy.exp(-z))
        return activation

    def predict(self, x):
        return 1 if self.output(x) > 0.5 else 0

    def train(self, x, output):
        diff = output - self.output(x)

        for i in range(len(self.weights)):
            self.weights[i] += self.learning_rate * diff * x[i]
        self.bias += self.learning_rate * diff

        return

    def evaluate(self, points, outputs):
        precission = 0
        for i in range(len(points)):
            precission += self.predict(points[i]) == outputs[i]

        return precission / len(points) * 100

def main():
    random.seed('holi ke ase')
    perceptron = SigmoidPerceptron()

    n = 1000
    iterations = 10
    batch_size = n // 5

    a = random.uniform(-1, 1)
    b = random.uniform(-5, 5)

    line_x = [-11, 11]
    line_y = [a * x + b for x in line_x]

    training_points = [(random.uniform(-10, 10), random.uniform(-10, 10)) for _ in range(n)]
    training_outputs = [a * x + b > y for (x, y) in training_points]

    test_points = [(random.uniform(-10, 10), random.uniform(-10, 10)) for _ in range(n)]
    test_outputs = [a * x + b > y for (x, y) in test_points]

    precissions = []

    for it in range(iterations):
        # shuffle training points each epoch
        if (it + 1) % 5 == 0:
            random.shuffle(training_points)
            training_outputs = [a * x + b > y for (x, y) in training_points]

        step = (it % 5)
        for i in range(batch_size * step, batch_size * (step + 1)):
            perceptron.train(training_points[i], training_outputs[i])

        precission = perceptron.evaluate(test_points, test_outputs)
        precissions.append(precission)

        points_0 = []
        points_1 = []
        for x in test_points:
            if perceptron.predict(x):
                points_0.append(x)
            else:
                points_1.append(x)

        plt.figure()
        plt.plot(line_x, line_y, '-g')
        plt.plot(*zip(*points_0), '.r')
        plt.plot(*zip(*points_1), '.b')
        plt.title(f'iteration {it + 1} got {precission:.1f}% acc')
        plt.show()

    plt.figure()
    plt.plot(range(iterations), precissions, '.-b')
    plt.title(f'precission on each iteration')
    plt.show()

    return

if __name__ == '__main__':
    main()


from django.shortcuts import render
import tweepy
import requests

CONSUMER_KEY = consumer_key
CONSUMER_SECRET = consumer_secret
CALLBACK_URL = 'http://127.0.0.1:8000/tweets/auth/'

def base(request):
    return render(request, 'authorization.html')

def authorization(request):
    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET, CALLBACK_URL)
    session = {}
    try:
        redirect_url = auth.get_authorization_url()
    except:
        print('Error! Failed to get request token')
    # auth.get_access_token("verifier_value")

    session['request_token'] = (auth.request_token['oauth_token'], auth.request_token['oauth_token_secret'])

    response = requests.get(redirect_url)

    return render(request, 'authorization.html')