from typing import List, Optional, Type

from aioarangodb import CursorCloseError
from aioarangodb.cursor import Cursor

from arangodantic.exceptions import CursorError, CursorNotFoundError

class ArangodanticCursor:
    """
    Wrapper for the aioarangodb.cursor.Cursor that will give back instances of the
    defined class rather than dictionaries.
    """

    __slots__ = [
        "cls",
        "cursor",
    ]

    def __init__(self, cls, cursor: Cursor):
        from arangodantic.models import Model

        self.cls: Type[Model] = cls
        self.cursor: Cursor = cursor

    def __aiter__(self):
        return self

    async def __anext__(self):  # pragma: no cover
        return await self.next()

    async def __aenter__(self):
        return self

    def __len__(self):
        return len(self.cursor)

    async def __aexit__(self, *_):
        await self.close(ignore_missing=True)

    def __repr__(self):
        cursor_id_str = ""
        if self.cursor.id:
            cursor_id_str = f" (Cursor: {self.cursor.id})"
        return f"<ArangodanticCursor ({self.cls.__name__}){cursor_id_str}>"

    async def close(self, ignore_missing: bool = False) -> Optional[bool]:
        """
        Close the cursor to free server side resources.

        :param ignore_missing: Do not raise an exception if the cursor is missing on the
        server side.
        :return: True if cursor was closed successfully, False if cursor was missing on
        the server side and **ignore_missing** was True, None if there were no cursors
        to close server-side.
        :raise CursorNotFoundError: If the cursor was missing and **ignore_missing** was
        False.
        """
        try:
            result: Optional[bool] = await self.cursor.close(
                ignore_missing=ignore_missing
            )
        except CursorCloseError as ex:
            if ex.error_code == 404:
                raise CursorNotFoundError(ex.error_message)
            raise

        return result

    async def next(self):
        return self.cls(**(await self.cursor.next()))

    async def to_list(self) -> List:
        """
        Convert the cursor to a list.
        """
        async with self as cursor:
            return [i async for i in cursor]

    @property
    def full_count(self) -> int:
        """
        Get the full count.

        :return: The full count.
        :raise CursorError: If the cursor statistics do not contain the full count.
        """
        stats = self.cursor.statistics()
        try:
            full_count: int = stats["fullCount"]
        except KeyError as e:
            raise CursorError(
                "Cursor statistics has no full count, did you use full_count=True?"
            ) from e

        return full_count


from scrapy.exceptions import DropItem
from twisted.internet import defer, reactor

from basedata.models import (AccountingSubject, Concept, ConceptStock,
                             Industry, IndustryStock, Report, ReportItem,
                             ReportType, Stock, Territory, XReport,
                             XReportItem)
from tdxStock.abstract_models import DynamicModel

class StockPipeline(object):
    """下载股票到数据库"""

    def __init__(self):
        self.stock_map = {}
        for stock in Stock.objects.only('code', 'name').all():
            self.stock_map[stock.code] = stock.name

    def process_item(self, item, spider):
        if spider.name == "stock_list":
            if item['code'] in self.stock_map:
                if item['name'] == self.stock_map[item['code']]:
                    # 去重
                    raise DropItem("Duplicate item found: %s" % item)

                # else 要更新
                self.stock_map.update({item['code']: item['name']})

                def update_stock():
                    Stock.objects.filter(code=item['code']).update(name=item['name'], pinyin=item["pinyin"])

                reactor.callInThread(update_stock)
            else:
                # 新增的
                self.stock_map.update({item['code']: item['name']})

                def create_stock():
                    Stock.objects.create(**dict(item))

                reactor.callInThread(create_stock)

        elif spider.name == "stock_detail":
            def update_stock():
                del item['name']
                Stock.objects.filter(code=item['code']).update(**dict(item))

            reactor.callInThread(update_stock)

        else:
            return item

class IndustryPipeline(object):

    def process_item(self, item, spider):
        if spider.name in ("industry_zjh", "industry_sw"):
            # item is dict(name, code, industry_id)
            async def process_industry_stock():
                industry = Industry.objects.filter(pk=item['industry_id']).first()
                stock = Stock.objects.filter(code__endswith=item['code']).first()
                if stock and industry:
                    IndustryStock.objects.get_or_create(
                        stock=stock,
                        industry=industry
                    )

            defer.ensureDeferred(process_industry_stock())
        else:
            return item

class ConceptPipeline(object):

    def process_item(self, item, spider):
        if spider.name in ("concept",):
            # item is dict(name, code, concept_id)
            async def process_concept_stock():
                concept = Concept.objects.filter(pk=item['concept_id']).first()
                stock = Stock.objects.filter(code__endswith=item['code']).first()
                if stock and concept:
                    ConceptStock.objects.get_or_create(
                        stock=stock,
                        concept=concept
                    )

            defer.ensureDeferred(process_concept_stock())
        else:
            return item

class TerritoryPipeline(object):

    def process_item(self, item, spider):
        if spider.name == "territory":
            # item is dict(name, code, territory_id)
            async def process_territory_stock():
                territory = Territory.objects.filter(pk=item['territory_id']).first()
                if territory:
                    Stock.objects.filter(code__endswith=item['code']).update(
                        territory=territory
                    )

            defer.ensureDeferred(process_territory_stock())
        else:
            return item

class ReportPipeline(object):
    """处理Report的下载"""

    def __init__(self):
        self.report_types = {}
        self.subjects = {}

    def process_item(self, item, spider):
        if spider.name == "report":
            defer.ensureDeferred(self.download_report(item))
        else:
            return item

    async def get_report_type(self, slug):
        """优先从缓存获取 report_type"""
        report_type = self.report_types.get(slug)
        if report_type is None:
            report_type = ReportType.objects.get(slug=slug)
            self.report_types[slug] = report_type

        return report_type

    async def get_subject(self, report_type, slug):
        """优先从缓存获取 subject"""
        key = "type_%s__%s" % (report_type.id, slug)
        subject = self.subjects.get(key)
        if subject is None:
            subject, _ = AccountingSubject.objects.get_or_create(
                report_type=report_type,
                slug=slug,
                defaults={
                    "memo": "由 spider 创建"
                }
            )
            self.subjects.update({key: subject})

        return subject

    async def download_report(self, item):
        report_model = Report if item['is_single_quarter'] else XReport
        report_type = await self.get_report_type(item['report_type'])
        report, created = await self.get_or_create_report(
            report_model,
            stock_id=item['stock_id'],
            report_type=report_type,
            year=item['report_year'],
            quarter=item['report_quarter'],
            defaults={
                'report_date': item['report_date'],
                'name': item['report_name'],
            }
        )

        await self.download_report_items(report, created, item)

    async def download_report_items(self, report, created, item):
        if item['is_single_quarter']:
            report_item_model = DynamicModel(ReportItem, item['report_year'])
        else:
            report_item_model = DynamicModel(XReportItem, item['report_year'])

        async def has_items():
            return report_item_model.objects.filter(report=report).exists()

        if created or not await has_items():
            items_to_insert = list()
            for slug, value in item['report_data'].items():
                subject = await self.get_subject(report.report_type, slug)

                if isinstance(value, list):
                    value = value[0]

                if isinstance(value, int) or isinstance(value, float):
                    value_type = ReportItem.NUMBER_TYPE
                else:
                    value_type = ReportItem.STRING_TYPE

                items_to_insert.append(report_item_model(
                    report=report,
                    subject=subject,
                    value_number=value if value_type == ReportItem.NUMBER_TYPE else None,
                    value=value if value_type != ReportItem.NUMBER_TYPE else None,
                    value_type=value_type
                ))

            await self.bulk_create_items(report_item_model, items_to_insert)

        elif item['crawl_mode'] == 'all':
            for slug, value in item['report_data'].items():
                subject = await self.get_subject(report.report_type, slug)

                if isinstance(value, list):
                    value = value[0]

                if isinstance(value, int) or isinstance(value, float):
                    value_type = ReportItem.NUMBER_TYPE
                else:
                    value_type = ReportItem.STRING_TYPE

                await self.update_or_create_item(
                    report_item_model,
                    report=report,
                    subject=subject,
                    defaults={
                        "value_number": value if value_type == ReportItem.NUMBER_TYPE else None,
                        "value": value if value_type != ReportItem.NUMBER_TYPE else None,
                        "value_type": value_type,
                    },
                )

    async def get_or_create_report(self, report_model, *args, **kwargs):
        """检查是否已经下载, 去重"""
        return report_model.objects.get_or_create(*args, **kwargs)

    async def update_or_create_item(self, item_model, *args, **kwargs):
        """异步更新报表项"""
        return item_model.objects.update_or_create(*args, **kwargs)

    async def bulk_create_items(self, item_model, items_to_insert):
        """批量插入"""
        return item_model.objects.bulk_create(items_to_insert)


from pulp import *
from coptpy import *

#用scipy求解线性规划问题
from scipy.optimize import linprog

c = [-1,2,3]
A = [[-2,1,1],[3,-1,-2]]
b = [[9],[-4]]
Aeq = [[4,-2,-1]]
beq = [-6]
LB = [-10,0,None]
UB = [None]*len(c)
bound = tuple(zip(LB,UB))
print(bound)
res = linprog(c,A,b,Aeq,beq,bound)
print('目标函数最小值：',res.fun)
print('最优解为：',res.x)

def getresult(c, con):
# 设置对象
    prob = LpProblem('myPro', LpMinimize)
# 设置三个变量，并设置变量最小取值
    x1 = LpVariable("x1", lowBound=0)
    x2 = LpVariable("x2", lowBound=0)
    x3 = LpVariable("x3", lowBound=0)
    X = [x1, x2, x3]
# 目标函数
    z = 0
    for i in range(len(X)):
        z += X[i]*c[i]
    prob += z
# 载入约束变量
    prob += x1-2*x2+x3 <= con[0]# 约束条件1
    prob += -4*x1+x2+2*x3 >= con[1]
    prob += -2*x1+x3 == con[2]
    print('确认线性规划是否输入正确：',prob)
# 求解
    status = prob.solve()
    print('\naim ending is ',value(prob.objective))  # 计算结果
    for i in prob.variables():
        print('\nanswer is ',i.varValue)
if __name__ == '__main__':
    c = [-3,1,1]
    con = [11,3,1]
    getresult(c,con)


import math
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout
from keras import regularizers, optimizers

def predict_close_lstm(df, cur_day, num_of_days, window_size=10, look_back=4, epochs=100, plot=False, plot_callback=None, loss_threshold=1e-4):
    actuals = []
    predictions = []

    start = cur_day
    end = cur_day + num_of_days
    # Go through all requested days and make a new training for every day
    for i in range(start, end):
        print("Training day {}".format(i - start))
        # For each day, train a new model using Close values of previous days.
        # Number of previous days is defined by "window_size"
        samples_start = i - window_size
        samples_end = i - 1

        # Important!
        # The training set will be taken from [samples_start:samples_end + 1]
        # and the last sample is for the prediction of next day
        df_cur_dataset = df[samples_start:samples_end + 2]

        actual, predicted = train_lstm(df_cur_dataset, look_back, epochs, plot=plot, plot_callback=plot_callback, loss_threshold=loss_threshold)
        actuals.append(actual)
        predictions.append(predicted)

    # Build a DataFrame that includes the actual and predicted results on the given data set
    df_results = df.iloc[cur_day:cur_day + num_of_days]
    cols_pred = pd.Series(predictions, index=range(cur_day, cur_day + num_of_days))
    df_results["Prediction"] = pd.Series(cols_pred)

    return actuals, predictions, df_results

def train_lstm(df, look_back, epochs, plot=False, plot_callback=None, loss_threshold=10**-4):
    # Set multi cores and print cores info
    # tf_session = tf.Session(config=tf.ConfigProto(device_count={"CPU": 8}, intra_op_parallelism_threads=2, inter_op_parallelism_threads=2))
    # keras.backend.tensorflow_backend.set_session(tf_session)
    # print(tf_session)

    # Normalize the Close dataset (this will include Xs and Ys)
    scaler = MinMaxScaler(feature_range=(0, 1))
    close_vals = np.reshape(df["Close"].values, (len(df), 1))
    dataset = scaler.fit_transform(close_vals.astype('float32'))

    # reshape into X=t and Y=t+1
    train_x, train_y, x_inc_predict, y_inc_predict = create_lstm_dataset(dataset, look_back)

    # reshape input to be [samples, time steps, features]
    train_x = np.reshape(train_x, (train_x.shape[0], train_x.shape[1], 1))
    x_inc_predict = np.reshape(x_inc_predict, (x_inc_predict.shape[0], x_inc_predict.shape[1], 1))

    # Create the LSTM network
    batch_size = 1
    model = create_model(batch_size, look_back)

    # Fit the network - use multiple epochs
    for i in range(epochs):
        print("epoch {}".format(i + 1))
        res = model.fit(train_x, train_y, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
        model.reset_states()
        if res.history["loss"][0] < loss_threshold:
            print("Loss is lower than threshold of {}. Quitting.".format(loss_threshold))
            break

    # make predictions
    train_predict = model.predict(x_inc_predict, batch_size=batch_size)
    model.reset_states()

    # invert predictions
    train_y = scaler.inverse_transform([train_y])
    train_predict = scaler.inverse_transform(train_predict)

    # calculate root mean squared error
    trainScore = math.sqrt(mean_squared_error(train_y[0], train_predict[:-1, 0]))
    print('Train Score RMSE: %.2f' % (trainScore))

    # shift train predictions for plotting
    train_predict_plot = np.empty_like(dataset)
    train_predict_plot[:, :] = np.nan
    train_predict_plot[look_back:len(train_predict) + look_back, :] = train_predict

    if plot_callback:
        plot_callback(scaler.inverse_transform(dataset), train_predict_plot)

    if plot:
        # plot baseline and predictions
        plt.figure(figsize=(15, 10))
        plt.plot(scaler.inverse_transform(dataset))
        plt.plot(train_predict_plot)
        plt.show()

    actual = df.iloc[-1]["Close"]
    predicted = train_predict[-1, 0]

    return actual, predicted

def create_model(batch_size, look_back, regularization_factor=0, dropout=False, custom_optimizer=True):
    model = Sequential()
    model.add(LSTM(4, batch_input_shape=(batch_size, look_back, 1), stateful=True, return_sequences=True))
    if dropout:
        model.add(Dropout(0.5))
    model.add(LSTM(4, batch_input_shape=(batch_size, look_back, 1), stateful=True))
    if dropout:
        model.add(Dropout(0.5))
    model.add(Dense(1, kernel_regularizer=regularizers.l2(regularization_factor)))

    if custom_optimizer:
        learning_rate = 0.005
        optimzer = optimizers.RMSprop(lr=learning_rate)
    else:
        optimzer = 'adam'

    model.compile(loss='mean_squared_error', optimizer=optimzer)

    return model

def create_lstm_dataset(dataset, look_back=1):
    # Actual training set
    data_x, data_y = [], []

    # X and Y sets that include the last sample (which is not part of the training set)
    # This will be used for predicting and ploting the next day
    data_inc_predict_x, data_inc_predict_y = [], []

    def add_i_features(ds_x, ds_y, i):
        # Append to the list of timeseries, a 1 dimensional array that
        # includes values of our single feature (index 0) across current look_back range
        ds_x.append(dataset[i:(i + look_back), 0])

        # Append to the list of Ys, the y of the current look_back, which is our feature
        # of next day after our look_back range
        ds_y.append(dataset[i + look_back, 0])

    # Go through all training samples
    # For example
    # if the windows size is 10
    # 11 samples will be passed
    # samples 0-9 will be used for the training set
    # sample 10 will be used for the prediction
    for i in range(len(dataset) - look_back - 1):
        add_i_features(data_x, data_y, i)
        add_i_features(data_inc_predict_x, data_inc_predict_y, i)

    # Add the last sample for prediction purposes
    add_i_features(data_inc_predict_x, data_inc_predict_y, i + 1)

    return np.array(data_x), np.array(data_y), np.array(data_inc_predict_x), np.array(data_inc_predict_y)

# If we don't have a fixed seed, every time that I train the model, it predicts different results.
# Probably we should make enough epochs in order to get to a very small loss in order to get deterministic results
np.random.seed(7)

# Read data
df = pd.read_csv('~/code/dscience/stocks/data/facebook.csv', engine='python', skipfooter=3)
df.loc[:, 'Date'] = pd.to_datetime(df['Date'], format='%Y-%m-%d')

##############################3

# # A callback to save plot data from within the training function in order to be able to plot the data in a different cell in Jupyter.
# ds1 = None
# ds2 = None
# def save_plot_data(original_dataset, predicted_shifted_dataset):
#     global ds1
#     global ds2
#     ds1 = original_dataset
#     ds2 = predicted_shifted_dataset
# numebr_of_days = 1
# first_day = int(len(df) / 2)
# res_actuals, res_predicted, df_results = predict_close_lstm(df, first_day, numebr_of_days, window_size=10, look_back=4, epochs=600, plot=False, plot_callback=save_plot_data, loss_threshold=1e-3)
# root_mse = math.sqrt(mean_squared_error(res_actuals, res_predicted))
# print('Test Score RMSE: %.2f' % (root_mse))
#
# # plot baseline and predictions
# plt.figure(figsize=(15, 10))
# plt.plot(ds1)
# plt.plot(ds2)
# plt.show()

##############################3

numebr_of_days = 100
first_day = len(df) - 100  # Same start date that we used for the polynomial regression
res_actuals, res_predicted, df_results = predict_close_lstm(df, first_day, numebr_of_days, window_size=10, look_back=4, epochs=600, plot=False,
                                                            loss_threshold=1e-3)
root_mse = math.sqrt(mean_squared_error(res_actuals, res_predicted))
print('TestScore RMSE: %.2f' % (root_mse))

plt.figure(figsize=(15,10))
plt.plot(df_results[["Date"]], df_results[["Close"]], 'b-')
plt.plot(df_results[["Date"]], df_results[["Prediction"]], 'g-')
#plt.title("Prediction of 8th day ({})".format(predicted_date.date()))
plt.xlabel('Date')
plt.ylabel('Price [$]')
plt.legend(["Actual Close", "Predicted Close"])
plt.show()

# ax = df_results.plot(x='Date', y='Close', style='b-', grid=True)
# ax = df_results.plot(x='Date', y='Prediction', style='g-', grid=True, ax=ax)
# ax.set_xlabel("Date")
# ax.set_ylabel("Price [$]")
# ax.legend(["Actual Close", "Predicted Close"])

##############################3


# encoding: utf-8

"""
General Pyparsing classes used to parse pyloggr configuration files
"""

__author__ = 'stef'

import ast
import re
import sys
from itertools import product
from copy import copy

from pyparsing import Keyword, Suppress, Literal, quotedString, Word, delimitedList, Group, pythonStyleComment
from pyparsing import alphanums, alphas, operatorPrecedence, opAssoc, nums
from future.builtins import str as text
from .fix_unicode import to_unicode

# noinspection PyUnusedLocal
def _make_plain_field(s, loc, toks):
    return PlainField(toks[0])

def _compile_condition(toks):
    toks = toks[0]
    if isinstance(toks, Condition):
        cond = toks
    else:
        cond = Condition.factory(toks[1], toks[0], toks[2])
    cond.gen_lambda()
    return cond

thismodule = sys.modules[__name__]
field_names = [
    "severity", "facility", "app_name", "source", "programname", "syslogtag", "message", "uuid",
    "timereported", "timegenerated", "trusted_comm", "trusted_exe", "trusted_cmdline"
]

for field_name in field_names:
    setattr(thismodule, field_name, Keyword(field_name).setParseAction(
        _make_plain_field
    ))

class BuiltString(object):
    """
    Complex built string
    """
    def __init__(self, part_strings):
        self.part_strings = part_strings

    def _str(self):
        return "BString({})".format('+'.join(str(part) for part in self.part_strings))

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    def apply(self, ev):
        """
        Return the string value for the given event

        :param ev: event
        """
        strings = map(lambda s: s.apply(ev), self.part_strings)
        # eliminate empty strings
        strings = filter(lambda s: bool(s), strings)
        # some of the calculated part_strings may actually be sets, because of structured data multiple values
        copy_of_strings = map(lambda s: {s} if not isinstance(s, set) else s, strings)
        combinators = product(*copy_of_strings)
        return [''.join(combi) for combi in combinators]

    @classmethod
    def from_tokens(cls, toks):
        return BuiltString(toks)

def _make_built_string(toks):
    return BuiltString(toks[0])

class Constant(object):
    """
    Represents a constant value
    """
    def __init__(self, name):
        self.name = to_unicode(name)

    def _str(self):
        return "C({})".format(self.name)

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    # noinspection PyUnusedLocal
    def apply(self, ev):
        """
        Return the value of constant

        :param ev: event
        """
        return self.name

    def gen_ast(self):
        """
        Generate the AST for the constant value
        """
        return ast.Str(self.name)

# noinspection PyUnusedLocal
def _quoted_string_to_constant(s, loc, toks):
    return Constant(ast.literal_eval(toks[0]))

class Field(object):
    """
    Abstract field
    """
    def __init__(self, name):
        self.name = to_unicode(name)

    def _str(self):
        return "F({})".format(self.name)

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    def apply(self, ev):
        """
        Return the value of the field for the given event

        :param ev: event
        """
        raise NotImplementedError

    def gen_ast(self):
        """
        Generate the abstract syntax tree corresponding to the field access
        """
        raise NotImplementedError

class CustomField(Field):
    """
    Represents access to a custom event field
    """
    def __init__(self, name):
        Field.__init__(self, name)
        self.name = to_unicode(self.name.name if isinstance(self.name, Constant) else self.name)

    def _str(self):
        return "EField({})".format(self.name)

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    def apply(self, ev):
        """
        Return the value of the custom field for the given event

        :param ev: event
        """
        return ev[self.name]

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the custom field access
        """
        return ast.Subscript(value=ast.Name(id='ev', ctx=ast.Load()), slice=ast.Index(value=ast.Str(s=self.name)))

class PlainField(Field):
    """
    Represents access to a plain event field
    """
    def _str(self):
        return "PField({})".format(self.name)

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    def apply(self, ev):
        """
        Return the field value for the given event

        :param ev: event
        """
        return ev.__getattribute__(self.name)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the field access
        """
        return ast.Call(
            func=ast.Name(id='getattr', ctx=ast.Load()),
            args=[ast.Name(id='ev', ctx=ast.Load()), ast.Str(s=self.name)],
            keywords=[],
            starargs=None,
            kwargs=None
        )

class Condition(object):
    """
    Abstract condition
    """
    def __init__(self):
        self.fun = None

    @classmethod
    def factory(cls, operande, left, right):
        """
        Factory method to make the concrete conditions

        :param operande: 'and' or 'or' or 'not'
        :param left: left part of condition
        :param right: right part of condition (can be None)
        :return: concrete condition
        :rtype: Condition
        """
        if operande == "and":
            return AndCondition(left, right)
        elif operande == "or":
            return OrCondition(left, right)
        elif operande == "not":
            return NotCondition(left)

    def _str(self):
        raise NotImplementedError

    def __str__(self):
        return self._str()

    def __repr__(self):
        return self._str()

    def apply(self, ev):
        """
        Return the condition value for the given event

        :param ev: event
        """
        raise NotImplementedError

    def gen_ast(self):
        """
        Generate the abstract syntax tree corresponding to the condition
        """
        raise NotImplementedError

    def gen_lambda(self):
        """
        Compile the condition and store in self.fun
        """
        tree = ast.Expression(
            ast.Lambda(
                args=ast.arguments(
                    args=[ast.Name(id='ev', ctx=ast.Param())],
                    vararg=None, kwarg=None, defaults=[]
                ),
                body=self.gen_ast()
            )
        )
        compiled = compile(ast.fix_missing_locations(tree), '<string>', 'eval')
        self.fun = eval(compiled, {'re': re})

    def eval(self, ev):
        """
        Evaluate the condition using the compiled self.fun

        :param ev: event
        :return: bool
        """
        return self.fun(ev)

class AlwaysTrueCondition(Condition):
    def __init__(self):
        super(AlwaysTrueCondition, self).__init__()
        self.fun = lambda ev: True

    def apply(self, ev):
        return True

always_true_singleton = AlwaysTrueCondition()

class NotCondition(Condition):
    """
    NOT A condition
    """
    def __init__(self, left):
        super(NotCondition, self).__init__()
        self.operand = "not"
        self.left = left

    def _str(self):
        return "NOT({})".format(self.left)

    def apply(self, ev):
        """
        Test if the condition apply to the event

        :param ev: event
        :return: bool
        """
        return not self.left.apply(ev)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the condition
        """
        return ast.UnaryOp(op=ast.Not(), operand=self.left.gen_ast())

class AndCondition(Condition):
    """
    A AND B condition
    """
    def __init__(self, left, right):
        super(AndCondition, self).__init__()
        self.operand = "and"
        self.left = left
        self.right = right

    def _str(self):
        return "AND({}, {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Test if the condition apply to the event

        :param ev: event
        :return: bool
        """
        return self.left.apply(ev) and self.right.apply(ev)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the condition
        """
        return ast.BoolOp(op=ast.And(), values=[self.left.gen_ast(), self.right.gen_ast()])

class OrCondition(Condition):
    """
    A OR B condition
    """
    def __init__(self, left, right):
        super(OrCondition, self).__init__()
        self.operand = "or"
        self.left = left
        self.right = right

    def _str(self):
        return "OR({}, {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Test if the condition apply to the event

        :param ev: event
        :return: bool
        """
        return self.left.apply(ev) or self.right.apply(ev)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the condition
        """
        return ast.BoolOp(op=ast.Or(), values=[self.left.gen_ast(), self.right.gen_ast()])

class Predicate(Condition):
    """
    Abstract predicate
    """
    def __init__(self):
        super(Predicate, self).__init__()

    @classmethod
    def factory(cls, operation, left, right):
        """
        Factory method the make the concrete objects

        :param operation: type of predicate
        :param left: left part of predicate
        :param right: right part of predicate
        :return: concrete predicate
        :rtype: Predicate
        """
        if operation == '==':
            return Equals(left, right)
        elif operation == '!=':
            return Different(left, right)
        elif operation == 'in':
            return In(left, right)
        elif operation == 'notin':
            return Notin(left, right)
        elif operation == '~':
            return RegexpP(left, right)
        elif operation == '~*':
            return RegexpIP(left, right)

    def _str(self):
        raise NotImplementedError

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        raise NotImplementedError

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        raise NotImplementedError

class Equals(Predicate):
    """
    EQUALS predicate
    """
    def __init__(self, left, right):
        super(Equals, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "EQ({} == {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        return self.left.apply(ev) == self.right.apply(ev)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        return ast.Compare(left=self.left.gen_ast(), ops=[ast.Eq()], comparators=[self.right.gen_ast()])

class RegexpP(Predicate):
    """
    CASE SENSITIVE predicate
    """
    def __init__(self, left, right):
        super(RegexpP, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "REGEXP({} ~ {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        return re.search(self.right.apply(ev), self.left.apply(ev))

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        return ast.Call(
            func=ast.Attribute(
                value=ast.Name(id='re', ctx=ast.Load()),
                attr='search',
                ctx=ast.Load()
            ),
            args=[
                self.right.gen_ast(),
                self.left.gen_ast()
            ],
            keywords=[],
            starargs=None,
            kwargs=None
        )

class RegexpIP(Predicate):
    """
    CASE INSENSITIVE REGEXP predicate
    """
    def __init__(self, left, right):
        super(RegexpIP, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "REGEXP_I({} ~* {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        return re.search(self.right.apply(ev), self.left.apply(ev), flags=re.IGNORECASE)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        return ast.Call(
            func=ast.Attribute(
                value=ast.Name(id='re', ctx=ast.Load()),
                attr='search',
                ctx=ast.Load()
            ),
            args=[
                self.right.gen_ast(),
                self.left.gen_ast()
            ],
            keywords=[
                ast.keyword(
                    arg='flags',
                    value=ast.Attribute(
                        value=ast.Name(id='re', ctx=ast.Load()),
                        attr='IGNORECASE',
                        ctx=ast.Load()
                    )
                )
            ],
            starargs=None,
            kwargs=None
        )

class Different(Predicate):
    """
    DIFFERENT predicate
    """
    def __init__(self, left, right):
        super(Different, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "NEQ({} != {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        return self.left.apply(ev) != self.right.apply(ev)

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        return ast.Compare(left=self.left.gen_ast(), ops=[ast.NotEq()],
                           comparators=[self.right.gen_ast()])

class In(Predicate):
    """
    IN predicate
    """
    def __init__(self, left, right):
        super(In, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "IN({} in {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        left = self.left if isinstance(self.left, text) else self.left.apply(ev)
        if self.right == "tags":
            return left in ev.tags
        elif self.right == "custom_fields":
            return left in ev
        else:
            raise ValueError

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        left = ast.Str(self.left) if isinstance(self.left, text) else self.left.gen_ast()
        if self.right == "tags":
            return ast.Compare(left=left, ops=[ast.In()], comparators=[
                ast.Attribute(
                    value=ast.Name(id='ev', ctx=ast.Load()),
                    attr='tags',
                    ctx=ast.Load()
                )
            ])
        elif self.right == "custom_fields":
            return ast.Compare(left=left, ops=[ast.In()], comparators=[ast.Name(id='ev', ctx=ast.Load())])
        else:
            raise ValueError

class Notin(Predicate):
    """
    NOT IN predicate
    """
    def __init__(self, left, right):
        super(Notin, self).__init__()
        self.left = left
        self.right = right

    def _str(self):
        return "NOTIN({} notin {})".format(self.left, self.right)

    def apply(self, ev):
        """
        Apply the predicate to the given event

        :param ev: event
        :return: bool
        """
        left = self.left if isinstance(self.left, text) else self.left.apply(ev)
        if self.right == "tags":
            return left not in ev.tags
        elif self.right == "custom_fields":
            return left not in ev
        else:
            raise ValueError

    def gen_ast(self):
        """
        Generate the Abstract syntax tree corresponding to the predicate
        """
        left = ast.Str(self.left) if isinstance(self.left, text) else self.left.gen_ast()
        if self.right == "tags":
            return ast.Compare(left=left, ops=[ast.NotIn()], comparators=[
                ast.Attribute(
                    value=ast.Name(id='ev', ctx=ast.Load()),
                    attr='tags',
                    ctx=ast.Load()
                )
            ])
        elif self.right == "custom_fields":
            return ast.Compare(left=left, ops=[ast.NotIn], comparators=[ast.Name(id='ev', ctx=ast.Load())])
        else:
            raise ValueError

custom_fields   = Keyword("custom_fields")
tags            = Keyword("tags")

if_cond         = Suppress(Keyword("if"))
else_cond       = Suppress(Keyword("else"))

geoip           = Keyword("geoip")
grok            = Keyword("grok")
useragent       = Keyword("useragent")
router          = Keyword("router")

drop            = Keyword("drop")
stop            = Keyword("stop")

and_operand     = Keyword('and')
or_operand      = Keyword('or')
not_operand     = Keyword('not')

equals          = Literal('==')
assign          = Suppress(Literal(':='))
kw_arg_assign   = Suppress(Literal('=') | Literal(':'))
tags_assign     = Literal('+=') | Literal('-=')
different       = Literal('!=')
in_op           = Keyword('in')
notin_op        = Keyword('notin')
regexp_op       = Literal('~')
regexp_i_op     = Literal('~*')

open_par        = Literal('(')
open_acc        = Literal('{')
close_par       = Literal(')')
close_acc       = Literal('}')

open_delim      = Suppress(open_par | open_acc)
close_delim     = Suppress(close_par | close_acc)

integer         = Word(nums).setParseAction(lambda toks: int(toks[0]))

my_qs = quotedString.setParseAction(_quoted_string_to_constant)
label = Word(initChars=alphas, bodyChars=alphanums + "_")

# noinspection PyUnresolvedReferences
plain_field = severity | facility | app_name | source | programname | syslogtag | message | uuid | \
              timereported | timegenerated | trusted_comm | trusted_exe | trusted_cmdline

custom_field_name = my_qs | label
custom_field = (Literal('[') + custom_field_name + Literal(']')).setParseAction(
    lambda s, loc, toks: CustomField(toks[1])
)

filter_name = (geoip | grok | useragent | router)

comment_line = Suppress(pythonStyleComment)

equals_predicate = (
    (plain_field | custom_field | my_qs)
    + equals
    + (plain_field | custom_field | my_qs)
)

different_predicate = (
    (plain_field | custom_field | my_qs)
    + different
    + (plain_field | custom_field | my_qs)
)

in_predicate = (
    (my_qs | label)
    + in_op
    + (tags | custom_fields)
)

notin_predicate = (
    (my_qs | label)
    + notin_op
    + (tags | custom_fields)
)

regexp_predicate = (
    (plain_field | custom_field)
    + regexp_op
    + my_qs
)

regexpi_predicate = (
    (plain_field | custom_field)
    + regexp_i_op
    + my_qs
)

predicate = (equals_predicate | different_predicate | in_predicate | notin_predicate | regexp_predicate | regexpi_predicate).setParseAction(
    lambda s, loc, toks: Predicate.factory(toks[1], toks[0], toks[2])
)

condition = operatorPrecedence(
    predicate,
    [
        (not_operand, 1, opAssoc.RIGHT),
        (and_operand, 2, opAssoc.LEFT),
        (or_operand, 2, opAssoc.LEFT),
    ]
)
condition = condition.setParseAction(_compile_condition)
condition = condition.setResultsName('condition')

built_string = Group(delimitedList(expr=(plain_field | custom_field | my_qs), delim='+')).setParseAction(
    _make_built_string
)


from datetime import timedelta
from os import environ as os_environ

from celery.schedules import crontab

from .django import TIME_ZONE
from .redis import REDIS_URL

# Celery
CELERY_TIMEZONE = TIME_ZONE
CELERY_BROKER_URL = REDIS_URL
CELERY_RESULT_BACKEND = REDIS_URL
CELERY_WORKER_CONCURRENCY = os_environ.get("NEWARA_CELERY_CONCURRENCY")
CELERY_ACCEPT_CONTENT = ["json", "pickle"]  # datetime때문에 pickle 사용
CELERY_EVENT_SERIALIZER = "pickle"
CELERY_RESULT_SERIALIZER = "pickle"
CELERY_TASK_SERIALIZER = "pickle"
CELERY_TASK_RESULT_EXPIRES = int(os_environ.get("NEWARA_CELERY_EXPIRES", 600))

def create_scheduler_config(name, period=None, crontab=None):
    config = {"NAME": name, "PING_URL": os_environ.get(f"NEWARA_{name}_PING_URL", None)}
    if period is not None:
        config["PERIOD"] = timedelta(
            seconds=int(os_environ.get(f"NEWARA_{name}_PERIOD", period))
        )
    if crontab is not None:
        config["CRONTAB"] = crontab
    return config

SCHEDULERS = {
    "CRAWL_PORTAL": create_scheduler_config(
        "CRAWL_PORTAL", crontab=crontab(minute=0)
    ),  # 매 0분 (1시간마다)
    "SAVE_DAILY_BEST": create_scheduler_config(
        "SAVE_DAILY_BEST", crontab=crontab(minute=0)
    ),
    "SAVE_WEEKLY_BEST": create_scheduler_config(
        "SAVE_WEEKLY_BEST", crontab=crontab(minute=0)
    ),
    "SEND_EMAIL_FOR_REPLY_REMINDER": create_scheduler_config(
        "SEND_EMAIL_FOR_REPLY_REMINDER", crontab=crontab(hour=7, minute=0)
    ),  # 매일 오전 7시
}


import os

from aiohttp.web import middleware, HTTPUnauthorized

class Authentiction:
    @staticmethod
    @middleware
    async def handle(request, handler):
        _token = os.environ.get('AUTH_TOKEN', False)
        if _token:
            try:
                scheme, token = request.headers['Authorization'].strip().split(' ')
            except KeyError or ValueError:
                raise HTTPUnauthorized()

            if scheme.lower() != 'bearer' or token != _token:
                raise HTTPUnauthorized()

        return await handler(request)


"""
bit2img.py
@author Echo
@date 2022-05-13 13:51
@description 单独将txt中的比特流恢复成图片，若恢复的图片不能打开，则转为相应尺寸的噪声图

"""

import glob
import os

import numpy as np
from torchvision.transforms import ToPILImage
import torch
from PIL import Image

def img_to_bit(input_path, output_path):
    """
    将输入路径下的所有图片转为二进制比特流，并保存为txt
    :param input_path: 输入图片路径
    :param output_path: 输出txt路径
    :return: None
    """
    file = open(input_path, 'rb')  # 输入bpg压缩后的文件
    file_context = file.read()  # <class 'bytes'>字节流

    tmp_a = []
    bit_all = ''
    for i in file_context:
        tmp_a.append(i)  # int类型的数据
    tmp_b = np.array(tmp_a, dtype=np.uint8)
    for j in tmp_b:
        k = bin(j).replace('0b', '').rjust(8, '0')
        bit_all = bit_all + k
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(bit_all)
        f.close()

def cut_string(obj, sec):
    """
    切割字符串
    :param obj: 输入字符串
    :param sec: 切割的位数
    :return: 切割后的字符串
    """
    return [obj[i:i + sec] for i in range(0, len(obj), sec)]

def get_bitstring(input_path):
    """
    获得比特流
    :param input_path: 图片对应比特流存储的txt路径
    :return: 返回比特流字符串
    """
    with open(input_path, 'r') as f:
        f_context = f.read().strip()  # 读取字符串
        k_char = cut_string(f_context, 1)  # 字符串按8切割
        # int(a, 2)表示将二进制的字符串a表示为十进制的int
        k = [int(a, 2) for a in k_char]  # 字符串转换为int类型的数据
        bit_array = np.array(k)

        bitstring = ''
        for i in bit_array:
            bitstring += str(i)
        return bitstring

def random_noise(nc, width, height):
    """Generator a random noise image from tensor.

    If nc is 1, the Grayscale image will be created.
    If nc is 3, the RGB image will be generated.

    Args:
        nc (int): (1 or 3) number of channels.
        width (int): width of output image.
        height (int): height of output image.
    Returns:
        PIL Image.
    """
    img = torch.rand(nc, width, height)
    img = ToPILImage()(img)
    return img

def bit_to_img(string, img_dir, output_path):
    """
    将比特流字符串重新转换为图片，若转换后的图片无法打开则将之变为一副对应尺寸的随机噪声图
    :param img_dir: 输入图片路径
    :param string: 图片比特流字符串
    :param output_path: 输出图片文件夹路径
    :return: None
    """
    split_char = cut_string(string, 8)  # 字符串按8切割
    # int(a, 2)表示将二进制的字符串a表示为十进制的int
    int_8 = [int(a, 2) for a in split_char]  # 字符串转换为int类型的数据
    out_stream = np.array(int_8, dtype=np.uint8)
    # print(out_stream)
    # print(out_stream.size)
    directory = output_path + '/' + os.path.basename(os.path.dirname(img_dir))
    if not os.path.exists(directory):
        os.makedirs(directory)
    img_output_path = directory + '/' + os.path.basename(img_dir)
    out_stream.tofile(img_output_path)

    try:
        Image.open(img_output_path).convert('RGB')
    except IOError:
        print('Error')
        width = Image.open(img_dir).width
        height = Image.open(img_dir).height
        random_noise(3, width, height).save(img_output_path)

if __name__ == '__main__':

    input_txts = glob.glob(os.path.join('/Users/serendipity/Downloads/stl_original_bit', '**/*.txt'), recursive=True)

    total = len(input_txts)
    for txt_dir in input_txts:
        img_bitstring = get_bitstring(txt_dir)
        output_path = '/Users/serendipity/Downloads/WebP/voc_webp_0.25_bit_ldpc_snr_11dB_2img'
        # 需要找到txt对应原图
        base_dir = '/Users/serendipity/Downloads/WebP/voc_webp_0.25'
        img_dir = base_dir + '/' + txt_dir.split('/')[-2] + '/' + txt_dir.split('/')[-1].split('.')[0] + '.webp'
        bit_to_img(img_bitstring, img_dir, output_path=output_path)
        total = total - 1
        print(total)


import csv
import argparse
import os
import numpy as np
import gensim.models.word2vec as wv
import json
import subprocess
import re
import math

from copy import deepcopy
from Naked.toolshed.shell import muterun_js, execute_js
from scipy import spatial
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import wait

vuln_line_vectors = list()
acceptance = list()
model = None
index2word_set = set()
vuln_lines = list()
BASEDIR = os.getcwd()

def get_dev_block_values(start, end):
    length = (end-start)+1
    results = [0] * length
    if length % 2:
        center = math.floor(length/2)
        results[center] = length
        for i in range(1, center+1):
            results[center+i] = results[center-i] = length - i
    else:
        center = int(length/2)
        for i in range(0, center):
            results[center+i] = results[center-(i+1)] = length - i
    return results

# create a vector that's as long as the amount of features we have(may be an issue)
# if a word is in our model add it to the sentece's feature vector
# get the average
def avg_feature_vector(sentence, model, num_features, index2word_set):
    words = sentence.split()
    feature_vec = np.zeros((num_features, ), dtype='float32')
    n_words = 0
    for word in words:
        if word in index2word_set:
            n_words += 1
            feature_vec = np.add(feature_vec, model[word])
    if (n_words > 0):
        feature_vec = np.divide(feature_vec, n_words)
    return feature_vec

# get the lines minimum distance from one of the already stored vulnerable lines
# return a tuple of the vuln value and the line that's closest
def get_line_vuln(line, model, features, index2word_set):
    mini = 1
    reason = ''
    line_vec = avg_feature_vector(line, model, features, index2word_set)
    for vec, line in zip(vuln_line_vectors, vuln_lines):
        curr = spatial.distance.cosine(line_vec, vec)
        if curr < mini:
            mini = curr
            reason = line
    return (mini,reason)

# runs a js subscript that extracts the containing function
def get_container_func(line_data, path):
    return muterun_js(f'./js_subscripts/method-dispenser/index.js {path[2:]} {line_data["line_num"]}')

# predict wether a line is vulnerable or not
def predict(target, keywords, model, features, index2word_set, acceptance, file_name=None, modelName=None, rules=list(), count = -1):
    line_results = list()

    # if the file doesn't exist fail here
    with open(target, 'r'):
        pass

    # lex the file with the current model's corresponding lexer
    execute_js(f'./js_subscripts/lexfile/{(modelName+"_" if modelName else "IDname_")}lexfile.js {target} {keywords}')
    # calc all vuln line vectors
    # create a list of dict's where each dict contains the line, the chance of it being vulnerable, the reason for that given chance, and the lines number
    with open(target, 'r', errors='ignore') as fp:
        with open('./lexed.txt','r') as fp2:
            line_num = 0
            for line, lexed_line in zip(fp.readlines(),fp2.readlines()):
                (line_val,reason) = get_line_vuln(lexed_line, model, features, index2word_set)
                line_results.append(
                    {
                        'line_num' : line_num,
                        'vuln_prob' : 1 - line_val,
                        'line' : line,
                        'reason' : reason,
                        'distance': 1 - line_val,
                        'surrounding': 0,
                        'complexity': 0,
                        'git_surr' : 0,
                        'model': modelName
                    }
                )
                line_num += 1
    if file_name:
        os.chdir(f'./{modelName}_results'+('' if count == -1 else '_'+str(count))+f'/{file_name}')
    # perform the check for every specified acceptance value
    with ThreadPoolExecutor(max_workers=os.cpu_count() + 4) as executor:
        accept_results = dict()
        for accept in acceptance:
            accept_results[accept] = executor.submit(execute_rules, accept, line_results, rules, target)

        for accept in acceptance:
            # any rules that change the cwd are executed here
            # they are only executed if the base rule list contains them
            if 'rule_check_change_blocks' in accept_results[accept].result().keys():
                accept_results[accept].result()['check_change_blocks'] = [line for line in check_change_blocks(line_results, accept, target) if line['vuln_prob'] > accept]

        for accept in accept_results:
            if file_name:
                if not os.path.isdir(f'{int(accept*100)}%'):
                        os.mkdir(f'{int(accept*100)}%')

                os.chdir(f'{int(accept*100)}%')
            vuln_lines_processing_futures = list()
            for rule_name in accept_results[accept].result():
                vuln_lines_processing_futures.append(executor.submit(process_vuln_lines, accept_results[accept].result()[rule_name], rule_name))
            #this is needed so the vuln_line processing tasks finish while the cwd is in the desired location
            wait(vuln_lines_processing_futures)
            os.chdir('../')
        os.chdir(BASEDIR)

def execute_rules(accept, line_results, rules, target):
    rule_results = dict()

    # create a new list of vulnerable lines
    vuln_lines = [line for line in line_results if line['vuln_prob'] > accept]
    # before any rules are applied create output for the base results
    rule_results['no_rule'] = deepcopy(vuln_lines)
    # apply each specified rule
    for rule in rules:
        # this needs to be extended for every rule that changes cwd during it's runtime
        if rule.__name__ is not 'check_change_blocks':
            vuln_lines = [line for line in rule(line_results, accept, target) if line['vuln_prob'] > accept]
        rule_results[f'rule_{rule.__name__}'] = deepcopy(vuln_lines)

    return rule_results

# create the output cve file
def process_vuln_lines(vuln_lines, file_name):
    func_results = list()

    '''for vuln_line in vuln_lines:
        container = get_container_func(vuln_line, target).stdout.strip()
        if container not in func_results:
            func_results.append(container)'''

    with open(('sus.csv' if not file_name else f'{file_name}.csv'),'w', newline='') as fp:
        writer = csv.DictWriter(fp, fieldnames = ['num','prob','line','reason', 'distance', 'surrounding', 'complexity', 'git_surr'])
        writer.writerow({
                        'num' : 'Line_number',
                        'prob' : 'Chance_of_vuln',
                        'line' : 'Line_content',
                        'reason' : 'Reason_for_pred',
                        'distance': 'distance',
                        'surrounding': 'surrounding',
                        'complexity': 'complexity',
                        'git_surr' : 'git_surr'
                    })
        for line in vuln_lines:
            writer.writerow({
                        'num' : line['line_num'],
                        'prob' : line['vuln_prob'],
                        'line' : line['line'],
                        'reason' : line['reason'],
                        'distance': line['distance'],
                        'surrounding': line['surrounding'],
                        'complexity': line['complexity'],
                        'git_surr' : line['git_surr']
                    })
    print('CSV done')

# reads in all the given vulnerable lines
def readVulnLines(model, features, index2word_set, file_name='vuln_lines_new_lex.txt'):
    with open(file_name, 'r') as fp:
        for line in fp.readlines():
            vuln_lines.append(line)
            vuln_line_vectors.append(avg_feature_vector(line, model, features, index2word_set))

# this function servers to psuh values back to between 0-1
def weight_func(x, n=0):
    return 1 - ((1 + n)/(x + n))

# filters out all lines that only contain 1 word, this is applied permanently, and so it needs to be the first rule
def no_one_word_lines(line_data, acceptance, target):
    for line in line_data:
        if len(line['line'].split()) < 2:
            line['vuln_prob'] = 0
    return line_data
# checks how many vulnerable lines there are in the same vuln_block as the line given
def check_surroundings(line_data, acceptance, target):
    result = deepcopy(line_data)

    for (i, line), data in zip(enumerate(result), line_data):
        neighbourCount = 1
        if data['vuln_prob'] > acceptance:
            for j in range(i-1,0,-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            for j in range(i+1, len(line_data)-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break

            line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
            line['surrounding'] = neighbourCount

    return result

# prefer more complex lines
def prefer_complex(line_data, acceptance, target):
    result = deepcopy(line_data)

    lexed_lines = list()
    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()
    for line in result:

        if(len(set(lexed_lines[line['line_num']].split()))):
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(len(set(lexed_lines[line['line_num']].split())))) / 2
            line['complexity'] = len(set(lexed_lines[line['line_num']].split()))

    return result

# combine prefer_complex and check_surroundings, here the complexity preference is applied after the vuln_blocks are calculated
def complex_and_surrounded(line_data, acceptance, target):
    result = deepcopy(line_data)

    lexed_lines = list()
    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()

    for (i, line), data in zip(enumerate(result),line_data):
        neighbourCount = 1
        if data['vuln_prob'] > acceptance:
            for j in range(i-1,0,-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            for j in range(i+1, len(line_data)-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            if(len(set(lexed_lines[line['line_num']].split()))):
                line['vuln_prob'] = (line['vuln_prob'] + weight_func(len(set(lexed_lines[line['line_num']].split()))) + weight_func(neighbourCount)) / 3
                line['complexity'] = len(set(lexed_lines[line['line_num']].split()))
                line['surrounding'] = neighbourCount

    return result

# apply the check surroundings rule multiple times, to see if it converges
def iter_surrounding(line_data, acceptance, target):
    result = deepcopy(line_data)

    for (i, line), data in zip(enumerate(result), line_data):
        neighbourCount = 1
        if data['vuln_prob'] > acceptance:
            for j in range(i-1,0,-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            for j in range(i+1, len(line_data)-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break

            line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
            line['surrounding'] = (line['surrounding'] + neighbourCount) / 2

    for _ in range(0,100):
        prev_state = deepcopy(result)
        for (i, line), data in zip(enumerate(result), prev_state):
            neighbourCount = 1
            if data['vuln_prob'] > acceptance:
                for j in range(i-1,0,-1):
                    if line_data[j]['vuln_prob'] > acceptance:
                        neighbourCount += 1
                    else:
                        break
                for j in range(i+1, len(line_data)-1):
                    if line_data[j]['vuln_prob'] > acceptance:
                        neighbourCount += 1
                    else:
                        break

                line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
                line['surrounding'] = neighbourCount

    return result

# combine prefer_complex and check_surroundings, apply the complex preference before vuln_blocks are calculated
def complex_before_surr(line_data, acceptance, target):
    result = deepcopy(line_data)

    lexed_lines = list()
    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()

    for line in result:
        if(len(set(lexed_lines[line['line_num']].split()))):
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(len(set(lexed_lines[line['line_num']].split())))) / 2
            line['complexity'] = len(set(lexed_lines[line['line_num']].split()))

    for (i, line), data in zip(enumerate(result),line_data):
        neighbourCount = 1
        if data['vuln_prob'] > acceptance:
            for j in range(i-1,0,-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            for j in range(i+1, len(line_data)-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
            line['surrounding'] = neighbourCount

    return result

def complex_before_surr_reg(line_data, acceptance, target):
    result = deepcopy(line_data)

    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()

    with open(BASEDIR + '/coeffs.json', 'r') as fp:
        coeffs = json.load(fp)

    model_name = ''
    for line in result:
        if (len(set(lexed_lines[line['line_num']].split()))):
            model_name = line['model']
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(len(set(lexed_lines[line['line_num']].split())))) / 2
            line['complexity'] = len(set(lexed_lines[line['line_num']].split()))

    selected_values = ''
    for key in coeffs.keys():
        if str(key).startswith(model_name) and str(key).endswith('before_surr.csv'):
            selected_values = coeffs[key]
            break

    parts = selected_values.split('+')
    constant = float(parts[0].strip())
    coeff_distance = float(parts[1].split('*')[0].strip())
    coeff_surrounding = float(parts[2].split('*')[0].strip())
    coeff_complexity = float(parts[3].split('*')[0].strip())

    for (i, line), data in zip(enumerate(result), line_data):
        neighbourCount = 1
        if data['vuln_prob'] > 0:
            for j in range(i - 1, 0, -1):
                if line_data[j]['vuln_prob'] > 0:
                    neighbourCount += 1
                else:
                    break
            for j in range(i + 1, len(line_data) - 1):
                if line_data[j]['vuln_prob'] > 0:
                    neighbourCount += 1
                else:
                    break
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
            line['surrounding'] = neighbourCount

    for line in result:
        if constant + coeff_distance * line['distance'] + coeff_surrounding * line['surrounding'] + coeff_complexity * line['complexity'] > acceptance:
            line['vuln_prob'] = 1
        else:
            line['vuln_prob'] = 0

    return result

def complex_and_surrounded_reg(line_data, acceptance, target):
    result = deepcopy(line_data)

    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()

    with open(BASEDIR + '/coeffs.json', 'r') as fp:
        coeffs = json.load(fp)

    model_name = ''
    for line in result:
        model_name = line['model']
        break

    selected_values = ''
    for key in coeffs.keys():
        if str(key).startswith(model_name) and str(key).endswith('surrounded.csv'):
            selected_values = coeffs[key]
            break

    parts = selected_values.split('+')
    constant = float(parts[0].strip())
    coeff_distance = float(parts[1].split('*')[0].strip())
    coeff_surrounding = float(parts[2].split('*')[0].strip())
    coeff_complexity = float(parts[3].split('*')[0].strip())

    for (i, line), data in zip(enumerate(result), line_data):
        neighbourCount = 1
        if data['vuln_prob'] > 0:
            for j in range(i - 1, 0, -1):
                if line_data[j]['vuln_prob'] > 0:
                    neighbourCount += 1
                else:
                    break
            for j in range(i + 1, len(line_data) - 1):
                if line_data[j]['vuln_prob'] > 0:
                    neighbourCount += 1
                else:
                    break
            if (len(set(lexed_lines[line['line_num']].split()))):
                line['vuln_prob'] = (line['vuln_prob'] + weight_func(
                    len(set(lexed_lines[line['line_num']].split()))) + weight_func(neighbourCount)) / 3
                line['complexity'] = len(set(lexed_lines[line['line_num']].split()))
                line['surrounding'] = neighbourCount

    for line in result:
        if constant + coeff_distance * line['distance'] + coeff_surrounding * line['surrounding'] + coeff_complexity * line['complexity'] > acceptance:
            line['vuln_prob'] = 1
        else:
            line['vuln_prob'] = 0

    return result

def check_change_blocks(line_data, acceptance, target):
    result = deepcopy(line_data)

    current_dir = os.getcwd()

    os.chdir(BASEDIR)

    path = '/'.join(target.split('/')[:-1])

    filename = target.split('/')[-1]

    os.chdir(path)

    lines  = subprocess.getoutput(f'git blame {filename}').split('\n')
    names = list()

    for line in lines:
        pattern = re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}')
        line_elements = line.split()
        endName = [i for i, element in enumerate(line_elements) if pattern.match(element)]
        if len(endName) == 1:
            names.append(' '.join(line_elements[1:endName[0]]))
        else:
            print("ERROR")
            print(endName)

    blockValues = list()

    start = -1
    end = 0
    prev = ''

    for i,name in enumerate(names):
        if start == -1:
            start = 0
            prev = name
        elif i == len(names)-1:
            blockValues += get_dev_block_values(start, end)
        else:
            if prev == name:
                end = i
            else:
                if start == end:
                    blockValues.append(1)
                else:
                    blockValues += get_dev_block_values(start, end)
                start = i
                end = i
                prev = name

    os.chdir(current_dir)

    for res, value in zip(result, blockValues):
        res['vuln_prob'] = (res['vuln_prob'] + (1 - weight_func(value, 10))) / 2
        res['git_surr'] = value

    return result

def new_surrounded(line_data, acceptance, target):
    result = deepcopy(line_data)

    lexed_lines = list()
    with open(BASEDIR + '/lexed.txt', 'r') as fp:
        lexed_lines = fp.readlines()

    for line in result:
        if(len(set(lexed_lines[line['line_num']].split()))):
            line['vuln_prob'] = (line['vuln_prob'] + weight_func(len(set(lexed_lines[line['line_num']].split())))) / 2
            line['complexity'] = len(set(lexed_lines[line['line_num']].split()))

    for (i, line), data in zip(enumerate(result),line_data):
        neighbourCount = 1
        if data['vuln_prob'] > acceptance:
            for j in range(i-1,0,-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            for j in range(i+1, len(line_data)-1):
                if line_data[j]['vuln_prob'] > acceptance:
                    neighbourCount += 1
                else:
                    break
            if neighbourCount > 0:
                line['vuln_prob'] = (line['vuln_prob'] + weight_func(neighbourCount)) / 2
            else:
                line['vuln_prob'] = 0
            line['surrounding'] = neighbourCount

    return result

def main():
    # get target path from command line argument
    arg_parser = argparse.ArgumentParser(description='Look for potentially vulnreable lines in a js file')
    arg_parser.add_argument('-f', '--target_file', metavar='', help='Path to the file that needs to be checked')
    args = arg_parser.parse_args()

    # these are treated as global varaibles in case they would be needed for a rule
    global acceptance, model, index2word_set

    # js keywords are read in from a txt file
    with open('../word2vec/keywords.txt', 'r') as k:
        keywords = ''
        for line in k.readlines():
            keywords += line.rstrip() + ','

    # the predictor should be runnable for a specific file, as for now, this uses a preset of the IDname model, 80% acceptance, and no rules
    if args.target_file == 'k_fold':
        k = 10
        # for the creation of test results every available option is explored herefor acceptance values, rules, and models
        acceptance = [0.92]

        # the modelNames need to be in the same order as the txt files
        modelNames = ['SpecialValues']

        not_found = list()

        features = 100

        rules = [no_one_word_lines, check_surroundings, prefer_complex, complex_and_surrounded, complex_before_surr, iter_surrounding, new_surrounded]

        for i in range(1,k+1):
            with open(f'target_paths{i}.json','r') as fp:
                targets = json.load(fp)

            names = dict()

            for mName in modelNames:
                model = wv.Word2Vec.load(f"models/{mName}.model")
                file_name = f'vuln_lines_{mName}_{i}.txt'
                index2word_set = set(model.wv.index2word)

                if not os.path.isdir(f'./{mName}_results_{i}'):
                    os.mkdir(f'./{mName}_results_{i}')
                readVulnLines(model, features, index2word_set, file_name)
                for target in targets:

                    target_name = target.split('/')[-1]

                    if target_name not in names:
                        names[target_name] = 0
                    else:
                        names[target_name] += 1
                        target_name += str(names[target_name])

                    if not os.path.isdir(f'./{mName}_results_{i}/{target_name}'):
                        os.mkdir(f'./{mName}_results_{i}/{target_name}')
                    print(f'Begin {mName} test on {target_name}')
                    try:
                        predict(target, keywords, model, features, index2word_set, acceptance, target_name, mName, rules, i)
                    except FileNotFoundError as er:
                        not_found.append(er)
        print(len(not_found))
        print(not_found)

    elif args.target_file == 'create_test_results':
        # for the creation of test results every available option is explored herefor acceptance values, rules, and models
        acceptance = [0.75, 0.8, 0.85, 0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97]

        # the modelNames need to be in the same order as the txt files
        modelNames = ['SpecialValues','IDname', 'novarname']

        vuln_file_names = ['vuln_lines_extended_lex.txt', 'vuln_lines_new_lex.txt', 'vuln_lines_old_lex.txt']

        features = 100

        rules = [no_one_word_lines, check_surroundings, prefer_complex, complex_and_surrounded, complex_before_surr, iter_surrounding, new_surrounded]

        with open('target_paths.json','r') as fp:
            targets = json.load(fp)

        for mName, file_name in zip(modelNames, vuln_file_names):
            model = wv.Word2Vec.load(f"models/{mName}.model")

            index2word_set = set(model.wv.index2word)

            http_server_count = 1

            if not os.path.isdir(f'./{mName}_results'):
                os.mkdir(f'./{mName}_results')
            readVulnLines(model, features, index2word_set, file_name)
            for target in targets:

                target_name = target.split('/')[-1]

                if target_name == '_http_server.js':
                    target_name = (target_name if http_server_count == 1 else '_http_server' + str(http_server_count) + '.js')
                    http_server_count += 1

                if not os.path.isdir(f'./{mName}_results/{target_name}'):
                    os.mkdir(f'./{mName}_results/{target_name}')
                print(f'Begin {mName} test on {target_name}')
                predict(target, keywords, model, features, index2word_set, acceptance, target_name, mName, rules)
        else:
            acceptance = [0.8]

            model = wv.Word2Vec.load("models/IDname.model")

            features = 100

            index2word_set = set(model.wv.index2word)

            readVulnLines(model, features, index2word_set)

            predict(args.target_file, keywords, model, features, index2word_set, acceptance)

if __name__ == '__main__':
    main()


import os
import sys
import argparse
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))

import torch
import torch.nn as nn
import torch.optim as optim

import numpy as np
from cnn_1d import CNN1D
from brain_dataset import BrainDataset
from model_trainer import ModelTrainer

DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('--data', type=str, help='Data file path', required=True)
	parser.add_argument('--output', type=str, help='Output file path', required=True)
	parser.add_argument('--output_model', type=str, help='Model path', default=None)
	parser.add_argument('--level', type=int, default=0)
	parser.add_argument('--fold', type=int, default=2)
	parser.add_argument('--iter', type=int, default=1)
	parser.add_argument('--batch_size', type=int, default=16)
	parser.add_argument('--epoch', type=int, default=30)
	parser.add_argument('--random_state', type=int, default=None)
	args = parser.parse_args()

	dataset = BrainDataset(args.data, expand_dim=True, level=args.level)
	model = CNN1D(len(np.unique(dataset.label))).to(DEVICE)
	criterion = nn.CrossEntropyLoss()
	optimizer = optim.Adam(model.parameters(), lr=0.001)

	epochs = args.epoch
	batch_size = args.batch_size
	trainer = ModelTrainer(model, dataset, DEVICE)
	result = trainer.train(optimizer, criterion,
						   batch_size=batch_size,
						   epochs=epochs,
						   kfold=args.fold,
						   iteration=args.iter,
						   random_state=args.random_state)

	result = np.array(result)
	np.savetxt(args.output, result, delimiter=",")
	if args.output_model is not None:
		torch.save(model.state_dict(), args.output_model)

if __name__ == '__main__':
	main()


from flask import Flask, request, jsonify
import data_connection as dc
import os

app = Flask(__name__)

APP_ROOT = os.path.dirname(os.path.abspath(__file__))

#@app.route('/sample')
def assess_percent(qtr,busunit,process,geo,market):

    sqlSOX = """
    WITH Q1 AS
    (
    SELECT DISTINCT
    RPTG_QTR,
    A.COUNTRY,
    B.MARKET,
    BUSINESS_UNIT,
    PROCESS,
    B.GEO,
    A.ASSESSED_ENTITY_TYPE,
    A.ASSESSED_ENTITY_ID,
    RATING,
    CASE WHEN RATING = 'Satisfactory' THEN 1 ELSE 0 END AS SATISFACTORY,
    CASE WHEN RATING = 'Unsatisfactory' THEN 1 ELSE 0 END AS UNSAT,
    CASE WHEN RATING = 'Marginal' THEN 1 ELSE 0 END AS MARGINAL,
    CASE WHEN RATING = 'Exempt' OR EXEMPT = 'E' THEN 1 ELSE 0 END AS EXEMPTS,
    CASE WHEN RATING = 'Pending' THEN 1 ELSE 0 END AS PENDING,
    CASE WHEN RATING = 'Pending' OR EXEMPT = 'E' THEN 0 ELSE 1 END AS TOTAL_RATING

    FROM SOX.CFOPORT_ASSESSMENT A
    LEFT JOIN SOX.MARKET B ON A.COUNTRY = B.COUNTRY

    )

    SELECT DISTINCT
    A.ASSESSED_ENTITY_TYPE,
    SUM(SATISFACTORY) AS SUM_SATISFACTORY,
    SUM(UNSAT) AS SUM_UNSATISFACTORY,
    SUM(MARGINAL) AS SUM_MARGINAL,
    SUM(EXEMPTS) AS SUM_EXEMPT,
    SUM(PENDING) AS SUM_PENDING,
    SUM(TOTAL_RATING) AS TOTAL_RATING,
    CAST((CAST(SUM(SATISFACTORY) AS DECIMAL(31,2)) / CAST(SUM(TOTAL_RATING) AS DECIMAL(31,2)) * 100) AS DECIMAL(31,2)) AS SAT_RATE

    FROM Q1 A

    WHERE A.RPTG_QTR IS NOT NULL
    AND A.TOTAL_RATING NOT IN (0)
    """

    sqlgroup = """

    GROUP BY A.ASSESSED_ENTITY_TYPE"""

    if busunit == "" and process == "" and market == "" and geo == "" and qtr == "":
        sqlSOX = sqlSOX + sqlgroup

    else:
        sqlSOX = concatFilter(sqlSOX, qtr, splitParam(qtr), 0, 'A.')
        sqlSOX = concatFilter(sqlSOX, busunit, splitParam(busunit), 1, 'A.')
        sqlSOX = concatFilter(sqlSOX, process, splitParam(process), 2, 'A.')
        sqlSOX = concatFilter(sqlSOX, geo, splitParam(geo), 3, 'A.')
        sqlSOX = concatFilter(sqlSOX, market, splitParam(market), 4, 'A.')

    sqlSOX = sqlSOX + sqlgroup

    # Execute the query
    assess_percent = list(dc.select_query(sqlSOX))

    print(assess_percent)

    return jsonify(assess_percent)

def concatFilter(sqlSOX, param, splitParam, cnt, alias): # Concatenate filters
    listParams = ['RPTG_QTR','BUSINESS_UNIT','PROCESS','GEO','MARKET']

    if param:
            sqlSOX +=" AND " + alias + listParams[cnt] + " IN (" + splitParam + ")"
    return sqlSOX

def splitParam(param): # Splitting multiple values for each parameters
    param = str(param)
    filters =param.split(',')
    itemfilter = ""
    for data in filters:
        if itemfilter == "":
            itemfilter = " '" + data +"' "
        else:
            itemfilter = itemfilter + " ,'" + data +"' "

    return itemfilter


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Mar  9 09:47:14 2019

@author: davidwilbur
"""
import os
import re
import lda
import time
import logging
import numpy as np
import sqlite3
import pandas as pd
from sklearn.manifold import TSNE
from Functions_Wilbur_Zimmermann import tokenize_only
from sklearn.feature_extraction.text import CountVectorizer
# Plotly plotting packages
import plotly
import plotly.graph_objs as go
from nltk.corpus import stopwords
from gensim.test.utils import common_texts
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from sklearn import metrics
import gensim.models as g
import codecs
from nltk.cluster import KMeansClusterer
import nltk

from gensim.models import Word2Vec
from nltk.cluster import KMeansClusterer
import nltk
import numpy as np
from sklearn import cluster
from sklearn import metrics
#%%
# training data
 # %%
# Set global path variables
directory = os.path.dirname('__file__')
input_file = os.path.join(directory,'..','Article Collection')
_dbFile = os.path.join(input_file,'articles_zenodo.db')

#%%
# Set up our connection to the database
#
_db = sqlite3.connect(_dbFile)
_cursor = _db.cursor()

# Pull the relevant bits of data out of the database and
# store them in a pandas dataframe.
#
print('Pulling article IDs, leanings, and text from database')
_cursor.execute("SELECT ln.id, ln.bias_final, cn.text " +
                "FROM train_lean ln, train_content cn " +
                "WHERE cn.`published-at` >= '2018-01-01' AND ln.id == cn.id AND ln.url_keep='1'")
_df = pd.DataFrame(_cursor.fetchall(), columns=('id', 'lean', 'text'))
_db.close()
#%%
_df.text = _df.text.str.lower()
nonAlpha = re.compile('[^a-zA-Z]+')
_df.text = _df.text.replace(nonAlpha, ' ')

cachedStopWords = stopwords.words('english')
_df.text = _df.text.apply(
                          lambda x: (' '.join([word for word in x.split()
                          if word not in cachedStopWords])))

#%%
print('Tokenizing Articles')
import gensim
from nltk.tokenize import word_tokenize
raw_documents = _df.text
gen_docs = [[w.lower() for w in word_tokenize(text)] for text in raw_documents]

 #%%

model = Word2Vec(gen_docs, min_count=1)

def sent_vectorizer(sent, model):
    sent_vec =[]
    numw = 0
    for w in sent:
        try:
            if numw == 0:
                sent_vec = model[w]
            else:
                sent_vec = np.add(sent_vec, model[w])
            numw+=1
        except:
            pass

    return np.asarray(sent_vec) / numw

X=[]
for sentence in gen_docs:
    X.append(sent_vectorizer(sentence, model))

NUM_CLUSTERS=5
kclusterer = KMeansClusterer(NUM_CLUSTERS, distance=nltk.cluster.util.cosine_distance, repeats=25)
assigned_clusters = kclusterer.cluster(X, assign_clusters=True)
print (assigned_clusters)

for index, sentence in enumerate(sentences):
    print (str(assigned_clusters[index]) + ":" + str(sentence))

kmeans = cluster.KMeans(n_clusters=NUM_CLUSTERS)
kmeans.fit(X)

labels = kmeans.labels_
centroids = kmeans.cluster_centers_

print ("Cluster id labels for inputted data")
print (labels)
print ("Centroids data")
print (centroids)

print ("Score (Opposite of the value of X on the K-means objective which is Sum of distances of samples to their closest cluster center):")
print (kmeans.score(X))

silhouette_score = metrics.silhouette_score(X, labels, metric='euclidean')

print ("Silhouette_score: ")
print (silhouette_score)

import matplotlib.pyplot as plt

from sklearn.manifold import TSNE

model = TSNE(n_components=3, random_state=0)
np.set_printoptions(suppress=True)

Y=model.fit_transform(X)
_lean = _df['lean'][_idx]

def SetColor(x):
    if(x == 'left'): return "blue"
    elif(x == 'left-center'): return "lightblue"
    elif(x == 'least'): return "lightgray"
    elif(x == 'right-center'): return "orangered"
    elif(x == 'right'): return "red"

tsne_lda_df = pd.DataFrame(Y, columns=['x', 'y', 'z'])
# tsne_lda_df['Resumes'] = tsne_lda_df['Resumes']
x = tsne_lda_df.x
y = tsne_lda_df.y
z = tsne_lda_df.z

trace1 =\
    go.Scatter3d(x=x, y=y, z=z,
                 mode='markers',
                 marker = dict(color=list(map(SetColor, _lean))))

data = [trace1]
layout = go.Layout(
                   scene=dict(
                    xaxis=dict(
                        nticks=4, range=[-20, 20], ),
                    yaxis=dict(
                        nticks=4, range=[-20, 20], ),
                    zaxis=dict(
                        nticks=4, range=[-20, 20], ), ),
                    width=700,
                    margin=dict(
                    r=20, l=10,
                    b=10, t=10)
                  )
fig = go.Figure(data=data, layout=layout)
plotly.offline.plot(fig)

# plt.scatter(Y[:, 0], Y[:, 1], c=assigned_clusters, s=290,alpha=.5)
#
#
#for j in range(len(sentences)):
#   plt.annotate(assigned_clusters[j],xy=(Y[j][0], Y[j][1]),xytext=(0,0),textcoords='offset points')
#   print ("%s %s" % (assigned_clusters[j],  sentences[j]))
#
#
#plt.show()


import requests
import matplotlib.pyplot as plt
from tqdm import tqdm
import numpy as np
import time

url = "http://codingweb.eu-central-1.elasticbeanstalk.com/test234"
a = []

start = time.time()

while len(a) < 100000:
	print(len(a))
	r = requests.get(url=url)
	if("429" in str(r)):
		print("sleeping")
		time.sleep(.1)
	else:
		data = r.json()
		#print(data)
		for d in data:
			a.append(float(d))

	#time.sleep(1)

print(time.time() - start)

plt.hist(a, 20)
plt.show()

mean = sum(a)/len(a)

t = 0
for num in a:
	t += (num - mean)**2

std = np.sqrt((1/len(a))*t)

print(mean, std)


import logging

logging.basicConfig(filename="human_resources.log", level=logging.DEBUG)

valid_employee_kvps = ['first_name', 'last_name', 'email', 'tasks', 'job_title', 'valid_tasks', 'is_active']

def get_departments(request):
    logging.debug('****GET DEPARTMENTS****')
    depts = Process.objects.filter(kind="department")
    logging.debug(depts)
    return [dept.as_dict() for dept in depts]

def get_dept_options(request):
    depts = Process.objects.filter(kind="department")
    return [{'value':dept.PROCESS_ID, 'label': dept.name} for dept in depts]

def all_employees(request):
    employees = Process.objects.filter(kind="employee", is_active=True)
    return [emp.as_dict() for emp in employees]

def get_pms(request):
    pms = Process.objects.filter(kind="employee", job_title="Project Manager", is_active=True)
    logging.debug(pms)
    return [pm.as_dict() for pm in pms]

def get_developers(request):
    emps = Process.objects.filter(kind="employee", is_active=True)
    devs = []
    for e in emps:
        if 'development' in e.valid_tasks:
            devs.append(e)
    return [dev.as_dict() for dev in devs]

def get_testers(request):
    emps = Process.objects.filter(kind="employee", is_active=True)
    testers = []
    for e in emps:
        if 'testing' in e.valid_tasks:
            emps.append(e)
    return [tester.as_dict() for tester in testers]

def find_employee(request):
    try:
        if not isinstance(request, Process) and not isinstance(request, NebriOS):
            if request.FORM is None:
                email = request.BODY.as_dict()['email']
            else:
                email = request.FORM.as_dict()['email']
        else:
            email = request.as_dict()['email']
        employee = Process.objects.get(kind="employee", is_active=True, email=email)
        return employee.as_dict(), employee.PARENT.PROCESS_ID
    except:
        return None, None

def update_employee(request):
    data = {}
    if request.FORM is not None:
        data = request.FORM.as_dict()
    else:
        data = request.BODY.as_dict()

    dept = data['department']
    logging.debug(data)
    # first, let's see if this is a new employee or update
    try:
        emp = Process.objects.get(kind="employee", email=email)
        for key, value in data:
            if key not in valid_employee_kvps:
                continue
            emp[key] = value
        emp.save()
    except:
        # need to create a new employee. let's find the right department.
        department = Process.objects.get(kind='department', PROCESS_ID=dept)
        emp = Process.objects.create(PARENT=department, kind="employee", is_active=True)
        for key, value in data.items():
            if key not in valid_employee_kvps:
                continue
            emp[key] = value
        emp.save()


"""empty message

Revision ID: 6f73b5fffdf1
Revises: 611ba463d901
Create Date: 2017-01-31 12:28:57.626097

"""

# revision identifiers, used by Alembic.
revision = '6f73b5fffdf1'
down_revision = '611ba463d901'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('amount', sa.Column('uniq_id', sa.String(length=240), nullable=True))
    op.create_unique_constraint(None, 'amount', ['uniq_id'])
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'amount', type_='unique')
    op.drop_column('amount', 'uniq_id')
    ### end Alembic commands ###


import sip
import sys

sip.setapi('QString', 2)

if True:
    from PyQt4 import QtGui

class Diag(QtGui.QDialog):
    def __init__(self):
        super(Diag, self).__init__()
        self.layout = QtGui.QVBoxLayout()
        self.list = QtGui.QListWidget()
        self.list.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
        self.addBtn = QtGui.QPushButton('add')
        self.removeBtn = QtGui.QPushButton('remove')
        self.layout.addWidget(self.list)
        self.layout.addWidget(self.addBtn)
        self.layout.addWidget(self.removeBtn)
        self.setLayout(self.layout)

        self.idx = 0

        self.addBtn.clicked.connect(self.add)
        self.removeBtn.clicked.connect(self.remove)

    def add(self):
        self.idx += 1
        self.list.addItem('item%d' % self.idx)

    def remove(self):
        for item in self.list.selectedItems():
            idx = self.list.row(item)
            self.list.takeItem(idx)

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)
    diag = Diag()
    diag.show()
    app.exec_()


import os
import torch as T
import torch.nn.functional as F
import numpy as np
from buffer import ReplayBuffer
from critic import CriticNetwork
from value import ValueNetwork
from actor import ActorNetwork
from torchviz import make_dot
from torch.utils.tensorboard import SummaryWriter

class Agent(object):
    def __init__(self, alpha=0.0003, beta=0.0003, input_dims=(1, 84, 84),
            env=None, gamma=0.99, n_actions=2, max_size=100000, tau=0.005,
            layer1_size=256, layer2_size=256, batch_size=16, reward_scale=2):
        self.summary_writer = SummaryWriter()
        self.gamma = gamma
        self.tau = tau
        self.memory = ReplayBuffer(max_size, input_dims, n_actions)
        self.batch_size = batch_size
        self.n_actions = n_actions

        self.actor = ActorNetwork(alpha, input_dims, n_actions=n_actions,
                    name='actor', max_action=env.action_space.high)
        self.critic_1 = CriticNetwork(beta, input_dims, n_actions=n_actions, name='critic_1' )
        self.critic_2 = CriticNetwork(beta, input_dims, n_actions=n_actions, name='critic_2')
        self.value = ValueNetwork(beta, input_dims, name='value')
        self.target_value = ValueNetwork(beta, input_dims, name='target_value')

        self.scale = reward_scale
        self.update_network_parameters(tau=1)

    def choose_action(self, observation):
        state = T.Tensor([observation]).to(self.actor.device)
        actions, _ = self.actor.sample_normal(state, reparameterize=False)  # actions is Tensor(1, 1,18,3)

        return actions.cpu().detach().numpy()

    def remember(self, state, action, reward, new_state, done):
        self.memory.store_transition(state, action, reward, new_state, done)

    def update_network_parameters(self, tau=None):
        if tau is None:
            tau = self.tau

        target_value_params = self.target_value.named_parameters()
        value_params = self.value.named_parameters()

        target_value_state_dict = dict(target_value_params)
        value_state_dict = dict(value_params)

        for name in value_state_dict:
            value_state_dict[name] = tau*value_state_dict[name].clone() + \
                    (1-tau)*target_value_state_dict[name].clone()

        self.target_value.load_state_dict(value_state_dict)

    def save_models(self):
        print('.... saving models ....')
        self.actor.save_checkpoint()
        self.value.save_checkpoint()
        self.target_value.save_checkpoint()
        self.critic_1.save_checkpoint()
        self.critic_2.save_checkpoint()

    def load_models(self):
        print('.... loading models ....')
        self.actor.load_checkpoint()
        self.value.load_checkpoint()
        self.target_value.load_checkpoint()
        self.critic_1.load_checkpoint()
        self.critic_2.load_checkpoint()

    def learn(self):
        if self.memory.mem_cntr < self.batch_size:
            return
        """"
          Step 1: Sample from the replay buffer.
        """

        state, action, reward, new_state, done = \
                self.memory.sample_buffer(self.batch_size)

        reward = T.tensor(reward, dtype=T.float).to(self.actor.device)
        done = T.tensor(done).to(self.actor.device)
        state_ = T.tensor(new_state, dtype=T.float).to(self.actor.device)
        state = T.tensor(state, dtype=T.float).to(self.actor.device)
        action = T.tensor(action, dtype=T.float).to(self.actor.device)

        """
            Step 2: Compute the values of the current and next state
        """
        value = self.value(state)
        value_ = self.target_value(state_).view(-1)  # value_ is Tensor(21504,)
        value_[done] = 0.0 #  the size of value_ is Tensor(21504,). done is of size Tensor(256,)d

        """
            Step 3: Predict the actions  and log probabilities of the current state.
        """
        actions, log_probs = self.actor.sample_normal(state, reparameterize=False)  # state is Tensor(256,1,84,84)
                                                            # action is Tensor(256,3) log_probs is Tensor(256,1)
         # log_probs = log_probs.view(-1)  # log_probs is Tensor(256,)
        q1_new_policy = self.critic_1.forward(state, actions)  # q1_new_policy is Tensor(256,1)
        q2_new_policy = self.critic_2.forward(state, actions)  # q2_new_policy is Tensor(256,1)
        critic_value = T.min(q1_new_policy, q2_new_policy)  # critic_value is Tensor(256,1)
        # critic_value = critic_value.view(-1)  # critic_value is now Tensor(256,)

        """
            Step 4: Backpropagate the value network.
        """
        self.value.optimizer.zero_grad()
        value_target = critic_value - log_probs # value_target is Tensor(256,1)
        the_mse_loss = F.mse_loss(value, value_target, reduction='none')  # the_mse_loss is Tensor(256,1)
        value_loss = 0.5 * the_mse_loss
        value_loss = value_loss.sum()
        value_loss.backward(retain_graph=True)  #value_loss is Tensor(256,1), keep getting error here. why?
        self.value.optimizer.step()

        actions, log_probs = self.actor.sample_normal(state, reparameterize=True)
        log_probs = log_probs.view(-1)

        q1_new_policy = self.critic_1.forward(state, actions)
        q2_new_policy = self.critic_2.forward(state, actions)
        critic_value = T.min(q1_new_policy, q2_new_policy)
        critic_value = critic_value.view(-1)

        actor_loss = log_probs - critic_value
        actor_loss = T.mean(actor_loss)
        self.actor.optimizer.zero_grad()
        actor_loss.backward(retain_graph=True)
        self.actor.optimizer.step()

        self.critic_1.optimizer.zero_grad()
        self.critic_2.optimizer.zero_grad()
        q_hat = self.scale*reward + self.gamma*value_
        q1_old_policy = self.critic_1.forward(state, action).view(-1)
        q2_old_policy = self.critic_2.forward(state, action).view(-1)
        critic_1_loss = 0.5 * F.mse_loss(q1_old_policy, q_hat)
        critic_2_loss = 0.5 * F.mse_loss(q2_old_policy, q_hat)

        critic_loss = critic_1_loss + critic_2_loss
        critic_loss.backward()
        self.critic_1.optimizer.step()
        self.critic_2.optimizer.step()

        self.update_network_parameters()


import torch
import os
import wandb
import yaml

class Logger:
    def __init__(self, args, models):
        self.args = args
        # save args
        self.models = models
        # TODO: this is a hack to avoid logging outside train.py FIXME
        if hasattr(args, "basedir"):
            self.basedir = args.basedir
            with open(os.path.join(args.basedir, "args.yaml"), "w") as f:
                yaml.dump(vars(args), f, sort_keys=False)
            if args.WANDB:
                self.wandb = True
                n_params = sum(p.numel() for p in models[0].parameters())
                wandb.init(
                    project="nuclr_iclr24",
                    entity="iaifi",
                    name=args.name,
                    notes="new and great runs",
                    tags=["good_performance_run"],
                    group="ICLR24",
                    config=dict(vars(args)),
                )
                wandb.config.update({"n_params": n_params})
                artifact = wandb.Artifact(
                    "nuclr", type="code", description="nuclr code"
                )
                nuclr_dir = os.path.dirname(os.path.abspath(__file__))
                artifact.add_dir(nuclr_dir)
                wandb.log_artifact(artifact)
            else:
                self.wandb = False
        else:

            self.basedir = None
        self.best_loss = float("inf")
        self._defined_metrics = None

    def define_metrics_wandb(self, metrics):
        if not self.wandb or self._defined_metrics is not None:
            return
        self._defined_metrics = metrics
        for metric in metrics:
            wandb.define_metric(metric, summary="min")

    def log(self, metrics, epoch, save_model=False):
        if self.basedir is None:
            return -1
        self.define_metrics_wandb(metrics)
        val_loss = metrics["val_loss_all"]
        # keep track of the best model
        if val_loss < self.best_loss:
            self.best_loss = val_loss
            self.best_models = [m.state_dict().copy() for m in self.models]
            [
                torch.save(
                    self.best_models[fold],
                    os.path.join(self.basedir, f"model_best.pt.{fold}"),
                )
                for fold in self.args.WHICH_FOLDS
            ]
        if self.args.WANDB:
            wandb.log(metrics, step=epoch)
        else:
            train_items = [
                f"{' '.join(k.split('_')[1:]):<20} | {v:<8.2e}"
                for k, v in metrics.items()
                if "train" in k
            ]
            val_items = [
                f"{v:<8.2e}" for k, v in metrics.items() if "val" in k
            ]
            items = [
                " | ".join([x, y]) for x, y in zip(train_items, val_items)
            ]
            msg = f"Epoch {epoch:<14} | {'Train':^8} | {'Val':^8}\n"
            msg += "\n".join(sorted(items, key=lambda x: x.split(" ")[0]))
            print(msg)

        if save_model:
            epoch_str = (
                str(epoch) if epoch != self.args.EPOCHS - 1 else "final"
            )
            [
                torch.save(
                    self.models[fold].state_dict(),
                    os.path.join(self.basedir, f"model_{epoch_str}.pt.{fold}"),
                )
                for fold in self.args.WHICH_FOLDS
            ]

        # at the end, make a file "done.txt"
        if epoch == self.args.EPOCHS - 1:
            print("done", os.path.join(self.basedir), sep="\n")
            with open(os.path.join(self.basedir, "done.txt"), "w") as f:
                f.write("finalized successfully")