# encoding: utf-8
import unittest

import responses

from jenkinsx import Jenkins
from jenkinsx.exceptions import ItemNotFoundError, \
    AuthenticationError, BadRequestError
from jenkinsx.item import snake
from jenkinsx.job import Folder
from jenkinsx.user import ApiToken
from .help import mock_get, load_test_json, \
    mock_post, remove_get, replace_get, JENKINS_URL

# logging.basicConfig(level=logging.DEBUG)
class TestJenkins(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.jenkins_json = load_test_json('jenkins/jenkins.json')
        cls.folder_json = load_test_json('job/folder.json')
        cls.crumb = load_test_json('jenkins/crumb.json')

    def setUp(self):
        mock_get(f'{JENKINS_URL}crumbIssuer/api/json',
                 json=self.crumb)
        mock_get(f'{JENKINS_URL}api/json', json=self.jenkins_json)
        mock_get(f'{JENKINS_URL}job/Level1_Folder1/api/json',
                 json=self.folder_json)
        self.jx = Jenkins(f'{JENKINS_URL}', auth=('admin', 'admin'))

    @responses.activate
    def test_init_with_invalid_auth(self):
        remove_get(f'{JENKINS_URL}crumbIssuer/api/json')
        mock_get(f'{JENKINS_URL}crumbIssuer/api/json', status=401)
        jx = Jenkins(f'{JENKINS_URL}', auth=('admin', 'admin'))
        with self.assertRaises(AuthenticationError):
            jx.api_json()

    def test_init_with_password(self):
        self.assertIsInstance(self.jx, Jenkins)
        self.assertEqual(str(self.jx), f'<Jenkins: {JENKINS_URL}>')
        self.assertEqual(self.jx.url, JENKINS_URL)
        self.assertIsNone(self.jx._crumb)
        self.assertIsNone(self.jx._token)

    @responses.activate
    def test_init_with_token(self):
        data = {'tokenName': 'mock-token-name',
                'tokenValue': 'mock-token-value',
                'tokenUuid': 'mock-token-uuid'}
        token = ApiToken(data['tokenName'],
                         data['tokenUuid'], data['tokenValue'])
        mock_post(f'{JENKINS_URL}user/admin/'
                  'descriptorByName/jenkins.security.ApiTokenProperty/'
                  'generateNewToken?newTokenName=',
                  json={'data': data})
        mock_get(JENKINS_URL)
        with unittest.mock.patch('weakref.finalize') as mock_fin:
            mock_fin.side_effect = lambda *args: None
            jx = Jenkins(JENKINS_URL,
                         auth=('admin', 'admin'), token=True)
        self.assertTrue(jx.exists())
        self.assertEqual(jx.url, JENKINS_URL)
        self.assertEqual(jx._crumb, self.crumb)
        self.assertEqual(jx._token, token)

    @responses.activate
    def test_dynamic_attributes(self):
        dynamic_attrs = {snake(k): v for k, v in self.jenkins_json.items()
                         if isinstance(v, (int, str, bool))}
        self.assertEqual(sorted(self.jx.attrs),
                         sorted(dynamic_attrs.keys()))
        for key, value in dynamic_attrs.items():
            with self.subTest(value=value):
                self.assertEqual(getattr(self.jx, key), value)

    @responses.activate
    def test_get_job_when_not_found(self):
        not_found = self.jx.get_job('Level1_Folder1/notfound')
        self.assertIsNone(not_found)

    @responses.activate
    def test_get_job(self):
        level1_folder1 = self.jx.get_job('Level1_Folder1')
        self.assertEqual(self.jx._crumb, self.crumb)
        self.assertIsNone(self.jx._token)
        self.assertIsInstance(level1_folder1, Folder)
        self.assertEqual(str(level1_folder1),
                         f'<Folder: {JENKINS_URL}job/Level1_Folder1/>')
        level2_folder1 = self.jx.get_job('Level1_Folder1/Level2_Folder1')
        self.assertIsInstance(level2_folder1, Folder)
        self.assertEqual(str(level2_folder1),
                         f'<Folder: {JENKINS_URL}job/Level1_Folder1/'
                         'job/Level2_Folder1/>')

    @responses.activate
    def test_iter_jobs(self):
        replace_get(f'{JENKINS_URL}api/json',
                    json=load_test_json('jenkins/jenkins_all_jobs.json'))
        jobs = list(self.jx.iter_jobs())
        self.assertEqual(len(jobs), 6)

    @responses.activate
    def test_create_job_when_exists(self):
        mock_post(
            f'{JENKINS_URL}job/Level1_Folder1/createItem',
            headers={
                'X-Error': "A job already exists with"
                " the name 'Level2_Folder1'"},
            status=400)
        with self.assertRaises(BadRequestError):
            self.jx.create_job('Level1_Folder1/Level2_Folder1', '')

    @responses.activate
    def test_create_job_when_name_illegal(self):
        mock_post(
            f'{JENKINS_URL}job/Level1_Folder1/createItem',
            headers={'X-Error': '@  is an unsafe character'},
            status=400)
        with self.assertRaisesRegex(BadRequestError,
                                    '@  is an unsafe character'):
            self.jx.create_job('Level1_Folder1/Level2@new', '')

    @responses.activate
    def test_create_job(self):
        mock_post(
            f'{JENKINS_URL}job/Level1_Folder1/createItem?name=Levevl2_new',
            match_querystring=True)
        self.jx.create_job('Level1_Folder1/Levevl2_new', '')

    @responses.activate
    def test_delete_job(self):
        mock_post(f'{JENKINS_URL}job/Level1_Folder1/'
                  'job/Level2_Folder1/doDelete',
                  match_querystring=True)
        self.jx.delete_job('Level1_Folder1/Level2_Folder1')

    @responses.activate
    def test_build_job_when_not_found(self):
        with self.assertRaisesRegex(ItemNotFoundError,
                                    'No such job: '
                                    'Level1_Folder1/Level2_notFound'):
            self.jx.build_job('Level1_Folder1/Level2_notFound')

    @responses.activate
    def test_build_job_when_job_is_unbuildable(self):
        with self.assertRaisesRegex(AttributeError,
                                    "'Folder' object has no attribute 'build'"):
            self.jx.build_job('Level1_Folder1')

    def test_url2full_name(self):
        with self.assertRaises(ValueError):
            self.jx.url2path('http://0.0.0.1/job/folder1/')
        full_name = self.jx.url2path(f'{JENKINS_URL}job/job/')
        self.assertEqual(full_name, '/job/')
        full_name = self.jx.url2path(
            f'{JENKINS_URL}job/job/job/job/')
        self.assertEqual(full_name, '/job/job/')
        full_name = self.jx.url2path(f'{JENKINS_URL}job/job/job/job')
        self.assertEqual(full_name, '/job/job')

    def test_full_name2url(self):
        self.assertEqual(self.jx.path2url(''), self.jx.url)
        for name in ['/job/', 'job/', '/job', 'job']:
            with self.subTest(name=name):
                self.assertEqual(self.jx.path2url(name),
                                 f'{JENKINS_URL}job/job/')
        for name in ['/job/job/', 'job/job/', '/job/job', 'job/job']:
            with self.subTest(name=name):
                self.assertEqual(self.jx.path2url(name),
                                 f'{JENKINS_URL}job/job/job/job/')

    @responses.activate
    def test_api_json(self):
        json = self.jx.api_json()
        self.assertEqual(json, self.jenkins_json)

    @responses.activate
    def test_exists(self):
        for status in [401, 403]:
            remove_get(JENKINS_URL)
            mock_get(JENKINS_URL, status=status)
            with self.subTest(status=status):
                self.assertTrue(self.jx.exists())

        for status in [404, 503]:
            remove_get(JENKINS_URL)
            mock_get(JENKINS_URL, status=status)
            with self.subTest(status=status):
                self.assertFalse(self.jx.exists())

    @responses.activate
    def test_handle_req(self):
        json = self.jx.handle_req('GET', 'api/json').json()
        self.assertEqual(json, self.jenkins_json)


from django import forms
from django.utils.translation import gettext_lazy as _

from faclab.widgets import PercentInput, PriceInput, Select2
from viewpack.forms import ModelForm

from .enums import PriceTypes
from .models import Price, Product, ProductCategory

class ProductCategoryForm(ModelForm):
    class Meta:
        model = ProductCategory
        fields = "__all__"
        widgets = {
            "parent": Select2(
                model="sales.ProductCategory",
                search_fields=["name__icontains"],
            )
        }

class ProductForm(ModelForm):
    cost_price = forms.FloatField(widget=PriceInput, label=_("price net"))
    cost_price_gross = forms.FloatField(widget=PriceInput, label=_("price gross"))

    class Meta:
        model = Product
        fieldsets = (
            "code",
            "name",
            "short_name",
            "description",
            ("is_inventoried", "apply_iva", "apply_ice"),
        )
        fields = ("cost_price", "cost_price_gross", "type", "category", "measure")
        widgets = {"type": forms.RadioSelect}

    def get_initial_for_field(self, field, field_name):
        if field_name == "cost_price":
            cost_price = getattr(self.instance, "cost_price", None)
            return cost_price.amount if cost_price else 0

        if field_name == "cost_price_gross":
            cost_price = getattr(self.instance, "cost_price", None)
            return cost_price.gross_amount if cost_price else 0

        return super().get_initial_for_field(field, field_name)

    def save(self, commit=True):
        obj = super().save(commit=False)

        if commit:
            obj.save()
            price = obj.prices.filter(type=PriceTypes.PURCHASE).first()
            cost_price = self.cleaned_data.get("cost_price")

            if price:
                price.amount = cost_price
                price.save(update_fields=["amount"])
            else:
                Price.objects.create(
                    type=PriceTypes.PURCHASE, amount=cost_price, revenue=0, product=obj
                )

        return obj

class PriceForm(ModelForm):
    gross_amount = forms.FloatField(widget=PriceInput, label=_("price gross"))
    percent_revenue = forms.FloatField(
        min_value=0, max_value=100, widget=PercentInput, label=_("percent revenue")
    )

    class Meta:
        model = Price
        fields = ("amount", "gross_amount", "percent_revenue", "revenue")
        labels = {"amount": _("price net")}
        widgets = {"amount": PriceInput, "revenue": PriceInput}

    def get_initial_for_field(self, field, field_name):
        if field_name in ("gross_amount", "percent_revenue"):
            return getattr(self.instance, field_name, None)

        return super().get_initial_for_field(field, field_name)

    def save(self, commit=True):
        obj = super().save(commit=False)
        obj.type = PriceTypes.SALE

        if commit:
            obj.save()

        return obj


import gc

import cudf
import cupy as cp
import numpy as np
import pandas as pd
import xfeat
from cuml.preprocessing import LabelEncoder
from src.features.base import Feature
from src.utils import timer
from tqdm import tqdm
from xfeat.types import XDataFrame

cat_cols = ["Name", "Platform", "Genre", "Publisher", "Developer", "Rating"]

class ConcatCategory(Feature):
    def create_features(
        self,
        train_df: pd.DataFrame,
        test_df: pd.DataFrame,
    ):

        with timer("load data"):
            train = train_df.copy()
            len_train = len(train)
            org_cols = train.columns.tolist()
            test = test_df.copy()

        with timer("concat train and test"):
            total = cudf.concat([train, test], ignore_index=True).reset_index()
            del train, test
            gc.collect()

        with timer("combi cats"):
            new_cat_df = cudf.concat(
                [
                    xfeat.ConcatCombination(drop_origin=True, r=r).fit_transform(
                        total[cat_cols].astype(str
                                               ).fillna("none")
                    )
                    for r in [2, 3, 4]
                ],
                axis="columns",
            )

            for col in new_cat_df.columns:
                le = LabelEncoder()
                new_cat_df[col] = le.fit_transform(new_cat_df[col]).astype("category")

            total = cudf.concat(
                [total, new_cat_df],
                axis="columns",
            )

        with timer("end"):
            total = total.sort_values("index")
            new_cols = [col for col in total.columns if col not in org_cols + ["index"]]

            self.train = total[new_cols].iloc[:len_train].reset_index(drop=True)
            self.test = total[new_cols].iloc[len_train:].reset_index(drop=True)


# Advent of code
# Author = Rob Lorimer
YEAR=2020; DAY=19; PART=2

# add project directory to path
import sys, os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from utils import aoc

@aoc.puzzle_timer()
def solve(input):
    return "todo"

if __name__ == '__main__':
    puzzle_input = aoc.puzzle_input(YEAR, DAY)
    print(f'Part {PART}: ', solve(puzzle_input))


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import cv2
import time
import random
from tqdm import tqdm
import numpy as np
import tensorflow as tf
from pathlib import Path

def create_test_data(CATEGORIES, DATADIR, img_shape):
    test_x = []  # 测试集的数据和标签
    for category in CATEGORIES:
        path = os.path.join(DATADIR, category)
        for img in tqdm(os.listdir(path)):  # iterate over each image
            try:
                img_array = cv2.imread(os.path.join(path, img))  # convert to array
                img_array = cv2.cvtColor(img_array, cv2.COLOR_BGR2RGB)
                new_array = cv2.resize(img_array, img_shape)  # resize to normalize data size
                new_array = new_array.astype(np.float32) / 255.
                new_array = np.expand_dims(new_array, axis=0)
                test_x.append(new_array)
            except Exception as e:  # in the interest in keeping the output clean...
                pass

    return test_x

def keras2tflite(keras_file, tflite_file, test_images):
    def representative_data_gen():
        for input_value in test_images:
            yield [input_value]

    # 恢复 keras 模型，并预测
    model = tf.keras.models.load_model(keras_file)

    # 动态量化 dynamic range quantization
    converter = tf.lite.TFLiteConverter.from_keras_model(model)
    converter.optimizations = [tf.lite.Optimize.DEFAULT]
    converter.representative_dataset = representative_data_gen
    # Ensure that if any ops can't be quantized, the converter throws an error
    converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
    # Set the input and output tensors to uint8 (APIs added in r2.3)
    converter.inference_input_type = tf.uint8
    converter.inference_output_type = tf.uint8

    tflite_model = converter.convert()

    tflite_file.write_bytes(tflite_model)
    print("convert model to tflite done...")

def tflite_infer(tflite_file, image_path, CATEGORIES):
    # tflite 模型推理
    interpreter = tf.lite.Interpreter(model_path=str(tflite_file))
    interpreter.allocate_tensors()

    # Get input and output tensors.
    input_details = interpreter.get_input_details()[0]
    output_details = interpreter.get_output_details()[0]
    height = input_details['shape'][1]
    width = input_details['shape'][2]

    # 单个测试样本数据
    image = cv2.imread(image_path)
    image = cv2.resize(image, (width, height))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = np.expand_dims(image, axis=0)
    image.flatten()
    interpreter.set_tensor(input_details['index'], image)

    start_time = time.time()
    interpreter.invoke()
    stop_time = time.time()

    # interpreter = tf.lite.Interpreter(model_content=tflite_model)
    input_type = interpreter.get_input_details()[0]['dtype']
    print('input: ', input_type)
    output_type = interpreter.get_output_details()[0]['dtype']
    print('output: ', output_type)

    output_data = interpreter.get_tensor(output_details['index'])
    print(output_data)
    print(f"prediction: {CATEGORIES[np.argmax(output_data)]}")
    print('time: {:.3f}ms'.format((stop_time - start_time) * 1000))
    print("model size: {:.2f} MB".format(os.path.getsize(tflite_file)/1024/1024))

def main():
    DATADIR = './quantitative_dataset'
    CATEGORIES = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
    img_shape = (32, 32)
    keras_file = './cifar10_float32.h5'
    tflite_file = Path("./cifar10_uint8.tflite")
    test_image_path = "./test_imgs/deer.jpg"

    test_x = create_test_data(CATEGORIES, DATADIR, img_shape)
    keras2tflite(keras_file, tflite_file, test_x)
    tflite_infer(tflite_file, test_image_path, CATEGORIES)

if __name__ == "__main__":
    main()


import re
import collections
n = int(input())
pattern = re.compile('[0-9]+')
res = collections.defaultdict(dict)
temp = list()
for i in range(n-1):
    a = [int(x) for x in pattern.findall(input())]
    u, v, w = a[0], a[1], a[2]
    temp.append(a)
    res[u][v] = w

# print(res[2])
m = int(input())
for i in range(m):
    a = pattern.findall(input())
    u, v = int(a[0]), int(a[1])
    temp.append(a)
    if u == v:
        print(0)
        continue
    start = u
    end = list(res[u].keys())[0]
    ans = res[u][end]
    while end != v:
        try:
            start = end
            end = list(res[start].keys())[0]
            ans = ans ^ res[start][end]
        except IndexError:
            print(101)
            break
    print(ans)


from logging import exception
from .Translate.Translog import TransLog
import logging
from .Translate.Replacer import Replacer

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()

class pytorch2caffe_converter(object):
    def __init__(self, model):
        self.layer_names = dict()
        self.translog = TransLog()
        self.model = model
        for name, layer in model.named_modules():
            self.layer_names[layer] = name

        self.replacer = Replacer()
        self.replacer.replace_functions(self.translog, self.layer_names)

    def set_input(
        self,
        input_var,
        source="",
        root_folder="",
        batch_size=1,
        new_height=224,
        new_width=224,
        channel=3
    ):
        self.translog.set_input(
            input_var, source, root_folder, batch_size, channel, new_height, new_width
        )
        self.input_var = input_var

    def trans_net(self, name="TransferedPytorchModel"):
        # import ipdb; ipdb.set_trace()
        self.translog.set_net_name(name)
        x = self.model.forward(self.input_var)
        # self.translog.set_softmaxwithloss(x)

        # post check
        self.translog.post_check()

        # torch to caffe names
        logger.debug("printing torch to caffe names: ")
        for key, value in self.translog.torch_to_caffe_names.items():
            logger.debug(f"torch name: {key}  --> caffe name: {value}")

        self.replacer.place_back()

    def save_prototxt(self, save_name):
        logger.info("saving prototxt to: " + save_name + " ...")
        self.translog.cnet.save_prototxt(save_name)

    def save_caffemodel(self, save_name):
        logger.info("saving caffemodel to: " + save_name + " ...")
        self.translog.cnet.save(save_name)

    def save_torch2caffe_names_json(self, save_name):
        logger.info("saving torch2caffe names to: " + save_name + " ...")
        import json

        with open(save_name, "w") as f:
            json.dump(f, self.trans_net.torch_to_caffe_names, sort_keys=True, indent=2)


#-*- coding: UTF-8 -*-
# xgboost+lr,excited!!!!!!
# final lb on the Santander Customer Match: 1) whole train data gbdt feature generation,lb: 0.822072; 2) sampled train data gbdt feature generation,lb: 0.821391
import time
import re
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import xgboost as xgb
from sklearn.cross_validation import StratifiedKFold
from sklearn.cross_validation import cross_val_score
from sklearn.metrics import roc_auc_score
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.feature_selection import SelectFromModel, VarianceThreshold
import xgboost as xgb
from sklearn.cross_validation import KFold
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import normalize
from sklearn.decomposition import PCA
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output
print("start time: %s" % time.ctime())
check_time = time.time()
time_name = str(time.ctime())
file_time = "_".join(time_name.split()[:3])
GBDT_FILE = 'gbdt_imbalance_%s.txt' % file_time    #tree node file
#GBDT_FILE = 'gbdt_imbalance_Fri_May_20.txt'   #test
COEF_FILE = 'Logistic_coef_%s.txt' % file_time
IMPORTANCE = 3    #最后输出相关性要保留的小数点位数
print(check_output(["ls", "../input"]).decode("utf8"))

# Any results you write to the current directory are saved as output.
print('Load data...')
train = pd.read_csv("../input/train.csv")       #update
train_id = train['ID'].values
target = train['TARGET'].values
#train = train.drop(['ID','TARGET'],axis=1)

test = pd.read_csv("../input/test.csv")          #update
test_id = test['ID'].values
#test = test.drop(['ID'],axis=1)
print('load data complete, train records count: <<<<%s, test records count: <<<<%s, train columns count: <<<<%s, test columns count: <<<<%s, time: <<<<%s' % (train.shape[0], test.shape[0], train.shape[1], test.shape[1], round(((time.time() - check_time)/60),2)))
check_time = time.time()
#------------------------------------------data process-------------------------------------------------------------
#removing outliers, MANUALLY
#train = train.replace(-999999,2)
#test = test.replace(-999999,2)

#replace na
train = train.fillna(-1)
test = test.fillna(-1)

# remove constant columns (std = 0)
remove = []
for col in train.columns:
    if train[col].std() == 0:
        remove.append(col)

train.drop(remove, axis=1, inplace=True)
test.drop(remove, axis=1, inplace=True)
len1 = len(remove)
print(train.shape, test.shape)

# remove duplicated columns
remove = []
cols = train.columns
for i in range(len(cols)-1):
    v = train[cols[i]].values
    for j in range(i+1,len(cols)):
        if np.array_equal(v,train[cols[j]].values):
            remove.append(cols[j])

train.drop(remove, axis=1, inplace=True)
test.drop(remove, axis=1, inplace=True)
len2 = len(remove)

#label encoder
len3 = 0
cols = train.columns
origin_cols = train.columns[1:-1]        #original features
print("origin_cols: %s" % origin_cols)
for col in origin_cols:
     if train[col].dtype=='object':
            print(col)
            len3 += 1
            lbl = preprocessing.LabelEncoder()
            lbl.fit(list(train[col].values) + list(test[col].values))
            train[col] = lbl.transform(list(train[col].values))
            test[col] = lbl.transform(list(test[col].values))

print("data process ended, row droped:<<<<%s, row encoded:<<<<%s, time spend: <<<<%s" %(len1+len2, len3, round(((time.time() - check_time)/60),2)))
check_time = time.time()

#-----------------------------------------------------GBDT model on sample------------------------------------------------------------------------
"""
folds = 10
skf = StratifiedKFold(target,
                          n_folds=folds,
                          shuffle=False,
                          random_state=1580)
a, sample_index = list(skf)[0]
train_sample = train.loc[sample_index,:]
target_sample = target[sample_index]
xgbc = xgb.XGBClassifier(n_estimators=500, seed=1580)
xgbc.fit(train_sample[origin_cols],target_sample)
xgbc.booster().dump_model(GBDT_FILE)
print('gbdt trained on sample, time spend:<<<<%s' % round(((time.time() - check_time)/60),2))
check_time = time.time()
"""
#-----------------------------------------------------GBDT model------------------------------------------------------------------------
xgbc = xgb.XGBClassifier(n_estimators=500, seed=1580)
xgbc.fit(train[origin_cols],target)
xgbc.booster().dump_model(GBDT_FILE)
print('gbdt trained on sample, time spend:<<<<%s' % round(((time.time() - check_time)/60),2))
check_time = time.time()

#-----------------------------------------------------READ FEATURE----------------------------------------------------------------------
f = open(GBDT_FILE,'r')
feature_set = set()
for line in f.readlines():
    if '<' in line:           #feature line
           line = line.split(':')[1].strip()
           feature_re = re.match('\[(.*)?\]', line)
           info = feature_re.group(0)              #should be only one group
           info = re.sub('\[|\]','',info)
           feature_set.add(info)

#feature encoder
for info in feature_set:
    key = info.split("<")[0].strip()
    value = float(info.split("<")[1].strip())
    train[info] = train[key].apply(lambda x: 1 if x<value else 0)
    test[info] = test[key].apply(lambda x: 1 if x<value else 0)

#remove original feature
train = train.drop(origin_cols, axis=1)
test = test.drop(origin_cols, axis=1)
print('feature generated base on gbdt sub-model, <<<<%s feature generated, time spend:<<<<%s' % (test.shape[1]-1, round(((time.time() - check_time)/60),2)))
check_time = time.time()

#--------------------------------------------------------UNDERSAMPLE-------------------------------------------------------------------
train1 = train[train['TARGET']==1]               #positive train samples
train2 = train[train['TARGET']==0]               #negative train samples
#train2 suppose to be the majority type, if not change it
if train2.shape[0]<train1.shape[0]:
    train1 = train[train['TARGET']==0]               #positive train samples
    train2 = train[train['TARGET']==1]               #negative train samples
train1 = train1.reset_index(drop=True)
train2 = train2.reset_index(drop=True)
fold = train2.shape[0] / train1.shape[0]
folds = int(fold / 4.)
skf1 = StratifiedKFold(train1.TARGET.values,
                          n_folds=folds,
                          shuffle=False,
                          random_state=1580)
skf2 = StratifiedKFold(train2.TARGET.values,
                          n_folds=folds,
                          shuffle=False,
                          random_state=1580)
#------------------------------------------------------LVL1 Logistic Regression--------------------------------------------------------
clf = LogisticRegression(penalty='l1', C=0.25, random_state=1580, n_jobs=-1)
features = list(train.columns)
features.remove('ID')
features.remove('TARGET')
df_train_pred = []
df_pred = []
result = []
for i, (a, neg_index) in enumerate(skf2):
        fold_tag = "fold_%s" % i
        pos_index = list(skf1)[i][0]
        train_pos = train1.loc[pos_index, :]
        trainner = pd.concat((train_pos, train2.loc[neg_index,:]),axis=0, ignore_index=True)
        y = trainner.TARGET.values
        X = trainner[features]
        clf.fit(X,y)
        train_pred = clf.predict_proba(train[features])[:,1]
        y_pred = clf.predict_proba(test[features])[:,1]
        df_train_pred.append(train_pred)
        df_pred.append(y_pred)
        result.append(clf.coef_[0])

#average results
train_preds = np.average(np.array(df_train_pred), axis=0)
test_preds = np.average(np.array(df_pred), axis=0)

#output feature weights
coef = np.average(np.array(result), axis=0)
result = sorted(zip(map(lambda x:round(x,IMPORTANCE), coef), features), reverse=True)
o = open(COEF_FILE, "w")
o.write("len of feature: %s\n" % len(result))
for (value, key) in result:
    if 0==value:
       pass
    else:
       writeln = "%s: %.3f\n" % (key, value)
       o.write(writeln)
o.close()

print('LVL1 Logistic Model trained, Average AUC: <<<<%s, time spend:<<<<%s' % (roc_auc_score(train.TARGET.values, train_preds), round(((time.time() - check_time)/60),2)))
check_time = time.time()
#-----------------------------------------------------OUTPUT----------------------------------------------------------------------------
pd.DataFrame({"ID": test_id, "TARGET": test_preds}).to_csv('gbdt_lr_baseline.csv',index=False)
print("end time: %s" % time.ctime())


import numpy as np
X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
Y = np.array([1, 1, 1, 2, 2, 2])

from sklearn.naive_bayes import GaussianNB

clf = GaussianNB()

# fit = train ( learns the pattern )

# X = features
# Y = labels

clf.fit(X, Y)

GaussianNB(priors=None)

print(clf.predict([[10.5, 18]]))


import csv
from datetime import datetime

from dateutil import parser
from github import Github
import pytz

g = Github("de9c704043596ad37b52983a804354b92906d21f")
repo = g.get_user('christopherwhittier').get_repo('IntuitiveWebSolutions/BriteCore')

with open('github-prs.csv', 'w') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(['head user', 'base repo', 'PR updated at', 'last commit modified', 'title'])

    for pull in repo.get_pulls():
        last_date = datetime(1900, 1, 1, 0, 0, 0, 0, pytz.UTC)
        for commit in pull.get_commits():
            commit_date = parser.parse(commit.last_modified)
            if commit_date > last_date:
                last_date = commit_date
        writer.writerow([pull.head.user.login, pull.base.label, pull.updated_at, last_date, pull.title])


# -*- coding:utf-8 -*-
# @Author         : 江湖一笑
# @Time           : 2020/4/9 9:01
# @Software       : Web-Autotest-Python
# @Python_verison : 3.7

# Xpath是XML_path语言的缩写，是一门在XML文档中查找信息的语言。
# 它在XML稳文档中通过元素和属性进行导航，主要用于在XML中选择节点。
# Xpath语言可以用于整个文档中沿着paht或step来寻找指定的节点。说白了就是通过路径来定位元素，一个元素就是一个节点。
# 比CSS定位灵活，但是速度稍逊。
from selenium import webdriver
from time import sleep
driver = webdriver.Chrome()
driver.maximize_window()
driver.get('http://news.baidu.com')
# driver.find_element_by_xpath('//*[@id="ww"]').send_keys('金蝶')    # 通过属性定位，查找,鼠标右键可以获取到浏览器给出的Xpath
driver.find_element_by_xpath('html/body/div/div/table/tbody/tr/td/table/tbody/tr/td/div/span/input').send_keys('金蝶')        # 通过绝对路径查找
sleep(2)
driver.quit()


#OpenGL library import
import OpenGL
OpenGL.ERROR_ON_COPY = True

#For creating window
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *

#Array of 4 tuple
#array for:
fourTupleArrayR = []	#right wing
fourTupleArrayB = []	#body
fourTupleArrayL = []	#left wing
fourTupleArrayN = []	#neck
fourTupleArrayTA = []	#tails part a ...
fourTupleArrayTB = []
fourTupleArrayTC = []
fourTupleArrayTD = []
fourTupleArrayTE = []
fourTupleArrayTF = []
fourTupleArrayTG = []
fourTupleArrayH = []	#head
fourTupleArrayHA = []	#head fur a...
fourTupleArrayHB = []

fourTupleArray2 = []

#Height and width of the image
imageWidth = "1500"
imageHeight = "705"

#Height and width of the window
windowWidth = int(1500)
windowHeight = int(705)

#Function to initiate the 2D orthogonal window
def initiate2DWindow(r,g,b):
    #Clear buffer
    glClearColor(r,g,b,0.0)
    #Set 2D orthogonal window with specified width and height
    glMatrixMode (GL_PROJECTION)
    gluOrtho2D (0.0, float(imageWidth), 0.0, float(imageHeight))

def drawDragonAsPolygonR():
    glColor3ub(255, 0, 0)
    glBegin(GL_POLYGON)
    for line in fourTupleArrayR:
        glVertex2i(line[0], int(imageHeight) - line[1])
        glVertex2i(line[2], int(imageHeight) - line[3])
    glEnd()

def drawDragonAsPolygonB():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayB:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonL():
	glColor3ub(255, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayL:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonN():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayN:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTA():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTA:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTB():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTB:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTC():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTC:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTD():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTD:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTE():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTE:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTF():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTF:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonTG():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayTG:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonH():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayH:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonHA():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayHA:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawDragonAsPolygonHB():
	glColor3ub(150, 0, 0)
	glBegin(GL_POLYGON)
	for line in fourTupleArrayHB:
		glVertex2i(line[0], int(imageHeight) - line[1])
		glVertex2i(line[2], int(imageHeight) - line[3])
	glEnd()

def drawBackup1():
    glColor3ub(0, 0, 0)
    glBegin(GL_POLYGON)
    glVertex2i(719, 705-197)
    glVertex2i(690, 705-149)
    glVertex2i(579, 705-75)
    glVertex2i(633, 705-13)
    glEnd()

def drawBackup2():
    glColor3ub(0, 0, 0)
    glBegin(GL_POLYGON)
    glVertex2i(865, 705-15)
    glVertex2i(921, 705-65)
    glVertex2i(901, 705-147)
    glVertex2i(823, 705-195)
    glEnd()

def drawDragonAsLines():
    #Set line color to black
    glColor3f(255.0, 255.0, 255.0)
    #Draw each line from array of points
    glLineWidth(3.0)
    for line in fourTupleArray2:
      #Line section begin
      glBegin(GL_LINES)
      #Set Point 1
      glVertex2i(line[0], int(imageHeight) - line[1])
      #Set Point 2
      glVertex2i(line[2], int(imageHeight) - line[3])
      #Line section end
      glEnd()

#Function to draw the display
def renderDisplay():
    #Clear buffer
    glClear(GL_COLOR_BUFFER_BIT)

    #Draw dragon
    drawDragonAsPolygonR()
    drawDragonAsPolygonB()
    drawDragonAsPolygonL()
    drawDragonAsPolygonN()
    drawDragonAsPolygonTA()
    drawDragonAsPolygonTB()
    drawDragonAsPolygonTC()
    drawDragonAsPolygonTD()
    drawDragonAsPolygonTE()
    drawDragonAsPolygonTF()
    drawDragonAsPolygonTG()
    drawDragonAsPolygonH()
    drawDragonAsPolygonHA()
    drawDragonAsPolygonHB()
    drawBackup1()
    drawBackup2()
    drawDragonAsLines()

    #Draw line and flush the buffer
    glFlush()

#Function to read array of points from file
def readPointsFromFileRight():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1rightwing", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayR.append(temporaryList)

            #Function to read array of points from file

def readPointsFromFileBody():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1body", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayB.append(temporaryList)

            #Function to read array of points from file
def readPointsFromFileLeft():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1leftwing", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayL.append(temporaryList)

#Function to read array of points from file
def readPointsFromFileN():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1neck", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayN.append(temporaryList)

            #Function to read array of points from file

  #Function to read array of points from file
def readPointsFromFileTA():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1taila", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTA.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTB():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1tailb", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTB.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTC():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1tailc", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTC.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTD():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1taild", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTD.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTE():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1taile", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTE.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTF():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1tailf", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTF.append(temporaryList)

  #Function to read array of points from file
def readPointsFromFileTG():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1tailg", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayTG.append(temporaryList)

def readPointsFromFileH():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1head", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayH.append(temporaryList)

def readPointsFromFileHA():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1ha", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayHA.append(temporaryList)

def readPointsFromFileHB():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1hb", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArrayHB.append(temporaryList)

def readPointsFromFile2():
    #File mode = read
    mode = 'r'
    #Open file
    with open("sampleDragon1", mode) as fileObject:
        #Read each line
        for line in fileObject:
            #Split line to space separated integer and insert the numbers to a temporary list
            temporaryList = [int(num) for num in line.split()]
            #Append the list to the main array
            fourTupleArray2.append(temporaryList)

#Function to print the main array for debugging purpose
def printTupleArray():
    for line in fourTupleArray:
        print(line[0])
        print(line[1])
        print(line[2])
        print(line[3])
        print()

#Main program

#Read data from file
readPointsFromFile2()
readPointsFromFileRight()
readPointsFromFileLeft()
readPointsFromFileBody()
readPointsFromFileN()
readPointsFromFileTA()
readPointsFromFileTB()
readPointsFromFileTC()
readPointsFromFileTD()
readPointsFromFileTE()
readPointsFromFileTF()
readPointsFromFileTG()
readPointsFromFileH()
readPointsFromFileHA()
readPointsFromFileHB()
#Initiate glut windoe
glutInit(sys.argv)
#Set display mode to single using RGB coloring method
glutInitDisplayMode (GLUT_SINGLE | GLUT_RGB)
#Set initial window size to
glutInitWindowSize (windowWidth, windowHeight)
#Set initial window position to bottom left
glutInitWindowPosition (0, 0)
#Create window with desired name
glutCreateWindow ("Grafika Ilustrasi Naga")
#Pop the window
initiate2DWindow(0.0,0.0,0.0)
#Render the display
glutDisplayFunc(renderDisplay)
#Run glut main loop to keep the window open until it's closed
glutMainLoop()


import arcade

# --- Constants ---
SCREEN_WIDTH = 1080
SCREEN_HEIGHT = 720
MOVEMENT_SPEED = 5

def dibujar_fondo():
    """ Draw the ground """
    arcade.draw_lrtb_rectangle_filled(0, SCREEN_WIDTH, SCREEN_HEIGHT / 3, 0, arcade.color.ARYLIDE_YELLOW)

class Dibujos:
    def __init__(self, position_x, position_y, change_x, change_y, radius):
        self.position_x = position_x
        self.position_y = position_y
        self.change_x = change_x
        self.change_y = change_y
        self.radius = radius

    def update(self):
        # Move the ball
        self.position_y += self.change_y
        self.position_x += self.change_x
        # See if the ball hit the edge of the screen. If so, change direction
        if self.position_x < self.radius:
            self.position_x = self.radius

        if self.position_x > SCREEN_WIDTH - self.radius:
            self.position_x = SCREEN_WIDTH - self.radius

        if self.position_y < self.radius:
            self.position_y = self.radius

        if self.position_y > SCREEN_HEIGHT - self.radius:
            self.position_y = SCREEN_HEIGHT - self.radius

    def dibujo_personaje(self):
        arcade.draw_rectangle_filled(self.position_x + 100, self.position_y, 4, 40, arcade.csscolor.BLACK)  # torso
        arcade.draw_circle_filled(self.position_x + 100, self.position_y + 20, 15, arcade.csscolor.SIENNA)  # cara
        arcade.draw_circle_filled(self.position_x + 95, self.position_y + 25, 2.5, arcade.csscolor.BLACK)  # ojo
        arcade.draw_circle_filled(self.position_x + 110, self.position_y + 25, 2.5, arcade.csscolor.BLACK)  # ojo
        arcade.draw_line(self.position_x + 100, self.position_y - 5, self.position_x + 125, self.position_y - 5,
                         arcade.color.BLACK, 3)
        arcade.draw_line(self.position_x + 100, self.position_y - 5, self.position_x + 75, self.position_y - 5,
                         arcade.color.BLACK, 3)
        arcade.draw_line(self.position_x + 100, self.position_y - 20, self.position_x + 110, self.position_y - 60,
                         arcade.color.BLACK, 3)
        arcade.draw_line(self.position_x + 100, self.position_y - 20, self.position_x + 90, self.position_y - 60,
                         arcade.color.BLACK, 3)

    def dibujo_camello(self):
        arcade.draw_line(self.position_x + 10, 235, self.position_x + 30, 215, arcade.color.BLACK, 4)
        arcade.draw_line(self.position_x - 10, 235, self.position_x - 30, 215, arcade.color.BLACK, 4)
        arcade.draw_ellipse_filled(self.position_x, 250, 70, 40, arcade.csscolor.SANDY_BROWN)  # cuerpo
        arcade.draw_circle_filled(self.position_x + 30, 270, 15, arcade.csscolor.SANDY_BROWN)  # cabeza
        arcade.draw_circle_filled(self.position_x + 25, 280, 2, arcade.csscolor.BLACK)
        arcade.draw_circle_filled(self.position_x + 40, 280, 2, arcade.csscolor.BLACK)

class MyGame(arcade.Window):
    """ Our Custom Window Class"""

    def __init__(self):
        """ Initializer """

        # Call the parent class initializer
        super().__init__(SCREEN_WIDTH, SCREEN_HEIGHT, "Lab 7 - User Control")

        arcade.set_background_color(arcade.color.BLUE)

        self.jugador = Dibujos(250, 270, 0, 0, -100)
        self.enemigo1 = Dibujos(-20, 280, 0, 0, 15)
        self.enemigo2 = Dibujos(0, 260, 0, 0, 15)
        self.enemigo3 = Dibujos(-50, 240, 0, 0, 15)
        self.camello = Dibujos(260, 290, 0, 0, 15)

    def on_draw(self):
        arcade.start_render()
        dibujar_fondo()
        self.jugador.dibujo_personaje()
        self.camello.dibujo_camello()
        self.enemigo1.dibujo_personaje()
        self.enemigo2.dibujo_personaje()
        self.enemigo3.dibujo_personaje()

    def update(self, delta_time):
        self.jugador.update()
        self.enemigo1.update()

    def on_key_press(self, key, modifiers):
        """ Called whenever the user presses a key. """
        if key == arcade.key.LEFT:
            self.jugador.change_x = -MOVEMENT_SPEED
        if key == arcade.key.RIGHT:
            self.jugador.change_x = MOVEMENT_SPEED
        if key == arcade.key.UP:
            self.jugador.change_y = MOVEMENT_SPEED
        if key == arcade.key.DOWN:
            self.jugador.change_y = -MOVEMENT_SPEED

        if key == arcade.key.A:
            self.enemigo1.change_x = -MOVEMENT_SPEED
        if key == arcade.key.D:
            self.enemigo1.change_x = MOVEMENT_SPEED
        if key == arcade.key.W:
            self.enemigo1.change_y = MOVEMENT_SPEED
        if key == arcade.key.S:
            self.enemigo1.change_y = -MOVEMENT_SPEED

    def on_key_release(self, key, modifiers):
        """ Called whenever a user releases a key. """
        if key == arcade.key.LEFT or key == arcade.key.RIGHT:
            self.jugador.change_x = 0
        if key == arcade.key.UP or key == arcade.key.DOWN:
            self.jugador.change_y = 0

        if key == arcade.key.A or key == arcade.key.D:
            self.enemigo1.change_x = 0
        if key == arcade.key.W or key == arcade.key.S:
            self.enemigo1.change_y = 0

def main():
    window = MyGame()
    arcade.run()

main()


#!/usr/bin/env python3
#
# Copyright 2019 PSB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""modeling_get_protein_mass_mapping.py

Command-line interface for get_protein_mass_mapping
"""

# IMPORTS
# External modules
import click
import cobra
from typing import Dict, Tuple, Union
# Internal modules
from submodules.apply_manual_changes import apply_manual_changes

# Set-up console arguments using click decorators
@click.command()
@click.option("--sbml_path_input",
              required=True,
              type=click.Path(exists=True, file_okay=True, dir_okay=True),
              prompt="Input SBML",
              help="Full path to the SBML of which the kcats shall be changed manually")
@click.option("--sbml_path_output",
              required=True,
              type=click.Path(exists=True, dir_okay=True),
              prompt="Output SBML",
              help="Full path to the newly created SBML with the manual kcat changes")
@click.option("--kcat_change_factors",
              required=True,
              type=str,
              prompt="Description of the manual kcat changes:",
              help="Textual description of the manual kcat changes. The format of this description is: . Basic format: basereaction_id,reaction_direction,change_factor;(...) "
                   "Example: if one wants to lower the kcat of the reverse reaction of ACALD by the factor 10, and to highen the kcat of the irreversible reaction CBD by the factor 5, then the textual description of the changes would be: "
                   "ACALD,reverse,1/10;CBD,,5")
# Actual CLI function
def apply_manual_changes_cli(sbml_path_input: str, sbml_path_output: str, kcat_change_factors: str) -> None:
    """Applies manually given kcat changes to the given sMOMENT model.

    This application is given by multiplying the stoichiometry of the protein pool pseudo-metabolite in a reaction with the
    inverse of the given manual change factor. The *inverse* is used as it is intended to change the kcat and as the protein
    pool stoichiometry is MW/kcat (MW is the molecular weight).

    Example
    ----------
    Suppose we want to lower the kcat of the reverse reaction of ACALD by the factor 10, and to highen the kcat of the irreversible reaction CBD by
    the factor 5, and the SBML of the model which shall be changed has the path 'C:\\models\\model.xml' and the newly created SBML with the manual
    changes has the path 'C:\\models\\model_new.xml', then the command would be:
    <pre>
    python optimization_apply_manual_changes --sbml_path_input C:\\models\\model.xml  --sbml_path_output C:\\models\\model_new.xml
    --kcat_change_factors ACALD,reverse,1/10;CBD,,5
    </pre>
    """
    model = cobra.io.read_sbml_model(sbml_path_input)
    kcat_change_factors_dict: Dict[str, Tuple[str, Union[float, int]]] = {}
    single_changes = kcat_change_factors.split(";")
    for single_change in single_changes:
        reaction = single_change.split(",")[0]
        direction = single_change.split(",")[1]
        change_factor = float(single_change.split(",")[2])
        kcat_change_factors_dict[reaction] = (direction, change_factor)
    model = apply_manual_changes(model, kcat_change_factors_dict)
    cobra.io.write_sbml_model(sbml_path_output, sbml_path_output)

# Start-up routine if script is called
if __name__ == '__main__':
    # Thanks to the click decorators, the command-line interface
    # function does not need to be called directly. The given
    # console arguments are added automatically.
    apply_manual_changes_cli()
    pass


import requests

import re

def genericErrorInfo():
   import os, sys
   exc_type, exc_obj, exc_tb = sys.exc_info()
   fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
   errorMessage = fname + ', ' + str(exc_tb.tb_lineno) + ', ' + str(sys.exc_info())
   print('\tERROR:', errorMessage)
   return errorMessage

urlList = []
with open("URLTable.txt") as inp:
    for line in inp:
        urlList.append(line)
'''
url = "http://www.odu.edu/compsci"
uHTML = requests.get(url).text
urlID = "CSTest"
print(uHTML, file=open(urlID+"RawHtml2.html", "w+"))
'''

urlTable = [['' for x in range(2)] for y in range(len(urlList))]

for x in range(len(urlList)):
    urlTable[x][0], temp, urlTable[x][1] = urlList[x].split(' ')

for x in range(len(urlTable)):
    try:
        uHTML = requests.get(urlTable[x][1]).text
        urlID = urlTable[x][0]
        #uHTML = requests.get("http://www.odu.edu/compsci").text
        print(uHTML, file=open(urlID+"RawHtml.html", "w+"))
        print(urlID, " Succeeded")
    except:
        print(genericErrorInfo(), file=open(urlID + "RawHtml.html", "w+"))
        print(urlID, " Failed")
        genericErrorInfo()


import argparse

import pytorch_lightning as pl
import torch
import torchmetrics as tm
import torchvision.transforms as tt
from torch.utils.data import DataLoader
from uniplot import plot

import satellighte as sat

def parse_arguments():
    """
    Parse command line arguments.

    Returns: Parsed arguments
    """
    arg = argparse.ArgumentParser()

    arg.add_argument(
        "--max_epoch",
        type=int,
        default=10,
        help="Maximum number of epochs to train the model",
    )
    arg.add_argument(
        "--device",
        type=int,
        default=1 if torch.cuda.is_available() else 0,
        choices=[0, 1],
        help="GPU device to use",
    )
    arg.add_argument(
        "--arch",
        type=str,
        default=sat.available_archs()[0],
        choices=sat.available_archs(),
        help="Model architecture",
    )
    arg.add_argument(
        "--config",
        type=str,
        default=sat.get_arch_configs(sat.available_archs()[0])[0],
        help="Model configuration",
    )
    arg.add_argument(
        "--save_dir",
        type=str,
        default="./checkpoints",
    )
    arg.add_argument(
        "--data_dir",
        type=str,
        default="satellighte/datas/resisc45",
        help="Path to the dataset",
    )
    arg.add_argument(
        "--batch_size",
        type=int,
        default=128,
        help="Batch size for training",
    )
    arg.add_argument(
        "--eval_batch_size",
        type=int,
        default=256,
        help="Batch size for evaluation",
    )
    arg.add_argument(
        "--num_workers",
        type=int,
        default=3,
        help="Number of workers for data loading",
    )
    arg.add_argument(
        "--seed",
        type=int,
        default=13,
        help="Seed for reproducibility",
    )
    arg.add_argument(
        "--overfit",
        action="store_true",
        help="Overfit the model",
    )

    # Hyperparameters
    arg.add_argument(
        "--criterion",
        type=str,
        default="cross_entropy",
        choices=["cross_entropy"],
        help="Criterion to use for training",
    )
    arg.add_argument(
        "--optimizer",
        type=str,
        default="sgd",
        choices=["sgd", "adam", "adamw"],
        help="Optimizer to use for training",
    )
    arg.add_argument(
        "--learning_rate",
        type=float,
        default=1e-2,
        help="Learning rate for training",
    )
    arg.add_argument(
        "--find_learning_rate",
        action="store_true",
        help="Find Learning rate for training",
    )
    arg.add_argument(
        "--momentum",
        type=float,
        default=0.9,
        help="Momentum for training",
    )
    arg.add_argument(
        "--weight_decay",
        type=float,
        default=3e-4,
        help="Weight decay for training",
    )
    arg.add_argument(
        "--betas",
        type=tuple,
        default=(0.9, 0.999),
        help="Betas for Adam optimizer",
    )
    arg.add_argument(
        "--eps",
        type=float,
        default=1e-08,
        help="Epsilon for Adam optimizer",
    )
    arg.add_argument(
        "--scheduler",
        type=str,
        default="steplr",
        choices=["steplr", "multisteplr"],
        help="Scheduler to use for training",
    )
    arg.add_argument(
        "--step_size",
        type=float,
        default=4,
        help="Step size for scheduler",
    )
    arg.add_argument(
        "--gamma",
        type=float,
        default=0.5,
        help="Gamma for scheduler",
    )
    arg.add_argument(
        "--milestones",
        type=list,
        default=[500000, 1000000, 1500000],
        help="Milestones for scheduler",
    )
    return arg.parse_args()

def main(args):
    """
    Main function that training the model.

    Args:
        args : Parsed arguments
    """

    # Set seed for reproducibility
    pl.seed_everything(args.seed, workers=True)

    model = sat.Classifier.build(args.arch, config=args.config)
    input_size = model.input_size

    # Build transforms for training, testing and validation
    train_tt = tt.Compose(
        [
            tt.Resize(input_size),
            tt.ColorJitter(
                brightness=0.2,
                contrast=0.2,
                saturation=0.15,
                hue=0.05,
            ),
            tt.RandomHorizontalFlip(
                p=0.4,
            ),
            tt.RandomRotation(
                degrees=15,
            ),
            tt.GaussianBlur(
                kernel_size=(5, 9),
                sigma=(0.1, 5),
            ),
            # tt.RandomPerspective(
            # distortion_scale=0.1,
            # p=0.2,
            # ),
            # tt.RandomPosterize(
            # bits=2,
            # ),
        ]
    )
    test_tt = tt.Compose(
        [
            tt.Resize(input_size),
        ]
    )
    val_tt = tt.Compose(
        [
            tt.Resize(input_size),
        ]
    )

    # Load datasets from the sat.dataset module
    train_ds = sat.datasets.RESISC45(
        root_dir=args.data_dir,
        phase="train",
        transforms=train_tt,
    )
    test_ds = sat.datasets.RESISC45(
        root_dir=args.data_dir,
        phase="test",
        transforms=test_tt,
    )
    val_ds = sat.datasets.RESISC45(
        root_dir=args.data_dir,
        phase="val",
        transforms=val_tt,
    )

    # Create a DataLoader for training and validation
    train_dl = DataLoader(
        train_ds,
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=args.num_workers,
    )

    test_dl = DataLoader(
        test_ds,
        batch_size=args.eval_batch_size,
        shuffle=False,
        num_workers=args.num_workers,
    )

    val_dl = DataLoader(
        val_ds,
        batch_size=args.eval_batch_size,
        shuffle=False,
        num_workers=args.num_workers,
    )

    # Define hyperparameters
    hparams = {
        "learning_rate": args.learning_rate,
        "momentum": args.momentum,
        "weight_decay": args.weight_decay,
        "milestones": [500000, 1000000, 1500000],
        "gamma": 0.1,
        "ratio": 10,
        "criterion": args.criterion,
    }

    # Build the model with given architecture and configuration with randomly initialized weights.
    model = sat.Classifier.build(
        args.arch,
        config=args.config,
        hparams=hparams,
        labels=train_ds.classes,
    )

    # If find_learning_rate is True, find the learning rate for the model
    if args.find_learning_rate:
        trainer = pl.Trainer(gpus=args.device, auto_lr_find=True, max_epochs=100)

        lr_finder = trainer.tuner.lr_find(
            model, train_dataloaders=train_dl, val_dataloaders=[val_dl]
        )
        plot(
            xs=lr_finder.results["lr"],
            ys=lr_finder.results["loss"],
            title="Learning Rate Finder Result",
            legend_labels=["Learning Rate", "Loss"],
            color=True,
            height=30,
            width=60,
        )
        print(f"Suggested Learning Rate: {lr_finder.suggestion()}")
        del model
        del trainer
        hparams["learning_rate"] = lr_finder.suggestion()
        model = sat.Classifier.build(
            args.arch,
            config=args.config,
            hparams=hparams,
            labels=train_ds.classes,
        )

    # Saved model name
    model_save_name = f"{args.arch}_{args.config}_best"

    # Define metrics
    model.add_metric("accuracy", tm.Accuracy())

    # Define checkpoint callback
    checkpoint_callback = pl.callbacks.ModelCheckpoint(
        dirpath=args.save_dir,  # Path to save the model
        verbose=True,  # Whether to print information about the model checkpointing
        filename=model_save_name,  # Filename to save the model
        monitor="metrics/accuracy",  # Quantity to monitor
        save_top_k=1,  # Number of models to keep
        mode="max",  # Save max metric value
    )

    # Define trainer
    trainer = pl.Trainer(
        default_root_dir=".",  # Default path for logs and weights
        accumulate_grad_batches=5,  # Accumulates grads every k batches
        callbacks=[
            checkpoint_callback,
            pl.callbacks.RichProgressBar(),
        ],  # Callback for checkpointing
        gpus=args.device,  # GPU device
        max_epochs=args.max_epoch,  # Stop training once this number of epochs is reached
        check_val_every_n_epoch=1,  # Check validation every n train epochs.
        deterministic=True,  # Set to True to disable randomness in the model
        overfit_batches=(
            0.01 if args.overfit else 0.0
        ),  # Overfit on a small number of batches
    )

    # Fit the model
    trainer.fit(model, train_dataloaders=train_dl, val_dataloaders=val_dl)

    # Evaluate the model
    trainer.test(dataloaders=test_dl)

if __name__ == "__main__":
    args = parse_arguments()
    main(args)


import sys
sys.path.append('../rt_erg_lib')

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import animation
from tqdm import tqdm

from single_integrator_3d import SingleIntegrator
from ergodic_control import RTErgodicControl
from target_dist_3d import TargetDist
from utils import convert_phi2phik, convert_ck2dist, convert_traj2ck, convert_phik2phi

env = SingleIntegrator()
model = SingleIntegrator()
t_dist = TargetDist(means=[0.2, 0.8, 3], num_nodes=2)
weights = {'R':np.diag([5 for _ in range(2)])}
erg_ctrl = RTErgodicControl(model, t_dist, horizon=15, num_basis=10, batch_size=-1, weights=weights)
erg_ctrl.phik = convert_phi2phik(erg_ctrl.basis, t_dist.grid_vals, t_dist.grid)

tf = 200
state = env.reset()
log = {'trajectory' : [state.copy()]}

for t in tqdm(range(tf)):
    ctrl = erg_ctrl(state, turtle_mode=False)
    state = env.step(ctrl)
    log['trajectory'].append(state)
print('doneee')
traj = np.array(log['trajectory'])

fig, ax = plt.subplots(1, 1)
ax.set_aspect('equal')

xy, vals = t_dist.get_grid_spec()
ax.contourf(*xy, vals, levels=20)
traj_plot = ax.scatter([], [], s=10, c='r')
pointer = ax.plot([], [], color='orange')

def animate_at_t(t, traj_plot, pointer, traj_data, show_traj=True):
    if show_traj is True:
        traj_plot.set_offsets(traj_data[:t])
    else:
        traj_plot.set_offsets(traj_data[t])
    print(traj_data[t])
    pointer[0].set_xdata([traj_data[t][0], traj_data[t][0]+0.05])#*np.cos(traj_data[t][2])])
    pointer[0].set_ydata([traj_data[t][1], traj_data[t][1]+0.05])#*np.sin(traj_data[t][2])])
    return [traj_plot, pointer[0]]

anim = animation.FuncAnimation(fig, animate_at_t, frames=tf, interval=100, fargs=(traj_plot, pointer, traj, True), repeat=True)

plt.show()

"""
xy, vals = t_dist.get_grid_spec()
lamk = np.exp(-0.8*np.linalg.norm(erg_ctrl.basis.k, axis=1))

num_plts = 4
fig, axs = plt.subplots(2, num_plts, sharex=True, sharey=True)

for i in range(num_plts):
    tt = int(tf/num_plts*(i+1))
    axs[0,i].set_aspect('equal')
    axs[0,i].set_title('Time: {}'.format( tt ), fontsize=20)
    axs[0,i].contourf(*xy, vals, levels=20)
    xt = np.stack(log['trajectory'])
    axs[0,i].scatter(xt[:tt,0], xt[:tt,1], c='red', s=2)
    axs[0,i].axis('off')

    axs[1,i].set_aspect('equal')
    path = np.stack(log['trajectory'])[:tt,model.explr_idx]
    ck = convert_traj2ck(erg_ctrl.basis, path)
    val = convert_ck2dist(erg_ctrl.basis, ck)
    axs[1,i].contourf(*xy, val.reshape(50,50), levels=20)
    axs[1,i].axis('off')

plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.04, right=0.96)
# plt.tight_layout()
plt.show()
plt.close()

fdiffs = []
for i in tqdm(range(tf)):
    path = np.stack(log['trajectory'])[:i+1,model.explr_idx]
    ck = convert_traj2ck(erg_ctrl.basis, path)
    fdiff = lamk * (ck - erg_ctrl.phik)**2
    fdiff = np.sum(fdiff.reshape(-1,1))
    fdiffs.append(fdiff)
plt.plot(np.arange(tf), fdiffs)
plt.title('Receding-horizon ergodic cost', fontsize=20)
plt.xlabel('Time', fontsize=20)
plt.ylabel('Ergodic Metric', fontsize=20)
plt.tick_params(axis='both', which='major', labelsize=20)
plt.show()
"""


# -*- coding:utf-8 -*-
"""
@contact: adonis_wu@outlook.com
@time: 2019-08-05
"""
__author__ = 'adowu'
from typing import List

def insertion_sort(nums: List[int], length: int, reverse: bool) -> List[int]:
    """
    O(n^2)
    :param nums: unsorted array
    :param length: array length
    :param reverse: increase or decrease
    :return: sorted array
    """
    for i in range(length):
        curr_value = nums[i]
        j = i - 1
        while j >= 0:
            flag = nums[j] < curr_value if reverse else nums[j] > curr_value
            if flag:
                nums[j + 1] = nums[j]
            else:
                break
            j -= 1
        nums[j + 1] = curr_value

    print(nums)
    return nums

insertion_sort([1, 3, 2, 5, 6, 4, 1, 0, 3, 5, 1, 9, 8, 6, 5], 15, False)


#!/usr/bin/env python
"""
Watches pending jobs and starts or creates EC2 instances if required
"""
import re
import time
import random
from collections import defaultdict

try:
    import simplejson as json
    assert json
except ImportError:
    import json

import boto.ec2
from boto.exception import BotoServerError
from boto.ec2.networkinterface import NetworkInterfaceCollection, \
    NetworkInterfaceSpecification
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
import sqlalchemy as sa
from sqlalchemy.engine.reflection import Inspector

import requests
import os
import logging
from bid import decide as get_spot_choices

log = logging.getLogger()

def find_pending(db):
    inspector = Inspector(db)
    # Newer buildbot has a "buildrequest_claims" table
    if "buildrequest_claims" in inspector.get_table_names():
        query = sa.text("""
        SELECT buildername, id FROM
               buildrequests WHERE
               complete=0 AND
               submitted_at > :yesterday AND
               submitted_at < :toonew AND
               (select count(brid) from buildrequest_claims
                       where brid=id) = 0""")
    # Older buildbot doesn't
    else:
        query = sa.text("""
        SELECT buildername, id FROM
               buildrequests WHERE
               complete=0 AND
               claimed_at=0 AND
               submitted_at > :yesterday AND
               submitted_at < :toonew""")

    result = db.execute(
        query,
        yesterday=time.time() - 86400,
        toonew=time.time() - 10
    )
    retval = result.fetchall()
    return retval

def find_retries(db, brid):
    """Returns the number of previous builds for this build request id"""
    q = sa.text("SELECT count(*) from builds where brid=:brid")
    return db.execute(q, brid=brid).fetchone()[0]

# Used by aws_connect_to_region to cache connection objects per region
_aws_cached_connections = {}

def aws_connect_to_region(region, secrets):
    """Connect to an EC2 region. Caches connection objects"""
    if region in _aws_cached_connections:
        return _aws_cached_connections[region]
    conn = boto.ec2.connect_to_region(
        region,
        aws_access_key_id=secrets['aws_access_key_id'],
        aws_secret_access_key=secrets['aws_secret_access_key']
    )
    _aws_cached_connections[region] = conn
    return conn

def aws_get_spot_requests(region, secrets, moz_instance_type):
    """retruns a list of all open and active spot requests"""
    conn = aws_connect_to_region(region, secrets)
    filters = {"tag:moz-type": moz_instance_type}
    req = conn.get_all_spot_instance_requests(filters=filters)
    return [r for r in req if r.state in ("open", "active")]

def aws_get_all_instances(regions, secrets):
    """
    Returns a list of all instances in the given regions
    """
    log.debug("fetching all instances for %s", regions)
    retval = []
    for region in regions:
        conn = aws_connect_to_region(region, secrets)
        reservations = conn.get_all_instances()
        for r in reservations:
            retval.extend(r.instances)
    return retval

def aws_filter_instances(all_instances, state=None, tags=None):
    retval = []
    for i in all_instances:
        matched = True
        if state and i.state != state:
            matched = False
            continue
        if tags:
            for k, v in tags.items():
                if i.tags.get(k) != v:
                    matched = False
                    continue
        if i.tags.get("moz-loaned-to"):
            # Skip loaned instances
            matched = False
            continue
        if matched:
            retval.append(i)
    return retval

def aws_get_reservations(regions, secrets):
    """
    Return a mapping of (availability zone, ec2 instance type) -> count
    """
    log.debug("getting reservations for %s", regions)
    retval = {}
    for region in regions:
        conn = aws_connect_to_region(region, secrets)
        reservations = conn.get_all_reserved_instances(filters={
            'state': 'active',
        })
        for r in reservations:
            az = r.availability_zone
            ec2_instance_type = r.instance_type
            if (az, ec2_instance_type) not in retval:
                retval[az, ec2_instance_type] = 0
            retval[az, ec2_instance_type] += r.instance_count
    return retval

def aws_filter_reservations(reservations, running_instances):
    """
    Filters reservations by reducing the count for reservations by the number
    of running instances of the appropriate type. Removes entries for
    reservations that are fully used.

    Modifies reservations in place
    """
    # Subtract running instances from our reservations
    for i in running_instances:
        if (i.placement, i.instance_type) in reservations:
            reservations[i.placement, i.instance_type] -= 1
    log.debug("available reservations: %s", reservations)

    # Remove reservations that are used up
    for k, count in reservations.items():
        if count <= 0:
            log.debug("all reservations for %s are used; removing", k)
            del reservations[k]

def aws_resume_instances(moz_instance_type, start_count, regions, secrets,
                         region_priorities, instance_type_changes, dryrun):
    """Resume up to `start_count` stopped instances of the given type in the
    given regions"""
    # Fetch all our instance information
    all_instances = aws_get_all_instances(regions, secrets)
    # We'll filter by these tags in general
    tags = {'moz-state': 'ready', 'moz-type': moz_instance_type}

    # If our instance config specifies a maximum number of running instances,
    # apply that now. This may mean that we reduce start_count, or return early
    # if we're already running >= max_running
    instance_config = json.load(open("configs/%s" % moz_instance_type))
    max_running = instance_config.get('max_running')
    if max_running is not None:
        running = len(aws_filter_instances(all_instances, state='running',
                                           tags=tags))
        if running + start_count > max_running:
            start_count = max_running - running
            if start_count <= 0:
                log.info("max_running limit hit (%s - %i)",
                         moz_instance_type, max_running)
                return 0

    # Get our list of stopped instances, sorted by region priority, then
    # launch_time. Higher region priorities mean we'll prefer to start those
    # instances first
    def _instance_sort_key(i):
        # Region is (usually?) the placement with the last character dropped
        r = i.placement[:-1]
        if r not in region_priorities:
            log.warning("No region priority for %s; az=%s; "
                        "region_priorities=%s", r, i.placement,
                        region_priorities)
        p = region_priorities.get(r, 0)
        return (p, i.launch_time)
    stopped_instances = list(reversed(sorted(
        aws_filter_instances(all_instances, state='stopped', tags=tags),
        key=_instance_sort_key)))
    log.debug("stopped_instances: %s", stopped_instances)

    # Get our current reservations
    reservations = aws_get_reservations(regions, secrets)
    log.debug("current reservations: %s", reservations)

    # Get our currently running instances
    running_instances = aws_filter_instances(all_instances, state='running')

    # Filter the reservations
    aws_filter_reservations(reservations, running_instances)
    log.debug("filtered reservations: %s", reservations)

    # List of (instance, is_reserved) tuples
    to_start = []

    # While we still have reservations, start instances that can use those
    # reservations first
    for i in stopped_instances[:]:
        k = (i.placement, i.instance_type)
        if k not in reservations:
            continue
        stopped_instances.remove(i)
        to_start.append((i, True))
        reservations[k] -= 1
        if reservations[k] <= 0:
            del reservations[k]

    # Add the rest of the stopped instances
    to_start.extend((i, False) for i in stopped_instances)

    # Limit ourselves to start only start_count instances
    log.debug("starting up to %i instances", start_count)
    log.debug("to_start: %s", to_start)

    started = 0
    for i, is_reserved in to_start:
        r = "reserved instance" if is_reserved else "instance"
        if not dryrun:
            log.debug("%s - %s - starting %s", i.placement, i.tags['Name'], r)
            try:
                # Check if the instance type needs to be changed. See
                # watch_pending.cfg.example's instance_type_changes entry.
                new_instance_type = instance_type_changes.get(
                    i.region.name, {}).get(moz_instance_type)
                if new_instance_type and new_instance_type != i.instance_type:
                    log.warn("Changing %s (%s) instance type from %s to %s",
                             i.tags['Name'], i.id, i.instance_type,
                             new_instance_type)
                    i.connection.modify_instance_attribute(
                        i.id, "instanceType", new_instance_type)
                i.start()
                started += 1
            except BotoServerError:
                log.debug("Cannot start %s", i.tags['Name'], exc_info=True)
                log.warning("Cannot start %s", i.tags['Name'])
        else:
            log.info("%s - %s - would start %s", i.placement, i.tags['Name'],
                     r)
            started += 1
        if started >= start_count:
            log.debug("Started %s instaces, breaking early", started)
            break

    return started

def request_spot_instances(moz_instance_type, start_count, regions, secrets,
                           region_priorities, spot_config, dryrun,
                           cached_cert_dir):
    started = 0
    spot_rules = spot_config.get("rules", {}).get(moz_instance_type)
    if not spot_rules:
        log.warn("No spot rules found for %s", moz_instance_type)
        return 0

    instance_config = json.load(open("configs/%s" % moz_instance_type))
    connections = []
    for region in regions:
        connections.append(aws_connect_to_region(region, secrets))
    spot_choices = get_spot_choices(connections, spot_rules)
    if not spot_choices:
        log.warn("No spot choices for %s", moz_instance_type)
        return 0

    to_start = {}
    for region in regions:
        # Check if spots are enabled in this region for this type
        region_limit = spot_config.get("limits", {}).get(region, {}).get(
            moz_instance_type)
        if not region_limit:
            log.debug("No spot limits defined for %s in %s, skipping...",
                      moz_instance_type, region)
            continue

        # check the limits
        active_count = len(aws_get_spot_requests(
            region=region, secrets=secrets,
            moz_instance_type=moz_instance_type))
        can_be_started = region_limit - active_count
        if can_be_started < 1:
            log.debug("Not starting. Active spot request count in %s region "
                      "hit limit of %s. Active count: %s", region,
                      region_limit, active_count)
            continue

        to_be_started = min(can_be_started, start_count - started)
        ami = get_ami(region=region, secrets=secrets,
                      moz_instance_type=moz_instance_type)
        to_start[region] = {"ami": ami, "instances": to_be_started}

    if not to_start:
        log.debug("Nothing to start for %s", moz_instance_type)
        return 0

    for choice in spot_choices:
        region = choice.region
        if region not in to_start:
            log.debug("Skipping %s for %s", choice, region)
            continue
        need = min(to_start[region]["instances"], start_count - started)
        log.debug("Need %s of %s in %s", need, moz_instance_type,
                  choice.availability_zone)

        log.debug("Using %s", choice)
        launched = do_request_spot_instances(
            amount=need,
            region=region, secrets=secrets,
            moz_instance_type=moz_instance_type,
            ami=to_start[region]["ami"],
            instance_config=instance_config, dryrun=dryrun,
            cached_cert_dir=cached_cert_dir,
            spot_choice=choice)
        started += launched

        if started >= start_count:
            break

    return started

def get_puppet_certs(ip, secrets, cached_cert_dir):
    """ reuse or generate certificates"""
    cert_file = os.path.join(cached_cert_dir, ip)
    if os.path.exists(cert_file):
        return open(cert_file).read()

    puppet_server = secrets["getcert_server"]
    url = "https://%s/deploy/getcert.cgi?%s" % (puppet_server, ip)
    auth = (secrets["deploy_username"], secrets["deploy_password"])
    req = requests.get(url, auth=auth, verify=False)
    req.raise_for_status()
    cert_data = req.content
    with open(cert_file, "wb") as f:
        f.write(cert_data)
    return cert_data

def do_request_spot_instances(amount, region, secrets, moz_instance_type, ami,
                              instance_config, cached_cert_dir, spot_choice,
                              dryrun):
    started = 0
    for _ in range(amount):
        try:
            do_request_spot_instance(
                region=region, secrets=secrets,
                moz_instance_type=moz_instance_type,
                price=spot_choice.bid_price,
                availability_zone=spot_choice.availability_zone,
                ami=ami, instance_config=instance_config,
                cached_cert_dir=cached_cert_dir,
                instance_type=spot_choice.instance_type, dryrun=dryrun)
            started += 1
        except (RuntimeError):
            log.warn("Cannot start", exc_info=True)
    return started

def do_request_spot_instance(region, secrets, moz_instance_type, price, ami,
                             instance_config, cached_cert_dir, instance_type,
                             availability_zone, dryrun):
    conn = aws_connect_to_region(region, secrets)
    interface = get_available_interface(
        conn=conn, moz_instance_type=moz_instance_type,
        availability_zone=availability_zone)
    if not interface:
        raise RuntimeError("No free network interfaces left in %s" % region)

    # TODO: check DNS
    fqdn = interface.tags.get("FQDN")
    if not fqdn:
        raise RuntimeError("Skipping %s without FQDN" % interface)

    log.debug("Spot request for %s (%s)", fqdn, price)

    if dryrun:
        log.info("Dry run. skipping")
        return

    spec = NetworkInterfaceSpecification(
        network_interface_id=interface.id)
    nc = NetworkInterfaceCollection(spec)
    ip = interface.private_ip_address
    certs = get_puppet_certs(ip, secrets, cached_cert_dir)
    user_data = """
FQDN="%(fqdn)s"
cd /var/lib/puppet/ssl || exit 1
%(certs)s
cd -
""" % dict(fqdn=fqdn, certs=certs)
    if instance_config[region].get("lvm"):
        user_data += """
mkdir -p /etc/lvm-init/
cat <<EOF > /etc/lvm-init/lvm-init.json
%s
EOF
/sbin/lvm-init
""" % json.dumps(instance_config[region])

    bdm = BlockDeviceMapping()
    for device, device_info in instance_config[region]['device_map'].items():
        bd = BlockDeviceType()
        if device_info.get('size'):
            bd.size = device_info['size']
        if ami.root_device_name == device:
            ami_size = ami.block_device_mapping[device].size
            if ami.virtualization_type == "hvm":
                # Overwrite root device size for HVM instances, since they
                # cannot be resized online
                bd.size = ami_size
            elif device_info.get('size'):
                # make sure that size is enough for this AMI
                assert ami_size <= device_info['size'], \
                    "Instance root device size cannot be smaller than AMI " \
                    "root device"
        if device_info.get("delete_on_termination") is not False:
            bd.delete_on_termination = True
        if device_info.get("ephemeral_name"):
            bd.ephemeral_name = device_info["ephemeral_name"]

        bdm[device] = bd

    sir = conn.request_spot_instances(
        price=str(price),
        image_id=ami.id,
        count=1,
        instance_type=instance_type,
        key_name=instance_config[region]["ssh_key"],
        user_data=user_data,
        block_device_map=bdm,
        network_interfaces=nc,
        instance_profile_name=instance_config[region].get("instance_profile_name"),
    )
    sir[0].add_tag("moz-type", moz_instance_type)

_cached_interfaces = {}

def get_available_interface(conn, moz_instance_type, availability_zone):
    global _cached_interfaces
    if not _cached_interfaces.get(availability_zone):
        _cached_interfaces[availability_zone] = {}
    if _cached_interfaces[availability_zone].get(moz_instance_type) is None:
        filters = {
            "status": "available",
            "tag:moz-type": moz_instance_type,
            "availability-zone": availability_zone,
        }
        avail_ifs = conn.get_all_network_interfaces(filters=filters)
        if avail_ifs:
            random.shuffle(avail_ifs)
        _cached_interfaces[availability_zone][moz_instance_type] = avail_ifs

    log.debug("%s interfaces in %s",
              len(_cached_interfaces[availability_zone][moz_instance_type]),
              availability_zone)
    if _cached_interfaces[availability_zone][moz_instance_type]:
        return _cached_interfaces[availability_zone][moz_instance_type].pop()
    else:
        return None

def get_ami(region, secrets, moz_instance_type):
    conn = aws_connect_to_region(region, secrets)
    avail_amis = conn.get_all_images(
        owners=["self"],
        filters={"tag:moz-type": moz_instance_type})
    last_ami = sorted(avail_amis,
                      key=lambda ami: ami.tags.get("moz-created"))[-1]
    return last_ami

def aws_watch_pending(dburl, regions, secrets, builder_map, region_priorities,
                      spot_config, dryrun, cached_cert_dir,
                      instance_type_changes):
    # First find pending jobs in the db
    db = sa.create_engine(dburl)
    pending = find_pending(db)

    # Mapping of instance types to # of instances we want to creates
    to_create_ondemand = defaultdict(int)
    to_create_spot = defaultdict(int)
    # Then match them to the builder_map
    for pending_buildername, brid in pending:
        for buildername_exp, instance_type in builder_map.items():
            if re.match(buildername_exp, pending_buildername):
                if find_retries(db, brid) == 0:
                    to_create_spot[instance_type] += 1
                else:
                    to_create_ondemand[instance_type] += 1
                break
        else:
            log.debug("%s has pending jobs, but no instance types defined",
                      pending_buildername)

    for instance_type, count in to_create_spot.items():
        log.debug("need %i spot %s", count, instance_type)
        started = request_spot_instances(
            moz_instance_type=instance_type, start_count=count,
            regions=regions, secrets=secrets,
            region_priorities=region_priorities, spot_config=spot_config,
            dryrun=dryrun, cached_cert_dir=cached_cert_dir)
        count -= started
        log.info("%s - started %i spot instances; need %i",
                 instance_type, started, count)

        # Add leftover to ondemand
        to_create_ondemand[instance_type] += count

    for instance_type, count in to_create_ondemand.items():
        log.debug("need %i ondemand %s", count, instance_type)
        if count < 1:
            continue

        # Check for stopped instances in the given regions and start them if
        # there are any
        started = aws_resume_instances(instance_type, count, regions, secrets,
                                       region_priorities,
                                       instance_type_changes, dryrun)
        count -= started
        log.info("%s - started %i instances; need %i",
                 instance_type, started, count)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-r", "--region", action="append", dest="regions",
                        required=True)
    parser.add_argument("-k", "--secrets", type=argparse.FileType('r'),
                        required=True)
    parser.add_argument("-c", "--config", type=argparse.FileType('r'),
                        required=True)
    parser.add_argument("-v", "--verbose", action="store_const",
                        dest="loglevel", const=logging.DEBUG,
                        default=logging.INFO)
    parser.add_argument("--cached-cert-dir", default="certs",
                        help="Directory for cached puppet certificates")
    parser.add_argument("-n", "--dryrun", dest="dryrun", action="store_true",
                        help="don't actually do anything")

    args = parser.parse_args()

    logging.basicConfig(level=args.loglevel,
                        format="%(asctime)s - %(message)s")
    logging.getLogger("boto").setLevel(logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARN)

    config = json.load(args.config)
    secrets = json.load(args.secrets)

    aws_watch_pending(
        dburl=secrets['db'],
        regions=args.regions,
        secrets=secrets,
        builder_map=config['buildermap'],
        region_priorities=config['region_priorities'],
        dryrun=args.dryrun,
        spot_config=config.get("spot"),
        cached_cert_dir=args.cached_cert_dir,
        instance_type_changes=config.get("instance_type_changes", {})
    )


import numpy as np
import torch
from torch import nn
from sklearn.metrics import precision_score,accuracy_score,f1_score,recall_score
import torch.nn.functional as F

def smooth_l1_loss(prediction, label):
    return torch.nn.SmoothL1Loss()(prediction,label)

def wce_dice_huber_loss(pred, gt):
    # print(gt.shape)
    # print(pred.shape)
    loss1 = cross_entropy_loss(pred, gt)
    loss2 = DiceLoss()(pred, gt)
    # loss3 = smooth_l1_loss(pred, gt)
    return 0.6 * loss1 + 0.4 * loss2

def wce_dice_huber_loss_stage2(pred1,pred2,gt):

    loss1 = cross_entropy_loss_stage2(pred2, gt, mask=pred1)
    # loss2 = DiceLoss()(pred2, gt)
    return loss1

class DiceLoss(nn.Module):
    def __init__(self):
        super(DiceLoss, self).__init__()

    def forward(self, input, target):
        N = target.size(0)

        smooth = 1

        input_flat = input.view(N, -1)
        target_flat = target.view(N, -1)

        intersection = input_flat * target_flat

        loss = 2 * (intersection.sum(1) + smooth) / (input_flat.sum(1) + target_flat.sum(1) + smooth)
        loss = 1 - loss.sum() / N

        return loss

class BinaryDiceLoss(nn.Module):
    """Dice loss of binary class
    Args:
        smooth: A float number to smooth loss, and avoid NaN error, default: 1
        p: Denominator value: \sum{x^p} + \sum{y^p}, default: 2
        predict: A tensor of shape [N, *]
        target: A tensor of shape same with predict
        reduction: Reduction method to apply, return mean over batch if 'mean',
            return sum if 'sum', return a tensor of shape [N,] if 'none'
    Returns:
        Loss tensor according to arg reduction
    Raise:
        Exception if unexpected reduction
    """
    def __init__(self, smooth=1, p=2, reduction='mean'):
        super(BinaryDiceLoss, self).__init__()
        self.smooth = smooth
        self.p = p
        self.reduction = reduction

    def forward(self, predict, target):
        assert predict.shape[0] == target.shape[0], "predict & target batch size don't match"
        predict = predict.contiguous().view(predict.shape[0], -1)
        target = target.contiguous().view(target.shape[0], -1)

        num = torch.sum(torch.mul(predict, target), dim=1) + self.smooth
        den = torch.sum(predict.pow(self.p) + target.pow(self.p), dim=1) + self.smooth

        loss = 1 - num / den

        if self.reduction == 'mean':
            return loss.mean()
        elif self.reduction == 'sum':
            return loss.sum()
        elif self.reduction == 'none':
            return loss
        else:
            raise Exception('Unexpected reduction {}'.format(self.reduction))

#
# class DiceLoss(nn.Module):
#     """Dice loss, need one hot encode input
#     Args:
#         weight: An array of shape [num_classes,]
#         ignore_index: class index to ignore
#         predict: A tensor of shape [N, C, *]
#         target: A tensor of same shape with predict
#         other args pass to BinaryDiceLoss
#     Return:
#         same as BinaryDiceLoss
#     """
#     def __init__(self, weight=None, ignore_index=None, **kwargs):
#         super(DiceLoss, self).__init__()
#         self.kwargs = kwargs
#         self.weight = weight
#         self.ignore_index = ignore_index
#
#     def forward(self, predict, target):
#         assert predict.shape == target.shape, 'predict & target shape do not match'
#         dice = BinaryDiceLoss(**self.kwargs)
#         total_loss = 0
#         predict = F.softmax(predict, dim=1)
#
#         for i in range(target.shape[1]):
#             if i != self.ignore_index:
#                 dice_loss = dice(predict[:, i], target[:, i])
#                 if self.weight is not None:
#                     assert self.weight.shape[0] == target.shape[1], \
#                         'Expect weight shape [{}], get[{}]'.format(target.shape[1], self.weight.shape[0])
#                     dice_loss *= self.weights[i]
#                 total_loss += dice_loss
#
#         return total_loss/target.shape[1]

def sigmoid_cross_entropy_loss(prediction, label):
    label = label.long()
    mask = (label != 0).float()
    num_positive = torch.sum(mask).float()
    num_negative = mask.numel() - num_positive
    #print (num_positive, num_negative)
    mask[mask != 0] = num_negative / (num_positive + num_negative)
    mask[mask == 0] = num_positive / (num_positive + num_negative)
    cost = torch.nn.functional.binary_cross_entropy_with_logits(
            prediction.float(),label.float(), weight=mask, reduce=False)
    return torch.sum(cost)

class FocalLoss(nn.Module):
    def __init__(self, gamma=2, weight=None, reduction='mean'):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.weight = weight
        self.reduction = reduction

    def forward(self, output, target):
        # convert output to pseudo probability
        out_target = torch.stack([output[i, t] for i, t in enumerate(target)])
        probs = torch.sigmoid(out_target)
        focal_weight = torch.pow(1 - probs, self.gamma)

        # add focal weight to cross entropy
        ce_loss = F.cross_entropy(output, target, weight=self.weight, reduction='none')
        focal_loss = focal_weight * ce_loss

        if self.reduction == 'mean':
            focal_loss = (focal_loss / focal_weight.sum()).sum()
        elif self.reduction == 'sum':
            focal_loss = focal_loss.sum()

        return focal_loss
def cross_entropy_loss(prediction, label):
    label = label.long()
    mask = (label != 0).float()

    num_positive = torch.sum(mask).float()
    num_negative = mask.numel() - num_positive
    # print (num_positive, num_negative)
    mask[mask != 0] = num_negative / (num_positive + num_negative) # 0.995
    mask[mask == 0] = num_positive / (num_positive + num_negative) # 0.005
    cost = torch.nn.functional.binary_cross_entropy(
            prediction.float(), label.float(), weight=mask)
    return cost
def cross_entropy_loss_stage2(prediction, label, mask):
    label = label.long()
    mask = (mask >= 0.2).float()
    cost = torch.nn.functional.binary_cross_entropy(
            prediction.float(), label.float(), weight=mask)
    return cost

def map8_loss_ce(prediction, label):
    cost = cross_entropy_loss(prediction,label)
    return cost

def weighted_nll_loss(prediction, label):
    label = torch.squeeze(label.long(), dim=0)
    nch = prediction.shape[1]
    label[label >= nch] = 0
    cost = torch.nn.functional.nll_loss(prediction, label, reduce=False)
    mask = (label != 0).float()
    num_positive = torch.sum(mask).float()
    num_negative = mask.numel() - num_positive
    mask[mask != 0] = num_negative / (num_positive + num_negative)
    mask[mask == 0] = num_positive / (num_positive + num_negative)
    cost = torch.mul(cost, mask)
    return torch.sum(cost)/(cost.size()[0]*cost.size()[1]*cost.size()[2]*cost.size()[3])

def weighted_cross_entropy_loss(prediction, label, output_mask=False):
    criterion = torch.nn.CrossEntropyLoss(reduce=False)
    label = torch.squeeze(label.long(), dim=0)
    nch = prediction.shape[1]
    label[label >= nch] = 0
    cost = criterion(prediction, label)
    mask = (label != 0).float()
    num_positive = torch.sum(mask).float()
    num_negative = mask.numel() - num_positive
    mask[mask == 1] = num_negative / (num_positive + num_negative)
    mask[mask == 0] = num_positive / (num_positive + num_negative)
    cost = torch.mul(cost, mask)
    if output_mask:
        return torch.sum(cost), (label != 0)
    else:
        return torch.sum(cost)

def l1_loss(prediction,label):
    return torch.nn.L1Loss()(prediction,label)

def CE_loss(prediction,label):
    cost = torch.nn.functional.binary_cross_entropy(prediction,label)
    return torch.sum(cost)

def wce_huber_loss(prediction,label):
    loss1 = cross_entropy_loss(prediction,label)
    loss2 = smooth_l1_loss(prediction,label)
    loss3 = l1_loss(prediction,label)
    return 0.6*loss1+0.35*loss2+0.05*loss3

def wce_huber_loss_8(prediction,label):
    loss1 = cross_entropy_loss(prediction,label)
    return loss1
def my_precision_score(prediction,label):
    y = prediction.reshape(-1)
    l = label.reshape(-1)
    y = np.array(y.cpu().detach())
    y = np.where(y > 0.5, 1, 0).astype('int')
    l = np.array(l.cpu().detach()).astype('int')
    return precision_score(y_pred=y,y_true=l, zero_division=1)

def my_acc_score(prediction,label):
    y = prediction.reshape(-1)
    l = label.reshape(-1)
    y = np.array(y.cpu().detach())
    y = np.where(y > 0.5, 1, 0).astype('int')
    l = np.array(l.cpu().detach()).astype('int')
    return accuracy_score(y_pred=y,y_true=l)

def my_f1_score(prediction,label):

    y = prediction.reshape(-1)
    l = label.reshape(-1)

    y = np.array(y.cpu().detach())
    y = np.where(y > 0.5, 1, 0).astype('int')
    l = np.array(l.cpu().detach()).astype('int')

    return f1_score(y_pred=y,y_true=l,zero_division=1)

def my_recall_score(prediction,label):

    y = prediction.reshape(-1)
    l = label.reshape(-1)
    y = np.array(y.cpu().detach())
    y = np.where(y > 0.5, 1, 0).astype('int')
    l = np.array(l.cpu().detach()).astype('int')

    return recall_score(y_pred=y,y_true=l, zero_division=1)

if __name__ == '__main__':
    a = torch.randn(1, 1, 320,320)
    b = np.random.randn(1,1,320,320)
    b = np.where(b>0.9,1,0).astype('float')
    b = torch.from_numpy(b)
    # b = torch.randn(1, 1, 320, 320)
    a = nn.Sigmoid()(a)
    loss = cross_entropy_loss(a,b)
    print(loss)