import numpy as np
import pandas as pd
import os
from PIL import Image
import seaborn as sns
import torch
import torch.functional as F
from torch.utils.data import DataLoader,Dataset
from torchvision.transforms import Normalize,CenterCrop,Resize,Compose
from torchvision import transforms
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import copy
import adabound

expt = "100kwasser3"

#######DATA Transforms
### Normalize the data
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

data_dir = 'MURA-v1.1/data/'
# new_data_dir = 'mura0/'
train_data = datasets.ImageFolder(data_dir+'train',
                                    transform=data_transforms['train'])
val_data = datasets.ImageFolder(data_dir + 'val',
                                    transform=data_transforms['val'])

train_loader=DataLoader(dataset=train_data,batch_size=64,num_workers=1,shuffle=True)
val_loader=DataLoader(dataset=val_data,batch_size=64,num_workers=1,shuffle=True)

dataloaders={
    'train':train_loader,
    'val':val_loader
}

dataset_sizes={
    'train':len(train_data),
    'val':len(val_data)
}

from sklearn import metrics

def metric(ori, pred):
    lis_ori = []
    lis_pred = []
    for i in ori:
        lis_ori.append(i.item())
    for i in pred:
        lis_pred.append(i.item())
    fpr, tpr, thresholds = metrics.roc_curve(lis_ori, lis_pred)
    roc_auc = metrics.auc(fpr, tpr)
    print(fpr, tpr, thresholds, roc_auc)

    plt.figure()
    lw = 2
    plt.plot(fpr, tpr, color='darkorange',
             lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
#     plt.title('ROC for digit=%d class' % 2)
    plt.legend(loc="lower right")
    plt.savefig("ROC_curve"+str(expt)+".png")

    #plt.show()

train_ac = []
train_pred = []
val_ac = []
val_pred = []

train_prob = []
val_prob = []

def train_model(model, criterion, optimizer, scheduler, num_epochs=1):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    old_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                print(phase, epoch)
                model.train()  # Set model to training mode
            else:
                print(phase, epoch)
                model.eval()  # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # if (phase == 'train'):
                    #     train_pred.extend(preds)
                    #     train_ac.extend(labels.view_as(preds))
                    #     train_prob.extend(np.exp(outputs.detach().numpy()[:, 1]))

                    if (phase == 'val'):
                        val_pred.extend(preds)
                        val_ac.extend(labels.view_as(preds))
                        val_prob.extend(np.exp(outputs.detach().numpy()[:, 1]))

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                #                 print(loss)
                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            checkpoint = {
                'epoch': epoch + 1,
                'valid_loss_min': epoch_acc,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(), }

            if (phase == 'val'):
                #metric(val_ac, val_prob)
                print(metrics.classification_report(val_ac, val_pred))
                print(metrics.confusion_matrix(val_ac, val_pred))

            # Save best model
            if (epoch_acc > old_acc):
                print(old_acc, epoch_acc)
                old_acc = epoch_acc
                file = 'model_retrained'+str(expt)+'.pth'
                torch.save(checkpoint, file)

                #if (phase == 'val'):
                    #metric(val_ac, val_prob)
                    #print(metrics.classification_report(val_ac, val_pred))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

def load_model():
    #model = models.resnet18(pretrained=True)
    #for param in model.parameters():
        #param.requires_grad = False
    #print("Model Loaded Resnet18")
    #model.fc = nn.Sequential(
               #nn.Linear(512, 2))#,
               #nn.ReLU(inplace=True),
               #nn.Linear(128, 2))
    model=models.resnet50(pretrained=True)
    for param in model.parameters():
        param.requires_grad = False

    model.fc = nn.Sequential(
               nn.Linear(2048, 128),
               nn.ReLU(inplace=True),
               nn.Linear(128, 2))

    return model

model = load_model()
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = model.to(device)
checkpoint = torch.load("model_syntheticsyn_wasser1.pth")
model.load_state_dict(checkpoint['state_dict'])

print("Start Retraining the Model")

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
#optimizer = optim.Adam(model.fc.parameters(), lr=0.0001)
#optimizer = optim.SGD(model.parameters(), lr=0.005, momentum=0.5)
optimizer = adabound.AdaBound(model.parameters(), lr=0.00005, final_lr=0.0005)

# Decay LR by a factor of 0.1 every 5 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)

model_ft = train_model(model, criterion, optimizer, exp_lr_scheduler,
                       num_epochs=100)


import copy
import logging

from six import iteritems

import base
from pgoapi.protos.POGOProtos import Enums_pb2

logger = logging.getLogger(__name__)

class ReleaseMethod(base.ReleaseMethod):

    def processConfig(self, config):
        self.config = config
        self.releaseMethodFactory = base.ReleaseMethodFactory({})
        self.baseConfig = copy.deepcopy(config)
        if 'RELEASE_METHOD_MULTI' in self.baseConfig:
            del self.baseConfig['RELEASE_METHOD_MULTI']

        self.handlers = {}
        self.multiConfig = config.get('RELEASE_METHOD_MULTI', {})
        self.DEFAULT_RELEASE_METHOD = self.multiConfig.get('MULTI_DEFAULT_RELEASE_METHOD', "CLASSIC")

        # build default config and override with the config values in its section in base config then override with
        # config values in its section in the multi config
        self.defaultConfig = copy.deepcopy(self.baseConfig)
        self.defaultConfig = base.filtered_dict_merge(self.defaultConfig, self.multiConfig, "POKEMON_CONFIGS")
        self.defaultConfig['RELEASE_METHOD'] = self.DEFAULT_RELEASE_METHOD
        self.defaultHandler = self.releaseMethodFactory.loadReleaseMethod(self.DEFAULT_RELEASE_METHOD, self.defaultConfig)

        for pokemonName, pokemonConfig in iteritems(self.multiConfig.get('POKEMON_CONFIGS', {})):
            pokeId = getattr(Enums_pb2, pokemonName)
            releaseMethod = pokemonConfig.get('RELEASE_METHOD', self.DEFAULT_RELEASE_METHOD)
            # initialize the config with the default config
            cfg = copy.deepcopy(self.defaultConfig)
            # apply pokemon specific overrides
            cfg = base.filtered_dict_merge(cfg, pokemonConfig)
            self.handlers[pokeId] = self.releaseMethodFactory.loadReleaseMethod(releaseMethod, cfg)

    def getPokemonToRelease(self, pokemonId, pokemons):
        if pokemonId in self.handlers:
            return self.handlers[pokemonId].getPokemonToRelease(pokemonId, pokemons)
        else:
            return self.defaultHandler.getPokemonToRelease(pokemonId, pokemons)


import torch.nn as nn

from BaseCNN import BaseConvNet

class AlexNet(BaseConvNet):
    def __init__(self, nClasses):
        features = nn.Sequential(
            # first layer
            nn.Conv2d(3, 96, 11, stride=4),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(3, stride=2),
            # second layer
            nn.Conv2d(96, 256, 5, stride=1, padding=2),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(3, stride=2),
            # third layer
            nn.Conv2d(256, 384, 3, padding=1),
            nn.ReLU(inplace=False),
            # fourth layer
            nn.Conv2d(384, 384, 3, padding=1),
            nn.ReLU(inplace=False),
            # fifth layer
            nn.Conv2d(384, 256, 3, padding=1),
            nn.ReLU(inplace=False),
            nn.MaxPool2d(3, stride=2))

        classifier = nn.Sequential(
            nn.Flatten(),
            nn.Linear(6*6*256, 4096, bias=True),
            nn.ReLU(inplace=True),
            nn.Linear(4096, 4096, bias=True),
            nn.ReLU(inplace=True),
            nn.Linear(4096, nClasses, bias=True)
        )

        super().__init__(features, classifier)


#! python3

# lucky.py - Opens several Google search results.
import requests, sys, webbrowser, bs4

print('NC lookup...') # display text while downloading
res = requests.get('https://www.bbc.co.uk/sport/football/teams/norwich-city')
res.raise_for_status()

# Retrieve top search result links.
results_page = bs4.BeautifulSoup(res.text)

#Find the score element
test = results_page.select('#live-scores-index-container')
print(test)


from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
import unittest
import login_disney_class as login

jun = "http://watchdisneyjunior.go.com/activate"
xd = "http://watchdisneyxd.go.com/activate"
chan = "http://watchdisneychannel.go.com/activate"

class ActivateDisneyXD(login.ActivateDisney):
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.get(xd)

class ActivateDisneyJun(login.ActivateDisney):
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.get(jun)

class ActivateDisneyChan(login.ActivateDisney):
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.get(chan)

if __name__ == '__main__':
    unittest.main()


import numpy as np
from numpy import random as rand
import utils
from scipy import sparse as sps
import os
utils.__file__
# DO NOT CHANGE THE NAME OF THIS METHOD OR ITS INPUT OUTPUT BEHAVIOR

# INPUT CONVENTION
# X: n x d matrix in csr_matrix format containing d-dim (sparse) features for n test data points
# k: the number of recommendations to return for each test data point in ranked order

# OUTPUT CONVENTION
# The method must return an n x k numpy nd-array (not numpy matrix or scipy matrix) of labels with the i-th row
# containing k labels which it thinks are most appropriate for the i-th test point. Labels must be returned in
# ranked order i.e. the label yPred[i][0] must be considered most appropriate followed by yPred[i][1] and so on

# CAUTION: Make sure that you return (yPred below) an n x k numpy (nd) array and not a numpy/scipy/sparse matrix
# The returned matrix will always be a dense matrix and it terribly slows things down to store it in csr format
# The evaluation code may misbehave and give unexpected results if an nd-array is not returned

def using_tocoo(x):
    f1 = open("PfastreXML/Data/test_data.X","w+")
    (n, d) = x.shape
    s=""
    s+=("%d %d"%(n,d))
    cx = x.tocoo()
    prev = -1
    for (i,j,v) in zip(cx.row,cx.col,cx.data):
        if(i == prev):
            s+=(" %d:%f"%(j,v))
        else:
            while(prev!=i):
                s+=("\n")
                if(prev%2000==0):
                    f1.write(s)
                    s=""
                prev+=1
            s+=("%d:%f"%(j,v))
    s+=("\n")
    f1.write(s)
    f1.close()
    return

def getReco( X, k ):
    # Find out how many data points we have
    (n, d) = X.shape
    using_tocoo(X)
    os.system("./predict_script.sh")
    fi=open('PfastreXML/Results/score_mat.txt','r')
    n,d=tuple(fi.readline().split(' '))
    n,d =int(n),int(d)
    arr = np.ndarray((n,k),dtype=int)
    for j,i in enumerate(fi.readlines()):
        l=list(map(lambda x:tuple(x.split(':')),i.split(' ')))
        arr[j]=list(map(lambda x:int(x[0]),(sorted(l,key=lambda x:float(x[1]),reverse=True))))[:k]
    fi.close()
    return arr


import os
import sys
from datetime import timedelta
from collections import Counter
from GenericUtils.Utils import Utils
from LoggingUtils.Logging import Logging
from SemiStructuredDataAccess.SemiStructuredDataAccess import SemiStructuredDataAccess

class BigNews():
    def __init__(self):
        """
        :Date: 2017-01-17
        :Version: 0.2
        :Author: Jan Alejandro Medina - Pontificia Universidad Javeriana Cali

        Constructor for the class
        """
        Logging.configure_log('BigNewsLog')
        self.semi_structured_access = SemiStructuredDataAccess('BigNews', ['database'])

    def counter_news(self, date_search, time_type):

        """
        :Date: 2017-01-16
        :Version: 0.1
        :Author: Jan Alejandro Medina

        This function counts words for a determined time (week, month, year).

        :param date_search: Start date to count the stem words.
        :type date_search: string

        :param time_type: Time weekly, monthly or annually
        :type time_type: str

        :return: Return a json where ha a count of word in a week
        :rtype: json
        """
        try:
            dic_count = dict()
            result_count = dict()

            end_date = Utils.convert_python_string_to_date(date_search)
            word_counter = Counter()
            if time_type == 'daily':
                start_date = end_date - timedelta(days = 1)
            elif time_type == 'weekly':
                start_date = end_date - timedelta(days = 6)
            elif time_type == 'monthly':
                start_date = end_date - timedelta(days = 30)
            elif time_type == 'annually':
                start_date = end_date - timedelta(days = 365)

            cursor = self.semi_structured_access.get_from_database("conversation", "find",
                                                                   ["conversation.processed_text.words", start_date,
                                                                    end_date])

            for word in cursor:
                word_dic = word['conversation']['processed_text']['words']
                N = len(word_dic)
                for i in range(1, N):
                    stem = word_dic[i]['stem']
                    word_counter.update(stem.split(" "))
            top = dict(word_counter.most_common(10))
            freq = dict(word_counter)
            N = sum(freq.values())
            relative_freq = map(lambda x: x /N, freq.values())

            for i,j in zip(freq.items(),relative_freq):
                result_count.setdefault(i[0],{'freq':i[1],'rel_freq':j})
            dic_count.setdefault('start_date', start_date)
            dic_count.setdefault('end_date',end_date)
            dic_count.setdefault('time_type', time_type)
            dic_count.setdefault('result', result_count)
            dic_count.setdefault('top_ten', top)
            self.semi_structured_access.insert_into_database("words_frequency", dic_count)
        except:
            Logging.write_standard_error(sys.exc_info())


# -*- coding: utf-8 -*-
"""
Profile: http://hl7.org/fhir/StructureDefinition/Signature
Release: STU3
Version: 3.0.2
Revision: 11917
Last updated: 2019-10-24T11:53:00+11:00
"""
from typing import Any, Dict
from typing import List as ListType

from pydantic import Field, root_validator

from . import element, fhirtypes

class Signature(element.Element):
    """Disclaimer: Any field name ends with ``__ext`` does't part of
    Resource StructureDefinition, instead used to enable Extensibility feature
    for FHIR Primitive Data Types.

    A digital Signature - XML DigSig, JWT, Graphical image of signature, etc..
    A digital signature along with supporting context. The signature may be
    electronic/cryptographic in nature, or a graphical image representing a
    hand-written signature, or a signature process. Different signature
    approaches have different utilities.
    """

    resource_type = Field("Signature", const=True)

    blob: fhirtypes.Base64Binary = Field(
        None,
        alias="blob",
        title="The actual signature content (XML DigSig. JWT, picture, etc.)",
        description=(
            "The base64 encoding of the Signature content. When signature is not "
            "recorded electronically this element would be empty."
        ),
        # if property is element of this resource.
        element_property=True,
    )
    blob__ext: fhirtypes.FHIRPrimitiveExtensionType = Field(
        None, alias="_blob", title="Extension field for ``blob``."
    )

    contentType: fhirtypes.Code = Field(
        None,
        alias="contentType",
        title="The technical format of the signature",
        description=(
            "A mime type that indicates the technical format of the signature. "
            "Important mime types are application/signature+xml for X ML DigSig, "
            "application/jwt for JWT, and image/* for a graphical image of a "
            "signature, etc."
        ),
        # if property is element of this resource.
        element_property=True,
    )
    contentType__ext: fhirtypes.FHIRPrimitiveExtensionType = Field(
        None, alias="_contentType", title="Extension field for ``contentType``."
    )

    onBehalfOfReference: fhirtypes.ReferenceType = Field(
        None,
        alias="onBehalfOfReference",
        title="The party represented",
        description=(
            "A reference to an application-usable description of the identity that "
            "is represented by the signature."
        ),
        # if property is element of this resource.
        element_property=True,
        # Choice of Data Types. i.e onBehalfOf[x]
        one_of_many="onBehalfOf",
        one_of_many_required=False,
        # note: Listed Resource Type(s) should be allowed as Reference.
        enum_reference_types=[
            "Practitioner",
            "RelatedPerson",
            "Patient",
            "Device",
            "Organization",
        ],
    )

    onBehalfOfUri: fhirtypes.Uri = Field(
        None,
        alias="onBehalfOfUri",
        title="The party represented",
        description=(
            "A reference to an application-usable description of the identity that "
            "is represented by the signature."
        ),
        # if property is element of this resource.
        element_property=True,
        # Choice of Data Types. i.e onBehalfOf[x]
        one_of_many="onBehalfOf",
        one_of_many_required=False,
    )
    onBehalfOfUri__ext: fhirtypes.FHIRPrimitiveExtensionType = Field(
        None, alias="_onBehalfOfUri", title="Extension field for ``onBehalfOfUri``."
    )

    type: ListType[fhirtypes.CodingType] = Field(
        ...,
        alias="type",
        title="Indication of the reason the entity signed the object(s)",
        description=(
            "An indication of the reason that the entity signed this document. This"
            " may be explicitly included as part of the signature information and "
            "can be used when determining accountability for various actions "
            "concerning the document."
        ),
        # if property is element of this resource.
        element_property=True,
    )

    when: fhirtypes.Instant = Field(
        ...,
        alias="when",
        title="When the signature was created",
        description="When the digital signature was signed.",
        # if property is element of this resource.
        element_property=True,
    )
    when__ext: fhirtypes.FHIRPrimitiveExtensionType = Field(
        None, alias="_when", title="Extension field for ``when``."
    )

    whoReference: fhirtypes.ReferenceType = Field(
        None,
        alias="whoReference",
        title="Who signed",
        description=(
            "A reference to an application-usable description of the identity that "
            "signed  (e.g. the signature used their private key)."
        ),
        # if property is element of this resource.
        element_property=True,
        # Choice of Data Types. i.e who[x]
        one_of_many="who",
        one_of_many_required=True,
        # note: Listed Resource Type(s) should be allowed as Reference.
        enum_reference_types=[
            "Practitioner",
            "RelatedPerson",
            "Patient",
            "Device",
            "Organization",
        ],
    )

    whoUri: fhirtypes.Uri = Field(
        None,
        alias="whoUri",
        title="Who signed",
        description=(
            "A reference to an application-usable description of the identity that "
            "signed  (e.g. the signature used their private key)."
        ),
        # if property is element of this resource.
        element_property=True,
        # Choice of Data Types. i.e who[x]
        one_of_many="who",
        one_of_many_required=True,
    )
    whoUri__ext: fhirtypes.FHIRPrimitiveExtensionType = Field(
        None, alias="_whoUri", title="Extension field for ``whoUri``."
    )

    @root_validator(pre=True)
    def validate_one_of_many(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """https://www.hl7.org/fhir/formats.html#choice
        A few elements have a choice of more than one data type for their content.
        All such elements have a name that takes the form nnn[x].
        The "nnn" part of the name is constant, and the "[x]" is replaced with
        the title-cased name of the type that is actually used.
        The table view shows each of these names explicitly.

        Elements that have a choice of data type cannot repeat - they must have a
        maximum cardinality of 1. When constructing an instance of an element with a
        choice of types, the authoring system must create a single element with a
        data type chosen from among the list of permitted data types.
        """
        one_of_many_fields = {
            "onBehalfOf": ["onBehalfOfReference", "onBehalfOfUri"],
            "who": ["whoReference", "whoUri"],
        }
        for prefix, fields in one_of_many_fields.items():
            assert cls.__fields__[fields[0]].field_info.extra["one_of_many"] == prefix
            required = (
                cls.__fields__[fields[0]].field_info.extra["one_of_many_required"]
                is True
            )
            found = False
            for field in fields:
                if field in values and values[field] is not None:
                    if found is True:
                        raise ValueError(
                            "Any of one field value is expected from "
                            f"this list {fields}, but got multiple!"
                        )
                    else:
                        found = True
            if required is True and found is False:
                raise ValueError(f"Expect any of field value from this list {fields}.")

        return values


from django.urls import path
from django.conf import urls
from . import views

urlpatterns = [
    path('', views.rmp, name='rmp'),
    path('accident/', views.accident, name='accident'),
    path('states/', views.state_accidents, name='state_accidents'),
    path('test/', views.test, name='test'),
    path('facility_search/', views.facility_search, name='facility_search'),
    path('location_search/', views.location_search, name='location_search'),
    path('location_search/search_by_state/', views.search_by_state, name="search_by_state"),
    path('location_search/search_by_city/', views.search_by_city, name="search_by_city"),
    path('facility_search/search_by_facility/', views.search_by_facility, name="search_by_facility"),
]


from django.db import models
from django.utils.translation import gettext_lazy as _

from django_extensions.db.fields import AutoSlugField

import os

MEDIA_PATH=os.path.join(os.path.dirname(__file__),'media')
# http://docs.python.org/library/imghdr.html
# http://stackoverflow.com/questions/765396/exif-manipulation-library-for-python

class VideoOnly( models.Model):
    title=models.CharField( verbose_name=_("Title"),
        max_length=255, )
    slug = AutoSlugField( verbose_name=_("Slug"),
        unique=True,
        blank=False,
        null=False,
        max_length=255,
        populate_from="title",
        separator="-",
        overwrite=False, )

    class Meta:
        verbose_name=_("Video Only")
        verbose_name_plural=_("Video Onlys")

class Video( models.Model):
    title=models.CharField( verbose_name=_("Title"),
        max_length=255, )
    slug = AutoSlugField( verbose_name=_("Slug"),
        unique=True,
        blank=False,
        null=False,
        max_length=255,
        populate_from="title",
        separator="-",
        overwrite=False, )

    class Meta:
        verbose_name=_("Video")
        verbose_name_plural=_("Videos")

class Audio(models.Model):
    title=models.CharField( verbose_name=_("Title"),
        max_length=255, )
    slug = AutoSlugField( verbose_name=_("Slug"),
        unique=True,
        blank=False,
        null=False,
        max_length=255,
        populate_from="title",
        separator="-",
        overwrite=False, )

    class Meta:
        verbose_name=_("Audio")
        verbose_name_plural=_("Audios")

class Image( models.Model):
    title=models.CharField( verbose_name=_("Title"),
        max_length=255, )
    slug = AutoSlugField( verbose_name=_("Slug"),
        unique=True,
        blank=False,
        null=False,
        max_length=255,
        populate_from="title",
        separator="-",
        overwrite=False, )

    @models.permalink
    def get_absolute_url( self):
        return "None"

    @property
    def template( self):
        return "mediagrid/partials/image.html"

    image=models.ImageField(upload_to=MEDIA_PATH,
        verbose_name=_("Image"), )

    width=models.BigIntegerField( verbose_name=_("Width"),
        default=0, )

    height=models.BigIntegerField( verbose_name=_("Height"),
        default=0, )

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name=_("Media Item")
        verbose_name_plural=_("Media Items")


import numpy as np
import cv2 as cv
import sys
sys.path.insert(0, '..')
import config as cfg

def get_resized_image(image):
  im = image.astype(np.float32, copy=True)
  target_size = cfg.vgg_size
  im_new = cv.resize(im, (target_size, target_size), interpolation=cv.INTER_LINEAR)
  im_new -= cfg.PIXEL_MEANS
  #im_new /= cfg.PIXEL_SCALE
  return im_new

def get_blobs(img, label):
  im_new = get_resized_image(img)

  #reshape these things to match the network
  #the image needs a channel swap
  channel_swap = (2, 0, 1)
  im_new = im_new.transpose(channel_swap)
  im_new = im_new.reshape((1, im_new.shape[0], im_new.shape[1], im_new.shape[2]))

  label_new = np.array(label).reshape((1,1,1,-1))

  return im_new, label_new


import os
import numpy as np
import matplotlib.pyplot as plt
import cv2
import histomicstk.preprocessing.color_normalization as hist

if __name__ == '__main__':
    pink_path = "/media/krukts/HDD/BioDiploma/Timofey/dataset_PINK_300k_Stage2_WithErrorNormCls/Tumor/Tumor_001_tilec_TUM2_1.png"
    blue_path = "/media/krukts/HDD/BioDiploma/Timofey/dataset_BLUE_300k/Tumor/tumor_071_tilec_TUM2_15.png"

    pink = cv2.imread(pink_path)
    pink = cv2.cvtColor(pink, cv2.COLOR_BGR2RGB)
    blue = cv2.imread(blue_path)
    blue = cv2.cvtColor(blue, cv2.COLOR_BGR2RGB)

    plt.title("PINK")
    plt.imshow(pink)
    plt.show()
    plt.close()

    plt.title("BLUE")
    plt.imshow(blue)
    plt.show()
    plt.close()
    # cv2.imwrite("l.png",pink)

    # Normalization
    result_pink = hist.deconvolution_based_normalization(pink)
    result_blue = hist.deconvolution_based_normalization(blue)

    plt.title("PostPink")
    plt.imshow(result_pink)
    plt.show()
    plt.close()

    plt.title("PostBlue")
    plt.imshow(result_blue)
    plt.show()
    plt.close()
    # cv2.imwrite("pink.png",pink)
    # cv2.imwrite("resultpink.png",result_pink)


import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
#from PIL import Image
from scipy import ndimage

def predict(X,Y,w,b):
    m = X.shape[1]
    Y_prediction = np.zeros((1, m))
    A=sigmoid(np.dot(w.T, X) + b)
    # for x in A:
    #     print(x)
    for i in range(A.shape[1]):

        # Convert probabilities A[0,i] to actual predictions p[0,i]
        ### START CODE HERE ### (≈ 4 lines of code)
        if A[0, i] >= 0.5:
            Y_prediction[0, i] = 1

        else:
            Y_prediction[0, i] = 0
            ### END CODE HERE ###
    assert (Y_prediction.shape == (1, m))

    return Y_prediction

def feedforward(X,Y,w,b):

    m = X.shape[1]
    A = sigmoid(np.dot(w.T, X) + b)

    cost = (-1 / m) * np.sum((Y * np.log(A) + (1 - Y) * np.log(1 - A)), axis=1)

    dw = (1. / m) * np.dot(X, ((A - Y).T))
    db = (1. / m) * np.sum(A - Y, axis=1)
    cost = np.squeeze(cost)

    grads = {"dw": dw,
             "db": db}

    return grads, cost

def sigmoid(z):
    return 1./(1+np.exp(-z))

def load_dataset():
    train_dataset = h5py.File("../Datasets/train_catvnoncat.h5", "r")
    test_dataset = h5py.File("../Datasets/test_catvnoncat.h5", "r")
    train_set_x_orig = np.array(train_dataset["train_set_x"][:])
    train_set_y_orig = np.array(train_dataset["train_set_y"][:])

    test_dataset = h5py.File('../Datasets/test_catvnoncat.h5', "r")
    test_set_x_orig = np.array(test_dataset["test_set_x"][:])  # your test set features
    test_set_y_orig = np.array(test_dataset["test_set_y"][:])  # your test set labels

    classes = np.array(test_dataset["list_classes"][:])  # the list of classes

    train_set_y_orig = train_set_y_orig.reshape(1, train_set_y_orig.shape[0])
    test_set_y_orig = test_set_y_orig.reshape((1, test_set_y_orig.shape[0]))

    return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes

def optimize(X, Y, w, b,l):
    for i in range(2000):
        grad, cost  =feedforward(X=X, Y=Y, w=w, b=b)
        w = w - l * grad["dw"]
        b = b - l * grad["db"]
    weights = {"w": w,
             "b": b
             }
    return weights

if __name__=="__main__":
    train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, classes=load_dataset()
    m_train=train_set_x_orig.shape[0]
    m_test= test_set_x_orig.shape[0]
    train_x=train_set_x_orig.reshape(train_set_x_orig.shape[0],-1).T
    test_x = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T
    w= np.zeros((train_x.shape[0],1),  dtype=np.float32)
    b=0
    l=0.0052
    #for i in range (1,10):
    X=train_x/255.
    X_test=test_x/255.
    Y=train_set_y
    Y_test=test_set_y

    weights=optimize(X,Y,w,b,l)
    y_pred=predict(X,Y,weights["w"],weights["b"])
    y_pred_test = predict(X_test, Y_test, weights["w"], weights["b"])
    print("train accuracy: {} %".format(100 - np.mean(np.abs(y_pred - Y)) * 100))
    print("test accuracy: {} %".format(100 - np.mean(np.abs(y_pred_test - Y_test)) * 100))


# -*- coding:utf-8 -*-
import matplotlib.pyplot as plt
import numpy as np

def loadDataSet(fileName):
    """
    :description load data set from a text file
    :param fileName:
    :return:
        xArr - x Data Set
        yArr - y Data Set
    """
    # obtain the num of dimensions of the data set
    dimension = len(open(fileName).readline().split('\t')) - 1
    xArr = []
    yArr = []

    file = open(fileName)
    for line in file.readlines():
        lineArr = []
        currentLine = line.strip().split('\t')
        for i in range(dimension):
            lineArr.append(float(currentLine[i]))
        xArr.append(lineArr)
        # y - the result set: the last column of the data set
        yArr.append(float(currentLine[-1]))
    return xArr, yArr

def caculateData(xArr, yArr):
    """
    :description Calculate regression coefficient from xArr&yArr
    :param xArr:
    :param yArr:
    :return: ws
    """
    xMat = np.mat(xArr)
    yMat = np.mat(yArr).T
    xTx = xMat.T * xMat
    if np.linalg.det(xTx) == 0.0:
        print("xTx无逆矩阵")
        return

    # 核心公式：回归系数 ws = (xTx)^(-1) * xT * y
    ws = xTx.I * (xMat.T * yMat)
    return ws

def plotRegression():
    """
    :description: draw points of data set and the regression curve
    :return:
    """
    xArr, yArr = loadDataSet('ex0.txt')
    nums = len(yArr)
    ws = caculateData(xArr, yArr)
    xMat = np.mat(xArr)
    yMat = np.mat(yArr)
    # 计算用于绘制回归曲线的x，y值
    curveX = xMat.copy()
    curveX.sort(0)
    curveY = curveX * ws
    # 绘图
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.plot(curveX[:, 1],curveY, c='black')
    ax.scatter(xMat[:, 1].flatten().A[0], yMat.flatten().A[0], s=10, c='blue', alpha=.5)
    plt.title('DataSet')
    plt.xlabel('X')
    plt.show()
    error = calculateError(xMat, yMat, ws, nums)

def calculateError(originX, originY, ws, nums):
    """
    计算原始数据集和预测数据集之间的误差函数
    :param origin:
    :param nums: 数据组个数m
    :param regression:
    :return: error:
    """
    regressionY = originX * ws
    delta = originY.T - regressionY
    squareDelta = np.matrix.tolist(delta.T * delta)[0][0]
    error = 1/nums*squareDelta
    return error

if __name__ == '__main__':
    plotRegression()


import os
from PIL import Image
from scipy.ndimage import gaussian_filter
import astropy.io
from astropy.visualization import astropy_mpl_style
import matplotlib.pyplot as plt
from astropy.io import fits
from astropy.utils.data import get_pkg_data_filename
import numpy as np
import cv2
import matplotlib.image as mpimg

class dustDetection:

  def __init__(self, path_arr):
    self.path = path_arr
    self.img_path_arr = []
    self.img_data = dict()
    self.brightened = dict()
    self.noise_removed_array=dict()

  def extract_fits(self):
    #print('Extracting all fits image details from:\n', self.path)
    print('Extracting all fits image details...')
    self.img_path_arr = os.listdir(self.path)
    for i,item in enumerate(self.img_path_arr):
      hdul = fits.open(self.path+item)
      data = hdul[0].data
      image_flipped = np.flip(data,1)
      self.img_data[self.img_path_arr[i]] = image_flipped[0, :, :]
    print('Number of images in path:', len(self.img_data))
    print('Done!')

  def brighten(self, img_name, change, visible):
    im1 = Image.fromarray(img_name)
    im2 = im1.point(lambda p: p * change)
    if visible:
      plt.figure(dpi=1200)
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(im2, cmap='gray')
      plt.show()
    return np.asarray(im2)

  def noise_removal(self,item, visible):
    f = np.fft.fft2(item)
    fshift=np.fft.fftshift(f)

    #calculate amplitude spectrum
    mag_spec = 20*np.log(np.abs(fshift))
    r= f.shape[0]/2     # number of rows/2
    c=f.shape[1]/2      # number of columns/2
    p=3
    n=1
    fshift2 = np.copy(fshift)
    # suppress upper part
    fshift2[int(r-p):int(r+p),0:int(c-n)] = 0.001
    # suppress lower part
    fshift2[int(r-p):int(r+p), int(c+n):int(c+c)] = 0.001
    # calculate new amplitude spectrum
    mag_spec2 = 20*np.log(np.abs(fshift2))
    inv_fshift=np.fft.ifftshift(fshift2)
    # reconstructing image
    img_recon= np.real(np.fft.ifft2(inv_fshift))
    if visible:
      plt.figure(dpi=1200)
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(img_recon, cmap='gray')
      plt.show()
    return img_recon

  def brighten_images(self, change = 5.0, visible=0):
    print('Smoothening/Brightening the images...')
    for key, value in self.img_data.items():
      self.brightened[key] = self.brighten(value, change, visible)
    print('Done!')

  def noise_remove_folder(self, visible=0):
    print('Removing noise from the images...')
    for key, value in self.brightened.items():
      self.noise_removed_array[key] = self.noise_removal(value, visible)
    print('Done!')

  def edge_detect(self, image, visible):
    t = 100
    img = np.uint8(image)
    gray = cv2.cvtColor(src = img, code = cv2.COLOR_BGR2GRAY)
    blur = cv2.GaussianBlur(src = gray, ksize = (5, 5), sigmaX = 0)
    (t, binary) = cv2.threshold(src = blur,thresh = t, maxval = 255, type = cv2.THRESH_BINARY)
    (_, contours, _) = cv2.findContours(image = binary, mode = cv2.RETR_EXTERNAL,method = cv2.CHAIN_APPROX_SIMPLE)
    print("Found %d objects." % len(contours))
    for (i, c) in enumerate(contours):
      print("\tSize of contour %d: %d" % (i, len(c)))
#   format = np.uint8(image)
#   edge_img = cv2.Canny(format,np.max(format)-100,np.max(format))
    if visible:
      plt.figure(dpi=1200)
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(edge_img, cmap='gray')
      plt.show()
    return edge_img

  def edge_folder(self, path, visible):
    list_all = os.listdir(path)
    for i, img in enumerate(list_all):
      image = Image.open(path+img).convert('L')
      #image = mpimg.imread(path+img)
      #gray = self.rgb2gray(image)
      #image_orig = cv2.imread(img);
      #image = cv2.cvtColor(image_orig, cv2.COLOR_BGR2GRAY)
      #image = self.rgb2gray(image_orig)
      if i == 0:
        self.edge_detect_biary(image, 6, 4, visible) #best=6,4
      else:
        continue
      '''cnt = 0
      for key, value in self.noise_removed_array.items():
        if cnt == 0:
          self.edge_detect_biary(value, 10, 3, visible)
          cnt += 1
        else:
          continue
      '''
  def edge_detect_biary(self, image1, pixel_kernel, pixel_kernel_small, visible):
    #img = np.uint8(image)
    image2 = np.asarray(image1)
    #filter_size = 2*np.ceil(2*4)+1
    image = gaussian_filter(image2,sigma=6, truncate=2) #sigma=6 also 4
    print('shape of image is ', image.shape)
    m,n = image.shape
    image_mod = np.zeros(shape=[m,n])
    for i in range(pixel_kernel,m-pixel_kernel+1):
      for j in range(pixel_kernel,n-pixel_kernel+1):
        mat_1 = image[i-pixel_kernel:i+pixel_kernel,j-pixel_kernel:j+pixel_kernel]
        count = len(mat_1[mat_1 > image[i,j]])
        if (count >= (0.9 * 1.5 * pixel_kernel * pixel_kernel)):
          image_mod[i][j] = 0
        else:
          image_mod[i][j] = 255
    print('max is ', np.max(image_mod))
    print('min is ', np.min(image_mod))
    if visible:
      plt.figure(1)
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(np.uint8(image_mod), cmap='gray')
      plt.title('Modified_0')
      plt.show()

    image_mod_1 = image_mod
    print('initial min of image_mod_1 is ',np.min(image_mod_1))
    print(image_mod_1.dtype)
    count = 0
    for i in range(pixel_kernel_small, m-pixel_kernel_small+1):
      for j in range(pixel_kernel_small, n-pixel_kernel_small+1):
        if image_mod[i][j] == 0:
          mat_2 = image_mod[i-pixel_kernel_small:i+pixel_kernel_small, j-pixel_kernel_small:j+pixel_kernel_small]
          count1 = len(mat_2[mat_2 < 255])
          if count1 >= 25:
            count += 1
            image_mod_1[i-1:i+1, j-1:j+1] = 0
          else:
            image_mod_1[i-1:i+1, j-1:j+1] = 255
    if visible:
      plt.figure(2)
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(np.uint8(image_mod_1), cmap='gray')
      plt.title('Modified_1')
      plt.show()

#  def rgb2gray(self, rgb):
#    return np.dot(rgb[...,:3], [0.299, 0.587, 0.144])

  def average_dark(self,visible=0):
    m,n=self.img_data[self.img_path_arr[0]].shape
    avg_img=np.zeros((m,n))
    for i in range(len(self.img_data)):
#      if i < 4:
      avg_img+=self.img_data[self.img_path_arr[i]]
      print('number of iteration', i)
    avg_img=avg_img/len(self.img_data)
    #avg_img=avg_img/4
    diff = self.img_data[self.img_path_arr[0]] - avg_img
    if visible:
      plt.figure()
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(np.uint16(diff), cmap='gray')
      plt.title('Difference image')
      plt.show()

      plt.figure()
      plt.axis('off')
      plt.grid(b=None)
      plt.imshow(np.uint16(avg_img), cmap='gray')
      plt.title('average')
      plt.show()
    return avg_img


from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from tensorflow.keras.models import load_model
import tensorflow as tf
import time
import os
from io import BytesIO
import urllib
import pathlib
import pymongo
from datetime import datetime
from dotenv import load_dotenv

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
#Disable GPU and force TF to use CPU only
tf.config.set_visible_devices([], 'GPU')
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

load_dotenv(verbose=True, override=True)

class SudokuPredict():

    def __init__(self, file_name=None, version=None):
        self.version = version if version is not None else os.environ['SUDOKU_MODEL_VERSION']
        self.model_file_name = file_name if file_name is not None else os.environ['SUDOKU_MODEL_FILE_NAME']
        print(f"Model will be loaded: {self.model_file_name} with version: {self.version}")

        self.model = load_model('model/' + self.model_file_name)

        mongo_client = pymongo.MongoClient(os.getenv('MONGO_URI'))
        self.mongo_db = mongo_client.sudoku

        self.log_doc = {}

    def __load_image_from_local(self, filepath):
        # load the image from File
        img = load_img(filepath, color_mode = "grayscale", target_size=(28, 28))
        self.log_doc["loaded_img_colormode"] = "grayscale"
        self.log_doc["loaded_img_target_size"] = (28, 28)
        return img

    def __load_image_from_URL(self, URL):
        # load the image from URL
        temp_filename = self.__generate_temp_filename()

        # download image to a temp location
        filepath, _ = urllib.request.urlretrieve(URL, temp_filename)
        img = self.__load_image_from_local(filepath)

        # remove temp image
        os.remove(temp_filename)
        return img

    def get_tensorflow_parameters(self):
        return tf.version.VERSION, tf.version.COMPILER_VERSION

    def __load_image_from_memory(self, image):
        # load the image from ByteIO
        temp_filename = self.__generate_temp_filename()

        pathlib.Path(temp_filename).write_bytes(image.getbuffer())
        img = self.__load_image_from_local(temp_filename)

        os.remove(temp_filename)
        return img

    def __preprocess_image(self, image):
        # convert to array
        img = img_to_array(image)
        self.log_doc["img_to_array"] = img.tolist()

        # reshape into a single sample with 1 channel
        img = img.reshape(1, 28, 28, 1)
        # prepare pixel data
        img = img.astype('float32')
        # invert colors
        img = (255.0 - img)
        img = img / 255.0
        self.log_doc["post_process"] = img.tolist()

        return img

    def __predict(self, img):
        result = self.model.predict(img)[0].tolist()
        self.log_doc["predict_result"] = result

        prob = max(result)
        self.log_doc["predict_prob"] = float(prob)

        digit = result.index(prob)
        self.log_doc["predict_digit"] = int(digit)

        #digit = self.model.predict_classes(img)

        return (int(digit), float(prob))

    def predict_local(self, filepath):
        img = self.__load_image_from_local(filepath)
        img = self.__preprocess_image(img)

        return self.__predict(img)

    def predict_URL(self, URL):
        img = self.__load_image_from_URL(URL)
        img = self.__preprocess_image(img)

        return self.__predict(img)

    def predict_memory(self, bytesio_image, transaction_id=None):
        self.__log_init(transaction_id)

        img = self.__load_image_from_memory(bytesio_image)
        img = self.__preprocess_image(img)
        result = self.__predict(img)

        self.__save_log()
        return result

    def __generate_temp_filename(self):
        return 'tmp/' + str(int(time.time()*1000000)) + ".png"

    def __save_log(self):
        self.mongo_db.predictions_log.insert_one(self.log_doc)
        pass

    def __log_init(self, transaction_id):
        self.log_doc = {}
        self.log_doc["transaction_id"] = transaction_id
        self.log_doc["model_file_name"] = self.model_file_name
        self.log_doc["version"] = self.version
        self.log_doc["datetime"] = datetime.utcnow()


# -*- coding: utf-8 -*-
"""somalier_extract_step module

See domain package __init__.py doc string.
See orchestration package __init__.py doc string.
"""
import logging
from typing import List, Dict

from libumccr.aws import libssm, libsqs

from data_portal.models.workflow import Workflow
from data_processors.pipeline.domain.config import SQS_SOMALIER_EXTRACT_QUEUE_ARN
from data_processors.pipeline.domain.somalier import SomalierReferenceSite
from data_processors.pipeline.domain.workflow import WorkflowType
from data_processors.pipeline.tools import liborca

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def perform(this_workflow: Workflow):

    # prepare job list and dispatch to job queue
    job_list = prepare_somalier_extract_jobs(this_workflow)

    if job_list:
        libsqs.dispatch_jobs(queue_arn=libssm.get_ssm_param(SQS_SOMALIER_EXTRACT_QUEUE_ARN), job_list=job_list)
    else:
        logger.warning(f"Calling to prepare_somalier_extract_jobs() return empty list, no job to dispatch...")

    return {
        "somalier_extract_step": job_list
    }

def prepare_somalier_extract_jobs(this_workflow: Workflow) -> List[Dict]:
    """
    TL;DR is if there is 1 dragen wgs qc or wts or tso workflow, there will be one somalier extraction step workflow
    :param this_workflow:
    :return:
    """

    job_list = []
    gds_bam_path = None
    reference = SomalierReferenceSite.HG38_RNA.value

    if this_workflow.type_name.lower() == WorkflowType.DRAGEN_WTS.value.lower():
        gds_bam_path = liborca.parse_transcriptome_output_for_bam_file(this_workflow.output)

    elif this_workflow.type_name.lower() == WorkflowType.DRAGEN_WGS_QC.value.lower():
        gds_bam_path = liborca.parse_wgs_alignment_qc_output_for_bam_file(this_workflow.output)

    elif this_workflow.type_name.lower() == WorkflowType.DRAGEN_TSO_CTDNA.value.lower():
        gds_bam_path = liborca.parse_tso_ctdna_output_for_bam_file(this_workflow.output)
        reference = SomalierReferenceSite.HG19_RNA.value  # switch reference site to HG19 if ctTSO

    if gds_bam_path is not None:
        job_list.append({
            "index": gds_bam_path,
            "reference": reference
        })

    return job_list


import folium
import pandas as pd

#store info into variable df=data frame
df = pd.read_csv('volcano.txt')

#take the average values to adjust the center point on the map
latmean = df['LAT'].mean()
lonmean = df['LON'].mean()

map=folium.Map(location=[latmean, lonmean], zoom_start=6, tiles='Stamen Terrain')

#function to define color
def color(elev):
    if elev in range(0,1000):
        col= 'green'
    elif elev in range(1001,1999):
        col = 'blue'
    elif elev in range(2000,2999):
        col= 'orange'
    else:
        col= 'red'
    return col

#loop to create markers
#put 4 iterators into zip function
#read as iterator and then where there is a df to "grab" that iterator values

for lat,lon,name,elev in zip(df['LAT'], df['LON'], df['NAME'], df['ELEV']):
    folium.Marker(location=[lat, lon], popup = name, icon = folium.Icon(color=color(elev), icon_color='yellow', icon='cloud')).add_to(map)

print(map.save('volcano_final.html'))


#!python2
# -*- coding: utf-8 -*-
'''
Created on Apr 22, 2014

@author: Wang Jinde
'''

from subprocess import Popen, PIPE
import sys
from exception import AppException
import os
import stat
from datetime import datetime, timedelta
import traceback
import logging

IS_CYGWIN = sys.platform == 'cygwin'

if IS_CYGWIN:
    FS = '\\'
else:
    FS = os.sep

ENCODING = None
if hasattr(sys.stdin, 'encoding'):
    ENCODING = sys.stdin.encoding
if ENCODING is None:
    import locale
    locale_name, ENCODING = locale.getdefaultlocale()
if ENCODING is None:
    ENCODING = "utf8"

CLEARTOOL = 'cleartool'
GIT = 'git'
MAIL_SUFFIX = 'microstrategy.com'

def popen(exe, cmd, cwd, env=None, decode=True, errors=True):
    try:
        cmd.insert(0, exe)

        logger = logging.getLogger(__name__)
        logger.info('> ' + ' '.join(map((lambda a: a if not a.count(' ') else '"%s"' % a), cmd)))

        pipe = Popen(cmd, cwd=cwd, stdout=PIPE, stderr=PIPE, env=env)
        (stdout, stderr) = pipe.communicate()
        if errors and pipe.returncode > 0:
            raise AppException((stderr + stdout).decode(ENCODING))
        return stdout if not decode else stdout.decode(ENCODING)
    finally:
        pipe.stdout.close()
        pipe.stderr.close()

def git_exec(git_dir, cmd, **args):
    return popen(GIT, cmd, git_dir, **args)

def cc_comment_2_git(cc_comment):
    if not cc_comment:
        return ('', '', '')
    cc_comment_content = cc_comment.strip()[2:-2]
    split = cc_comment_content.split(';')
    n_split = len(split)
    if n_split == 1:
        return ('', '', cc_comment_content)
    elif n_split == 2:
        return ('', split[0], split[1])

    git_comment = cc_comment_content[len(split[0]) + len(split[1]) + 2:]
    git_comment = git_comment.replace('\"', '').replace('\'', '')
    return (split[0], split[1], git_comment)

def git_comment_2_cc(team_name, user_name, git_comment):
    '''It converts the git comment to cc comment, and do automatic recover if the format is not correct.'''
    if not git_comment or len(git_comment) == 0:
        git_comment = '00000;no comments'
    else:
        # use the first line of comment if multiple lines included.
        lines = git_comment.splitlines()
        for line in lines:
            if len(line) > 0:
                git_comment = line
                break;

        # we don't need to replace '!' since we alwyas use '{#...#}' format.
        git_comment = git_comment.replace('{', '(').replace('}', ')').replace('#', '$')
        split = git_comment.split(';')
        n_split = len(split)
        if n_split < 2:
            git_comment = '00000;' + git_comment
        elif n_split >= 2:
            issue = split[0]
            if len(issue) == 0:
                issue = '00000'
            comment = git_comment[len(split[0]) + 1:].replace(';', ',')

            if len(comment) == 0:
                comment = 'no comments'

            git_comment = issue + ';' + comment

    return '{#' + team_name + ';' + user_name + ';' + git_comment + '#}'

def merge_git_comments(comments):
    if len(comments) == 0:
        return None

    issue_arr = []
    comment_arr = []
    for comment in comments:
        if comment and len(comment) > 0:
            split = comment.split(';')
            # concatenate comments
            pour_comment = split[-1]
            if len(pour_comment) > 0:
                comment_arr.append(pour_comment)

            # concatenate issue or features
            if len(split) > 1:
                issue = split[0]
                if len(issue) > 0:
                    issue_arr.append(issue)

    return __join_str(issue_arr) + ';' + __join_str(comment_arr)

def merge_cc_comments(comments):
    if not comments or len(comments) == 0:
        return ''

    if len(comments) == 1:
        return comments[0]

    team_arr = []
    user_arr = []
    issue_arr = []
    comment_arr = []
    for cc_comment in comments:
        cc_comment_content = cc_comment.strip()[2:-2]
        split = cc_comment_content.split(';')
        n_split = len(split)

        # ignore invalid comment.
        if n_split != 4:
            continue

        team_arr.append(split[0])
        user_arr.append(split[1])
        issue_arr.append(split[2])
        comment_arr.append(split[3])

    return '{#' + __join_str(team_arr) + ';' + __join_str(user_arr) + ';' + __join_str(issue_arr) + ';' + __join_str(comment_arr) + '#}'

# TODO include use info.
def get_cc_comment(git_commit, user_config):
    '''It get the comment from git commit object, and do automatic recover if the format is not correct.'''
    user_name = git_commit.committer.name
    if user_config and user_config.has_user(user_name):
        team_name = user_config.get_team_name(user_name)
    else:
        # the default team is 'ke'.
        team_name = 'ke'

    git_comment = git_commit.message.strip()
    return git_comment_2_cc(team_name, user_name, git_comment)

def str2int(s):
    if s == None:
        return None

    v = None
    try:
        s = s.strip()
        if len(s) > 0:
            # ignore the leftmost 0s.
            s = s.lstrip('0')
            if len(s) == 0:
                # all digits are 0 here.
                v = 0
            else:
                v = int(s)
    except Exception:
        v = None

    return v

def enum(*sequential, **named):
    '''from http://stackoverflow.com/questions/36932/how-can-i-represent-an-enum-in-python'''
    enums = dict(zip(sequential, range(len(sequential))), **named)
    reverse = dict((value, key) for key, value in enums.iteritems())
    enums['reverse_mapping'] = reverse
    return type('Enum', (), enums)

def write(file_path, blob):
    _write(file_path, blob)

def _write(file_path, blob):
    f = open(file_path, 'wb')
    f.write(blob)
    f.close()

def mkdirs(file_path):
    cur_dir = os.path.dirname(file_path)
    if not os.path.exists(cur_dir):
        os.makedirs(cur_dir)

def removeFile(file_path):
    if os.path.exists(file_path):
        # in windows, the read-only file cannnot be removed with "os.remove".
        file_attr = os.stat(file_path)[0]
        if (not file_attr & stat.S_IWRITE):
            os.chmod(file_path, stat.S_IWRITE)
        os.remove(file_path)

def get_since_utc_time(delta = 0):
    cur_time = datetime.utcnow()
    # we may need go back some time to allow local time incorrectness.
    # It doesn't matter for git because we cannot commit identified files.
    if delta != 0:
        cur_time = cur_time + timedelta(seconds=delta)
    return datetime.strftime(cur_time, '%d-%b-%Y.%H:%M:%SUTC')

def format_exception(e):
    exception_list = traceback.format_stack()
    exception_list = exception_list[:-2]
    exception_list.extend(traceback.format_tb(sys.exc_info()[2]))
    exception_list.extend(traceback.format_exception_only(sys.exc_info()[0], sys.exc_info()[1]))

    exception_str = "Traceback (most recent call last):\n"
    exception_str += "".join(exception_list)
    # Removing the last \n
    exception_str = exception_str[:-1]

    return exception_str

def __join_str(strs):
    '''Concatenate strings with ',', but the duplicated string will be concatenated only once.'''
    if not strs:
        return ''

    str_set = set()
    joined_str = ''
    for s in strs:
        if len(s) > 0 and not s in str_set:
                str_set.add(s)

                if len(joined_str) > 0:
                    joined_str += ','

                joined_str += s

    return joined_str

if __name__ == '__main__':
    pass


import cv2

bgr = cv2.imread('l1-1.jpg', 1)
yuv = cv2.cvtColor(bgr, cv2.COLOR_BGR2YUV)
y, u, v = cv2.split(yuv)

cv2.imshow('y', y)
cv2.imshow('u', u)
cv2.imshow('v', v)
cv2.waitKey(0)
cv2.destroyAllWindows()


from urllib.parse import urlparse,parse_qs
import urllib
import copy
import requests

xss_payloads = ['<xi4okv>', 'javascript::alert`1`']
sqli_payloads = ["%27","\\","/"]
lfi_payloads = ['../../../etc/passwd']

payloads = xss_payloads + sqli_payloads + lfi_payloads

def fuzz(url):
    check = []
    url = urllib.parse.unquote(url, encoding='utf-8', errors='replace')
    up = urlparse(url)
    try:
        if up.query == '':
            return
        for ext in ['.js','.jpg','.css','.gif','.png','.txt']:
            if ext in up.path:
                return
        print('fuzz ' + url)
    except:
        return

    query = parse_qs(up.query)

    #Fuzz
    for i in query:
        for j in payloads:
            queryCheck = copy.deepcopy(query)
            upCheck = copy.deepcopy(up)
            queryCheck[i][0] = j

            queryTarget = ''
            for i in queryCheck:
                data = i + '=' + queryCheck[i][0] + '&'
                queryTarget += data

            url = up.scheme + '://' + up.netloc + up.path + '?' + queryTarget

            check.append(url[:-1])

    for i in query:
        for j in payloads:
            queryCheck = copy.deepcopy(query)
            upCheck = copy.deepcopy(up)
            queryCheck[i][0] = urllib.parse.quote(j)

            queryTarget = ''
            for i in queryCheck:
                data = i + '=' + queryCheck[i][0] + '&'
                queryTarget += data

            url = up.scheme + '://' + up.netloc + up.path + '?' + queryTarget

            check.append(url[:-1])

    for url in check:
        try:
            text = requests.get(url,timeout=5).text
            for i in xss_payloads:
                if i in text:#and 'text' in text.headers['Content-Type']:
                    print ("XSS ==> " + url)
            if 'SQL syntax' in text or 'mysql_fetch' in text:
                print("SQLi ==> " + url)
            if 'root:/root:' in text:
                print("LFI ==> " + url)
        except:
            pass

#fuzz("http://testphp.vulnweb.com/artists.php?artist=3")


import unittest

import mock

import advanced_example_code

class TestAdvancedExampleCode(unittest.TestCase):
    @mock.patch('serial.Serial')
    def test_open_serial_port(self, serial_mock):
        serial_port_mock = mock.MagicMock()
        serial_mock.return_value = serial_port_mock

        serial_port = ors_tutorial_code.open_serial_port()

        serial_mock.assert_called_once_with('/dev/ttyACM0', 9600)
        self.assertEqual(serial_port, serial_port_mock)

    def test_read_serial_port(self):
        serial_port_mock = mock.MagicMock()

        ors_tutorial_code.read_serial_port(serial_port_mock)

        serial_port_mock.readline.assert_called_once_with()

    @mock.patch('sensor_msgs.msg.NavSatFix')
    def test_clean_data(self, nav_sat_fix_mock):
        data_template = '{latitude},{longitude},{altitude}'
        data = data_template.format(
            latitude=-71.4, longitude=42.3, altitude=20.5)

        ors_tutorial_code.clean_data(data)

        nav_sat_fix_mock.assert_called_once_with(
            latitude=-71.4, longitude=42.3, altitude=20.5)

    @mock.patch('sensor_msgs.msg.NavSatFix')
    def test_calculate_boat_position(self, nav_sat_fix_mock):
        sensor_position_mock = mock.MagicMock()
        sensor_position_mock.latitude = 1.1
        sensor_position_mock.longitude = 2.2
        sensor_position_mock.altitude = 4.0

        ors_tutorial_code.calculate_boat_position(sensor_position_mock)

        nav_sat_fix_call = nav_sat_fix_mock.mock_calls[0]
        nav_sat_fix_kwargs = nav_sat_fix_call[2]
        self.assertEqual(nav_sat_fix_kwargs['latitude'], 1.1)
        self.assertEqual(nav_sat_fix_kwargs['longitude'], 2.2)
        self.assertAlmostEqual(nav_sat_fix_kwargs['altitude'], 0.1)

        self.assertEqual(sensor_position_mock.latitude, 1.1)
        self.assertEqual(sensor_position_mock.longitude, 2.2)
        self.assertEqual(sensor_position_mock.altitude, 4.0)

    @mock.patch('ors_tutorial_code.calculate_boat_position')
    @mock.patch('ors_tutorial_code.clean_data')
    @mock.patch('ors_tutorial_code.read_serial_port')
    def test_publish_position(self, read_serial_port_mock, clean_data_mock,
                              calculate_boat_position_mock):
        serial_port_mock = mock.MagicMock()
        publisher_mock = mock.MagicMock()

        read_serial_port_mock.return_value = 1
        clean_data_mock.return_value = 2
        calculate_boat_position_mock.return_value = 3

        ors_tutorial_code.publish_position(serial_port_mock, publisher_mock)

        read_serial_port_mock.assert_called_once_with(serial_port_mock)
        clean_data_mock.assert_called_once_with(1)
        calculate_boat_position_mock.assert_called_once_with(2)
        publisher_mock.publish.assert_called_once_with(3)

if __name__ == '__main__':
    unittest.main()


from django.db import models
from django.db.utils import IntegrityError
import twitter
from . import twittercreds
import pandas as pd
import os

countries = pd.read_csv(
    os.path.join(os.path.dirname(__file__), 'csv/countries.csv'))
# Keep only country names
country_names = countries.iloc[:, 0:1].values.tolist()

# countries = countries[]
t = twitter.Twitter(
    auth=twitter.OAuth(
        twittercreds.token,
        twittercreds.token_secret,
        twittercreds.consumer_key,
        twittercreds.consumer_secret))

class TwitterManager(models.Manager):
    """
    Manager extension to add the fetch method to collect tweets.
    """
    def fetch(self, user, n_latest):
        """
        Fetch the n_latest latest tweets from @user and store them.
        """
        collected = t.statuses.user_timeline(
            screen_name=user,
            count=n_latest,
            tweet_mode='extended')
        for tuit in collected:
            # print(tuit.get('user').get('screen_name'))
            # print(tuit.get('full_text'))
            # print(tuit.get('retweeted_status', False))
            try:
                rt = tuit.get('retweeted_status', False)
                if rt:
                    text = rt.get('full_text')
                else:
                    text = tuit.get('full_text')
                # This is very far from optimal, but very flexible for an
                # iterative development:
                tweet_country=False
                for country in country_names:
                    if country[0] in text:
                        tweet_country = country[0]
                        break
                Tweet.objects.create(user=tuit.get('user').get('screen_name'),
                                     message=text,
                                     country=tweet_country
                                     )
            except IntegrityError:
                print("This tweet is already in the database")

class Tweet(models.Model):
    """
    Simplistic tweet model
    """
    # Not caring yet about the actual max length on twitter:
    message = models.CharField(max_length=300)
    user = models.CharField(max_length=35)
    # Starting as a charfield instead of another model
    country = models.CharField(max_length=50, null=True, blank=True)
    objects = TwitterManager()

    def __str__(self):
        return self.user + ': ' + self.message[:20] + '...'

    class Meta:
        unique_together = ("user", "message")


import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import DBR

# Real world environnment
Ngrid = 100
N1 = Ngrid
N2 = Ngrid
dx = 5
epsi = 12.25
eps0 = 1.

minwave = 400
maxwave = 1200
wavestep = 10
wavelength = np.array([np.arange(minwave,maxwave,wavestep)])
tarwave = 800

# Input and output size based on the Env
input_size = Ngrid
output_size = Ngrid
learning_rate = 0.01

# These lines establish the feed-forward part of the network used to
# choose actions
with tf.device('/device:GPU:1'):
    X = tf.placeholder(shape=[1,input_size],dtype=tf.float32) # state input

    W1 = tf.Variable(tf.random_uniform([input_size,N1],minval=0,maxval=0.01)) # weight
#    b1 = tf.Variable(tf.random_uniform([N1],minval=0,maxval=0.01)) # weight
    L1 = tf.nn.relu(tf.matmul(X,W1))

    W2 = tf.Variable(tf.random_uniform([N1,N2],minval=0,maxval=0.01)) # weight
#    b2 = tf.Variable(tf.random_uniform([N2],minval=0,maxval=0.01)) # weight
    L2 = tf.nn.relu(tf.matmul(L1,W2))

    W3 = tf.Variable(tf.random_uniform([N2,output_size],minval=0,maxval=0.01)) # weight
#    b3 = tf.Variable(tf.random_uniform([output_size],minval=0,maxval=0.01)) # weight
    Qpred = tf.matmul(L2,W3) # Out Q prediction
    Y = tf.placeholder(shape=[1,output_size],dtype=tf.float32)

    loss = tf.reduce_sum(tf.square(Y-Qpred))
    train = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss) # Y label

# Set Q-learning related parameters
dis = .99
num_episodes = 100
num_iter = 100

# Create lists to contain total rewards and steps per episode
rList = []

init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
    sess.run(init)
    for i in range(num_episodes):
        # Reser environment and get first new observation
        s = np.zeros((1,Ngrid))
        e = 1/(1+i)
        rAll = 0
        prereward = 0
        local_loss = []

        # The Q-Network training
        for iteridx in range(num_iter):
            # Choose an action by greedily (with a chance of random action) from the Q-network
            Qs = sess.run(Qpred,feed_dict={X: s})
            if np.random.rand(1) < e:
                a = np.random.randint(Ngrid)
            else:
                a = np.argmax(Qs)

            # Get new state and reward
            R = DBR.calR(s,Ngrid,wavelength,dx,epsi,eps0)
            rawreward = DBR.reward(Ngrid,wavelength,R,tarwave)
            reward = rawreward - prereward # the Q factor does not belong to action.
            s1 = DBR.step(s,a)

            if iteridx == num_iter-1:
                # Update Q, and no Qs+1, since it's a terminal state
                Qs[0,a] = reward
            else:
                # Obtain the Q_s1 values by feeding the new state through our network
                Qs1 = sess.run(Qpred,feed_dict={X: s1})
                # Update Q
                Qs[0,a] = reward + dis*np.max(Qs1)

            # Train our network using target (Y) and predicted Q (Qpred) values
            sess.run(train, feed_dict={X: s, Y: Qs})

            rAll += reward
            prereward = rawreward
            s = s1.copy()

        rList.append(rAll)

print("percent of successful episodes: " +str(sum(rList) / num_episodes) + "%")
plt.bar(range(len(rList)), rList, color="blue")
plt.show()

plt.imshow(s, cmap = "gray")
plt.show()


from django.views.generic.base import TemplateView
from django.template import RequestContext
from django.shortcuts import render_to_response
from tutorial.models import Tutorial

class SomeView(TemplateView):

	def get_context_data(self, **kwargs):
		return {'tutorial': Tutorial.objects.all()[:1] }

def tutorial_show(request, slug):
	t = Tutorial.objects.get(slug=slug)
	sidebar = Tutorial.objects.all()
	var = {
		'tutorial': t,
		'sidebar':sidebar,
	}
	return render_to_response(
        "base-tutorial.html",
        var,
        context_instance = RequestContext(request)
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup
import re

print('https://v.qq.com/x/cover/g4y2d81zw85x9f8/e0022d9gfyi.html')
web = input('Input URL:\n')
# 从视频页面获取VID
if re.search(r'vid=', web):
    patten = re.compile(r'vid=(.*)')
    vid = patte.findall(web)
    vid = vid[0]
else:
    newurl = (web.split('/')[-1])
    vid = newurl.replace('.html', ' ')

# 打开网页的函数
def getpage(url):
    req = Request(url)
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit'
    req.add_header('User-Agent', user_agent)
    try:
        response = urlopen(url)
    except HTTPError as e:
        print('The server couldn\'t fulfill the request.')
        print('Error code:', e.code)
    except URLError as e:
        print('We failed to reach a server.')
        print('Reason:', e.reason)
    html = response.read().decode('utf-8')
    return(html)

getinfo = 'https://vv.video.qq.com/getinfo?vids={vid}  \
           &defaultfmt=auto&otype=json'.format(vid=vid.strip())
a = getpage(getinfo)
print(a)

# 找到ID和拼接FILENAME
soup = BeautifulSoup(a, 'html.parser')
for e1 in soup.find_all('url'):
    ippattent = re.compile(
        r'((?:(2[0-4]\d)|(25[0-5])|([01]?\d\d?))\.){3}(?:(2[0-4]\d)|(255[0-5])|([01]?\d\d?))')
    if re.search(ippattent, e1.get_text()):
        ip = (e1.get_text())
for e2 in soup.find_all('id'):
    idpattent = re.compile(r'\d{5}')
    if re.search(idpattent, e2.get_text()):
        id = (e2.get_text())
filename = vid.strip() + '.p' + id[2:] + '.2.mp4'
print(filename)

# 利用getinfo中的信息拼接getkey网址
getkey = 'http://vv.video.qq.com/getkey?format={id}&charge=0&vid={vid}&otype=json\
          &filename={filename}&platform=10901'.format(id=id, vid=vid.strip(), filename=filename)
b = getpage(getkey)
print(b)

key = (re.findall(r'<key>(.*)<\\\\/key>', b))
videourl = ip + filename + '?' + 'vkey=' + key[0]
print('Open URL:\n' + videourl)


from CreditRisk import CreditRisk
from FeatureProcessing import FeatureProcessing
from information_value import WOE
from xgboost import XGBClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score, KFold
import xgboost as xgb
import sys
import uuid
import math
import pandas as pd
import numpy as np
import os
import time

from imblearn.combine import SMOTETomek # used for balancing the dataset
from collections import Counter

class GradientTreeBoosting:
    def __init__(self, url, test_size, ignore_features=None):
        self.url = url
        self.test_size = test_size
        #TODO should be either list of string or list with index representing position (1 on, 0 off)
        self.ignore_features = ignore_features
        self.model_type = "GRADIENT_TREE_BOOSTING"

    # returns kfold cross validation scores
    def kfold_cv(self, CR, X, y, n_splits=3, shuffle=False, random_state=None):
        scores = []
        kfold = KFold(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
        for train_idx, test_idx in kfold.split(X):
            #print("TRAIN:", train_idx, "TEST:", test_idx)
            X_train, X_test = X[train_idx], X[test_idx]
            y_train, y_test = y[train_idx], y[test_idx]
            model = XGBClassifier(nthread=-1) # -1 to use all cores in system
            model.fit(X_train, y_train)
            scores.append(CR.get_model_accuracy(model, X_test, y_test))
        return scores

    # TODO implement stratified cross validation
    def stratified_cv():
        raise NotImplementedError

    def run(self, default_column='DEFAULT PAYMENT NEXT MONTH', corr_threshold=1.0):
        start_time = time.time()
        # download file from url
        filename = self.url.rsplit('/', 1)[-1]
        uuid = os.path.splitext(os.path.basename(filename))[0]
        print("GradientTreeBoosting.self.url:", self.url)
        CR = CreditRisk(self.url, uuid)
        Processing = FeatureProcessing()

        # check if file exists locally, if not download
        df = CR.read_file()

        df = Processing.sort_df_by_feature_names(df)

        # TODO should save newly created df to server with timestamp and uuid

        #print(list(df))
        print("Number original features: ", len(list(df)))

        df, classes = CR.write_file(df, default_column, uuid)

        # drop ignored features
        if self.ignore_features is not None:
            for feature in self.ignore_features:
                df = df.drop(feature, axis=1)
        #df = CR.bin_numerical_features(df)

        # remove highly correlated features
        features_to_drop = Processing.list_highly_corr_features(df, corr_threshold)
        #print(features_to_drop)
        df = Processing.remove_features(df, features_to_drop)

        # partition
        features = list(df)
        #print(features)
        #print("Number kept features: ", len(list(df)))
        X, y = CR.test_train_matrix(df, default_column)

        # added to balance dataset
        # TODO  may want to check if is needed
        X_resampled, y_resampled = SMOTETomek().fit_sample(X, y)
        print(Counter(y_resampled))
        X_train, X_test, y_train, y_test = CR.split_train_test(
            X_resampled, y_resampled, self.test_size, False, len(features))

        model = XGBClassifier(nthread=-1) # -1 to use all cores in system
        model.fit(X_train, y_train)

        # TODO this function creates data for predicting from original dataset
        # uncomment to see prediction results on original dataset
        X_train, X_test, y_train, y_test = CR.split_train_test(
            X, y, self.test_size, False, len(features))

        # TODO also want to predict probabilites on unsampled data
        # will want to return predictions on unsampled data so we return exact number of instances we started with
        y_train_pred = model.predict_proba(X_train)[:, 1] # 1 = yes default, 0 = no default?
        y_test_pred = model.predict_proba(X_test)[:, 1]

        print("gradient boosted trees accuracy: " + str(CR.get_model_accuracy(model, X_test, y_test)))

        # CV
        #print("kfolds cv: " + str(kfold_cv(CR, X, y)))

        uuid_t = CR.save_model(model)

        # TODO the following is the native implementation of xgboost
        # may not be needed
        '''
        dtrain = xgb.DMatrix(X_train, label=y_train)
        dtest = xgb.DMatrix(X_test, label=y_test)
        num_round = 5
        evallist = [(dtest, 'eval'), (dtrain, 'train')]
        param = {'objective': 'binary:logistic',
                 'silent': 1, 'eval_metric': ['error', 'logloss']}
        bst = xgb.train(param, dtrain, num_round, evallist)

        # save model
        #uuid_t = uuid.uuid4().hex
        uuid_t = CR.save_model(bst)

        # get probability estimates
        y_train_pred = bst.predict(dtrain)
        y_test_pred = bst.predict(dtest)'''

        # get target col index
        if default_column is not None:
            default_col_index = features.index(default_column)

        #kfolds_cv_score = self.kfold_cv(CR, X.as_matrix(), y.as_matrix())
        #print(kfolds_cv_score)

        c_nparray = df.as_matrix()
        #df_o = filename
        score = []
        woe_dict = {}
        features.pop()

        woe = WOE()
        for feat in features:
            woe_dict[feat] = woe.woe_single_x_score(c_nparray[:, features.index(feat)], c_nparray[:, default_col_index].astype(
                bool))
        for index, row in df.iterrows():
            woe_val = 0
            # Assume that As the score points are just another way to denote the scorecard, they do not affect its predictive power.
            # Now, we can assume that 600 score points correspond to odds (bads to goods) of 1:50,
            #  while each additional 20 points double the odds (620 points double the odds to 1:100, 640 score points double to 1:200 etc.).
            startScore = 600
            pdo = 20
            factor = pdo / math.log(2)
            offset = round(startScore - (factor * math.log(50)))
            for l in features:
                cell_value = df.at[index, l]
                col_index = features.index(l)
                woe_val = woe_val + woe_dict[l][cell_value]
            score.append(round(offset + woe_val))
            #print('Rows:: ', index,'Score:: ',score[index])

        print("total xgb: %s seconds" % (time.time() - start_time))

        return CR.scores(score, y_train, y_train_pred,
                         y_test, y_test_pred, classes, uuid_t)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
from rest_framework.views import APIView
from rest_framework.viewsets import ViewSetMixin, ModelViewSet

from rest_framework.response import Response
from api import models
from api.serializer.article import ArticleModelSerializer

class ArticleView(ViewSetMixin, APIView):
    def get(self, request, *args, **kwargs):
        response = {'status': False}
        try:
            queryset = models.Article.objects.all()
            ser = ArticleModelSerializer(instance=queryset, many=True)
            response['data'] = ser.data
            response['status'] = True
        except Exception as e:
            response['error'] = str(e)
        return Response(response)

    def retrieve(self, request, pk):
        response = {'status': False}
        try:
            queryset = models.Article.objects.get(pk)
            ser = ArticleModelSerializer(instance=queryset, many=False)
            response['data'] = ser.data
            response['status'] = True
        except Exception as e:
            response['error'] = str(e)
        return Response(response)


from authenticationsdk.core.Authorization import *
from authenticationsdk.payloaddigest.PayLoadDigest import *
from authenticationsdk.core.MerchantConfiguration import *
import CyberSource.logging.log_factory as LogFactory
from authenticationsdk.util.PropertiesUtil import *
import authenticationsdk.util.ExceptionAuth
from importlib.machinery import SourceFileLoader
from pathlib import Path

request_file = os.path.join(os.getcwd(), "samples/authentication/data", "RequestData.py")
request_data = SourceFileLoader("module.name", request_file).load_module()

class PostGenerateHeaders:
    def __init__(self):
        # REQUEST TARGET [EDITABLE]
        self.request_target = "/pts/v2/payments"
        # REQUEST-TYPE [NOT-EDITABLE]
        self.request_type = "POST"
        # REQUEST-JSON-PATH [NOT-EDITABLE]
        self.request_json_path = "samples/authentication/Resources/request.json"
        self.merchant_config = None
        self.date = None

    def post_generate_header(self):
        try:
            util_obj = PropertiesUtil()
            util_obj.cybs_path = os.path.join(os.getcwd(), "samples/authentication/Resources", "cybs.json")
            details_dict1 = util_obj.properties_util()

            mconfig = MerchantConfiguration()
            mconfig.set_merchantconfig(details_dict1)

            mconfig.validate_merchant_details(details_dict1, mconfig)

            mconfig.request_json_path_data = request_data.json_file_data(self.request_json_path, mconfig)

            self.merchant_config = mconfig
            self.merchant_config.request_host = mconfig.request_host
            self.merchant_config.request_type_method = self.request_type
            mconfig.request_target = self.request_target
            self.date = mconfig.get_time()
            self.post_method_headers()
            write_log_audit(200)
        except ApiException as e:
            print(e)
            write_log_audit(400)
        except KeyError as e:
            print(GlobalLabelParameters.NOT_ENTERED + str(e))
            write_log_audit(400)
        except IOError as e:
            print(GlobalLabelParameters.FILE_NOT_FOUND + str(e.filename))
            write_log_audit(400)
        except Exception as e:
            print(repr(e))
            write_log_audit(400)

    # This method prints values obtained in our code by connecting to AUTH sdk
    def post_method_headers(self):
        logger = LogFactory.setup_logger(self.__class__.__name__, self.merchant_config.log_config)
        try:
            auth = Authorization()
            digest = DigestAndPayload()
            authentication_type = self.merchant_config.authentication_type

            print("Request Type    :" + self.request_type)
            print(GlobalLabelParameters.CONTENT_TYPE + "       :" + GlobalLabelParameters.APPLICATION_JSON)

            if authentication_type.upper() == GlobalLabelParameters.HTTP.upper():
                print(" " + GlobalLabelParameters.USER_AGENT + "          : " + GlobalLabelParameters.USER_AGENT_VALUE)
                print(" MerchantID          : " + self.merchant_config.merchant_id)
                print(" Date                : " + self.merchant_config.get_time())
                print("digest               :" + GlobalLabelParameters.DIGEST_PREFIX + digest.string_digest_generation(
                    self.merchant_config.request_json_path_data).encode("utf-8").decode("utf-8"))

                temp_sig = auth.get_token(self.merchant_config, self.date)
                print("Signature Header      :" + str(temp_sig))
                print("Host                  :" + self.merchant_config.request_host)
            else:
                temp_sig = auth.get_token(self.merchant_config, self.date)
                print("Authorization Bearer            :" + str(temp_sig.encode("utf-8").decode("utf-8")))
            if self.merchant_config.log_config.enable_log is True:
                logger.info("END> ======================================= ")
                logger.info("\n")
        except ApiException as e:
            authenticationsdk.util.ExceptionAuth.log_exception(logger, e, self.merchant_config)
        except Exception as e:
            authenticationsdk.util.ExceptionAuth.log_exception(logger, repr(e), self.merchant_config)

def write_log_audit(status):
    print(f"[Sample Code Testing] [{Path(__file__).stem}] {status}")

if __name__ == "__main__":
    post_generate_obj = PostGenerateHeaders()
    post_generate_obj.post_generate_header()


import pprint
from collections import OrderedDict

import os

import anymarkup

from typing import Dict, List

try:
    from convertlib import is_null, expand_attr_list_single, singleton_list_to_value, expand_attr_list, ensure_list
except ModuleNotFoundError:
    from .convertlib import is_null, expand_attr_list_single, singleton_list_to_value, expand_attr_list, ensure_list

VO_SCHEMA_LOCATION = "https://my.opensciencegrid.org/schema/vosummary.xsd"

def expand_contacttypes(contacts: Dict) -> Dict:
    """Expand
    {"Submitter Contact": ["a", "b"],
     "Miscellaneous Contact": ["c", "d"]}
    to
    {"ContactType": [{"Type": "Submitter Contact", {"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"}]}}},
                     {"Type": "Miscellaneous Contact", {"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"}]}}}
                    ]
    }
    """
    new_contacttypes = []
    for type_, list_ in contacts.items():
        contact_data = singleton_list_to_value([{"Name": x} for x in list_])
        new_contacttypes.append({"Type": type_, "Contacts": {"Contact": contact_data}})
    return {"ContactType": singleton_list_to_value(new_contacttypes)}

def expand_reportinggroups(reportinggroups_list: List, reportinggroups_data: Dict) -> Dict:
    """Expand
    ["XXX", "YYY", "ZZZ"]
    using data from reportinggroups_data into
    {"ReportingGroup": [{"Contacts": {"Contact": [{"Name": "a"},
                                                 {"Name": "b"}
                                     },
                         "FQANs": {"FQAN": [{"GroupName": "...",
                                             "Role": "..."}]
                                  }
                         "Name": "XXX"
                       }]
    }
    """
    new_reportinggroups = {}
    for name, data in reportinggroups_data.items():
        if name not in reportinggroups_list: continue
        new_reportinggroups[name] = {}
        newdata = new_reportinggroups[name]
        if not is_null(data, "Contacts"):
            new_contact = [{"Name": x} for x in data["Contacts"]]
            newdata["Contacts"] = {"Contact": singleton_list_to_value(new_contact)}
        else:
            newdata["Contacts"] = None
        if not is_null(data, "FQANs"):
            fqans = []
            for fqan in data["FQANs"]:
                fqans.append(OrderedDict([("GroupName", fqan["GroupName"]), ("Role", fqan["Role"])]))
            newdata["FQANs"] = {"FQAN": singleton_list_to_value(fqans)}
        else:
            newdata["FQANs"] = None
    new_reportinggroups = expand_attr_list(new_reportinggroups, "Name", ordering=["Name", "FQANs", "Contacts"])
    return {"ReportingGroup": new_reportinggroups}

def expand_oasis_managers(managers):
    """Expand
    {"a": {"DNs": [...]}}
    into
    {"Manager": [{"Name": "a", "DNs": {"DN": [...]}}]}
    """
    new_managers = managers.copy()
    for name, data in managers.items():
        if not is_null(data, "DNs"):
            new_managers[name]["DNs"] = {"DN": singleton_list_to_value(data["DNs"])}
        else:
            new_managers[name]["DNs"] = None
    return {"Manager": expand_attr_list(new_managers, "Name", ordering=["ContactID", "Name", "DNs"], ignore_missing=True)}

def expand_fields_of_science(fields_of_science):
    """Turn
    {"PrimaryFields": ["P1", "P2", ...],
     "SecondaryFields": ["S1", "S2", ...]}
    into
    {"PrimaryFields": {"Field": ["P1", "P2", ...]},
     "SecondaryFields": {"Field": ["S1", "S2", ...]}}
    """
    if is_null(fields_of_science, "PrimaryFields"):
        return None
    new_fields = OrderedDict()
    new_fields["PrimaryFields"] = {"Field": singleton_list_to_value(fields_of_science["PrimaryFields"])}
    if not is_null(fields_of_science, "SecondaryFields"):
        new_fields["SecondaryFields"] = {"Field": singleton_list_to_value(fields_of_science["SecondaryFields"])}
    return new_fields

def expand_vo(vo, reportinggroups_data):
    vo = vo.copy()

    if is_null(vo, "Contacts"):
        vo["ContactTypes"] = None
    else:
        vo["ContactTypes"] = expand_contacttypes(vo["Contacts"])
    vo.pop("Contacts", None)
    if is_null(vo, "ReportingGroups"):
        vo["ReportingGroups"] = None
    else:
        vo["ReportingGroups"] = expand_reportinggroups(vo["ReportingGroups"], reportinggroups_data)
    if is_null(vo, "OASIS"):
        vo["OASIS"] = None
    else:
        oasis = OrderedDict()
        oasis["UseOASIS"] = vo["OASIS"].get("UseOASIS", False)
        if is_null(vo["OASIS"], "Managers"):
            oasis["Managers"] = None
        else:
            oasis["Managers"] = expand_oasis_managers(vo["OASIS"]["Managers"])
        if is_null(vo["OASIS"], "OASISRepoURLs"):
            oasis["OASISRepoURLs"] = None
        else:
            oasis["OASISRepoURLs"] = {"URL": singleton_list_to_value(vo["OASIS"]["OASISRepoURLs"])}
        vo["OASIS"] = oasis
    if is_null(vo, "FieldsOfScience"):
        vo["FieldsOfScience"] = None
    else:
        vo["FieldsOfScience"] = expand_fields_of_science(vo["FieldsOfScience"])

    # Restore ordering
    if not is_null(vo, "ParentVO"):
        parentvo = OrderedDict()
        for elem in ["ID", "Name"]:
            if elem in vo["ParentVO"]:
                parentvo[elem] = vo["ParentVO"][elem]
        vo["ParentVO"] = parentvo
    else:
        vo["ParentVO"] = None

    for key in ["MembershipServicesURL", "PrimaryURL", "PurposeURL", "SupportURL"]:
        if key not in vo:
            vo[key] = None

    # TODO: Recreate <MemeberResources> [sic]
    #  should look like
    #  <MemeberResources>
    #    <Resource><ID>75</ID><Name>NERSC-PDSF</Name></Resource>
    #    ...
    #  </MemeberResources>

    # Restore ordering
    new_vo = OrderedDict()
    for elem in ["ID", "Name", "LongName", "CertificateOnly", "PrimaryURL", "MembershipServicesURL", "PurposeURL",
                 "SupportURL", "AppDescription", "Community",
                 # TODO "MemeberResources",
                 "FieldsOfScience", "ParentVO", "ReportingGroups", "Active", "Disable", "ContactTypes", "OASIS"]:
        if elem in vo:
            new_vo[elem] = vo[elem]

    return new_vo

def get_vos_xml():
    """
    Returns the serialized xml (as a string)
    """
    to_output = get_vos()

    return anymarkup.serialize(to_output, 'xml').decode()

def get_vos():
    to_output = {"VOSummary": {
        "@xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
        "@xsi:schemaLocation": VO_SCHEMA_LOCATION,
        "VO": []}}
    vos = []
    reportinggroups_data = anymarkup.parse_file("virtual-organizations/REPORTING_GROUPS.yaml")
    for file in os.listdir("virtual-organizations"):
        if file == "REPORTING_GROUPS.yaml": continue
        vo = anymarkup.parse_file("virtual-organizations/{0}".format(file))
        try:
            vos.append(expand_vo(vo, reportinggroups_data))
        except Exception:
            pprint.pprint(vo)
            raise
    to_output["VOSummary"]["VO"] = vos
    return to_output

if __name__ == "__main__":
    # We are running as the main script

    with open('new_vos.xml', 'w') as xml_file:
        xml_file.write(get_vos_xml())