#
# Copyright (c) SAS Institute Inc.
#
import copy
import difflib
import os
import re
import sys

from conary.lib import options

from .. import errors, utils
from ..lib import command

YES = ('y', 'Y', '')
NO = ('n', 'N')
QUIT = ('q', 'Q')
MACRO_RE = re.compile(r'%\(\w+\)s')

class MergeCommand(command.BaseCommand):
    help = 'Merge new/changed jobs into new/existing templates'
    commands = ['merge']
    paramHelp = '[options] [TEMPLATE]*'
    requireConfig = True

    def _mergeTemplate(self, template, job):
        newTemplate = copy.copy(template)
        template_pairs = self._mergeTemplateHelper(
            job.getroot(),
            template,
            job,
            )
        newTemplate['templates'] = dict((k, v) for k, v in template_pairs)
        return newTemplate

    def _mergeTemplateHelper(self, node, jobTemplate, jobConfig):
        template_pairs = []
        templates = jobTemplate.get('templates')
        if templates is None:
            templates = {}

        # process node
        xpath = jobConfig.getpath(node)
        old_template = templates.get(xpath, '')
        new_template = node.text or ''
        for macro, value in self.fromMacros.itermacros():
            new_template = new_template.replace(value, '%%(%s)s' % macro)

        isTemplated = MACRO_RE.search(new_template)
        prompt = None
        if old_template and isTemplated:
            # update existing node
            if old_template != new_template:
                prompt = 'Update the template for this node [Y/n/q]? '
        elif old_template and not isTemplated:
            # remove old node
            prompt = 'Remove this node from template [Y/n/q]? '
        elif not old_template and isTemplated:
            # add new templated node
            prompt = 'Add this node to template [Y/n/q]? '

        if prompt:
            sys.stdout.write(
                'File: ' + jobTemplate.get('name') % self.fromMacros + '\n')
            sys.stdout.write(xpath + ':\n')
            diff = difflib.ndiff([old_template + '\n'], [new_template + '\n'])
            sys.stdout.write(''.join(diff) + '\n')

            response = None
            while not response in YES + NO + QUIT:
                response = raw_input(prompt)
            sys.stdout.write('\n\n')

            if response in YES:
                if not prompt.startswith('Remove'):
                    template_pairs.append((xpath, new_template))
            elif response in NO:
                if old_template:
                    template_pairs.append((xpath, old_template))
            elif response in QUIT:
                raise KeyboardInterrupt
        elif old_template:
            template_pairs.append((xpath, old_template))

        for child in node.iterchildren():
            template_pairs.extend(
                self._mergeTemplateHelper(child, jobTemplate, jobConfig))

        return template_pairs

    def addLocalParameters(self, argDef):
        command.BaseCommand.addLocalParameters(self, argDef)
        argDef['project'] = (options.OPT_PARAM, 'Path to project, defaults to'
                             ' current working directory')
        argDef['from'] = options.MULT_PARAM, 'Macro to do replacement on'

    def mergeJobs(self, templateList):
        for templateFile in templateList:
            template = utils.readTemplate(templateFile)

            jobFile = os.path.join(
                self.jobDir, template.get('name') % self.fromMacros)

            jobData = utils.readJob(jobFile)

            try:
                mergedTemplate = self._mergeTemplate(template, jobData)
            except errors.TemplateError as err:
                raise errors.CommandError(
                    "%s parsing job '%s'" % (err, jobFile))

            utils.writeTemplate(templateFile, mergedTemplate)

    def runCommand(self, cfg, argSet, args, **kwargs):
        projectDir = argSet.pop('project', os.getcwd())
        fromMacros = argSet.pop('from', [])

        _, templateList = self.requireParameters(args, allowExtra=True)

        self.fromMacros = utils.parseMacros(fromMacros)

        self.projectDir = os.path.abspath(projectDir)

        # set up job directory
        self.jobDir = os.path.join(projectDir, cfg.jobDir)
        if not (os.path.exists(self.jobDir) and os.path.isdir(self.jobDir)):
            raise errors.CommandError(
                'no jobs directory found in %s' % projectDir)

        # set up template directory
        self.templateDir = os.path.join(self.projectDir, cfg.templateDir)
        if not (os.path.exists(self.templateDir)
                and os.path.isdir(self.templateDir)):
            raise errors.CommandError(
                'no template directory found in %s' % projectDir)

        if not templateList:
            templateList = [os.path.join(self.templateDir, f)
                            for f in os.listdir(self.templateDir)
                            if f.endswith('.yaml') or f.endswith('.yml')]

        self.mergeJobs(templateList)


import unittest
from noseparallel import ParallelPlugin

class PluginTest(unittest.TestCase):

    def test_want_file_should_accept_on_either_nodes_but_not_both(self):
        plugin = ParallelPlugin()
        plugin.salt = 'test'
        plugin.total_nodes = 2
        testFile = "tests/plugin_test.py"
        plugin.node_index = 0
        rv0 = plugin.wantFile(testFile)

        plugin.node_index = 1
        rv1 = plugin.wantFile(testFile)
        self.assertEqual({None, False}, {rv0, rv1})


import os
import sys
import argparse

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Recursively get the size of folders.')
    parser.add_argument('path', metavar='-p', default="cwd",
                        help='use this path instead of the current path')
    args = parser.parse_args()
    path = args.path[1]
    if path == "cwd":
        path = os.getcwd()


import unittest
from stack import Stack

class StackWithMin(Stack):
    def __init__(self):
        super().__init__()
        self.stack = Stack()

    def push(self, data):
        if data <= self.min():
            self.stack.push(data)
        super().push(data)

    def pop(self):
        data = super().pop()
        if data == self.min():
            self.stack.pop()
        return data

    def min(self):
        if self.stack.is_empty():
            return float('Inf')

        return self.stack.peek()

class Test(unittest.TestCase):
    values = [5, 7, 3, 10, 2]
    min_values = [2, 3, 3, 5, 5]

    def test_stack_min(self):
        stack = StackWithMin()
        for value in self.values:
            stack.push(value)

        for value in self.min_values:
            self.assertEqual(value, stack.min())
            stack.pop()

if __name__ == '__main__':
    unittest.main()


import json
from block import Block
from settings import MAP_PATH

class Map_Loader(object):
    def __init__(self):
        with open(MAP_PATH, encoding='utf-8') as data_file:
            data = json.load(data_file)
            self.blocks = data["blocks"]
            self.maps = data["maps"]
            self.title = None
        self.level = -1

    def next_map(self):
        if len(self.maps) - 1 > self.level:
            self.level += 1
            self.title = self.maps[self.level]["title"]
            return self.level
        return None

    def get_blocks(self):
        return [Block(block["x"], block["y"], *self.blocks[block["kind"]]) for block in self.maps[self.level]["blocks"]]


'''
Cross-compare blobs with incremental adjacency, within a frame
'''

from class_cluster import ClusterStructure, NoneType
import numpy as np
import cv2

class CderBlob(ClusterStructure):

    _blob = object
    mB = int
    dB = int
    dI = int
    mI = int
    dA = int
    mA = int
    dG = int
    mG = int
    dM = int
    mM = int

class CBblob(ClusterStructure):

    DerBlob = object
    blob_ = list

ave_mB = 20000  # ave can't be negative
ave_rM = .7  # average relative match at rL=1: rate of ave_mB decay with relative distance, due to correlation between proximity and similarity

def cross_comp_blobs(frame):
    '''
    root function of comp_blob: cross compare blobs with their adjacent blobs in frame.blob_, including sub_layers
    '''
    blob_ = frame.blob_

    for blob in blob_:  # each blob forms derBlob per compared adj_blob and accumulates adj_blobs'derBlobs:
        blob.DerBlob = CderBlob()
        comp_blob_recursive(blob, blob.adj_blobs[0], derBlob_=[], derBlob_id_=[])
        # derBlob_ and derBlob_id_ are local and frame-wide

    bblob_ = form_bblob_(blob_)  # form blobs of blobs, connected by mutual match

    visualize_cluster_(bblob_, blob_, frame)

    return bblob_

def comp_blob_recursive(blob, adj_blob_, derBlob_, derBlob_id_):
    '''
    called by cross_comp_blob to recursively compare blob to adj_blobs in incremental layers of adjacency
    '''
    for adj_blob in adj_blob_:

        id = blob.id; _id = adj_blob.id  # pairing function generates unique number from each comparand_pair, frame-wide, same for derBlob_:
        derBlob_id = 0.5 * (id+_id) * (id+_id+1) + _id  # 0.5*(a + b)(a + b + 1) + b: https://en.wikipedia.org/wiki/Pairing_function
        _derBlob_id = -derBlob_id     # different-order a and b yields different results, so we need both
        fderBlob = 0

        # existing derBlob
        if derBlob_id in derBlob_id_ :
            derBlob = derBlob_[derBlob_id_.index(derBlob_id)]
            accum_derBlob(blob, derBlob)  # also adj_blob.rdn += 1?
            fderBlob = 1
        elif _derBlob_id in derBlob_id_:
            derBlob = derBlob_[derBlob_id_.index(_derBlob_id)]
            accum_derBlob(blob, derBlob)  # also adj_blob.rdn += 1?
            fderBlob = 1

        # new derBlob
        elif adj_blob is not blob:  # adj of adj blob could be the blob itself
            derBlob = comp_blob(blob, adj_blob)  # compare blob and adjacent blob
            accum_derBlob(blob, derBlob)  # from all compared blobs, regardless of mB sign
            derBlob_id_.append(derBlob_id)  # unique comparand_pair identifier
            derBlob_.append(derBlob)  # also frame-wide
            fderBlob = 1

        if fderBlob: # derBlob is not exist if feval = 0
            if derBlob.mB > 0:
                # replace blob with adj_blob for continuing adjacency search:
                if not isinstance(adj_blob.DerBlob, CderBlob):  # if adj_blob.DerBlob: it's already searched in previous call
                    # but this search could be to a different depth?
                    adj_blob.DerBlob = CderBlob()
                    comp_blob_recursive(adj_blob, adj_blob.adj_blobs[0], derBlob_, derBlob_id_)

            elif blob.Dert.M + blob.neg_mB+ derBlob.mB > ave_mB:  # neg mB but positive comb M,
                # extend blob comparison to adjacents of adjacent, depth-first
                blob.neg_mB += derBlob.mB  # mB and distance are accumulated over comparison scope
                blob.distance += np.sqrt(adj_blob.A)
                comp_blob_recursive(blob, adj_blob.adj_blobs[0], derBlob_, derBlob_id_)

def comp_blob(blob, _blob):
    '''
    cross compare _blob and blob
    '''
    (_I, _Dy, _Dx, _G, _M, _Dyy, _Dyx, _Dxy, _Dxx, _Ga, _Ma, _Mdx, _Ddx), _A = _blob.Dert.unpack(), _blob.A
    (I, Dy, Dx, G, M, Dyy, Dyx, Dxy, Dxx, Ga, Ma, Mdx, Ddx), A = blob.Dert.unpack(), blob.A

    dI = _I - I  # d is always signed
    mI = min(_I, I)
    dA = _A - A
    mA = min(_A, A)
    dG = _G - G
    mG = min(_G, G)
    dM = _M - M
    mM = min(_M, M)

    mB = mI + mA + mG + mM - ave_mB * (ave_rM ** ((1+blob.distance) / np.sqrt(A)))
    # deviation from average blob match at current distance
    dB = dI + dA + dG + dM

    derBlob  = CderBlob(_blob=_blob, mB=mB, dB=dB)  # blob is core node, _blob is adjacent blob

    if _blob.fsliced and blob.fsliced:
        pass

    return derBlob

def form_bblob_(blob_):
    '''
    form blob of blobs as a cluster of blobs with positive adjacent derBlob_s, formed by comparing adj_blobs
    '''
    checked_ids = []  # checked for clustering, frame-wide vs. per blob
    bblob_ = []

    for blob in blob_:
        if blob.DerBlob.mB > 0 and blob.id not in checked_ids:  # init bblob with current blob
            checked_ids.append(blob.id)

            bblob = CBblob(DerBlob=CderBlob())
            accum_bblob(bblob, blob)
            form_bblob_recursive(bblob_, bblob, checked_ids)

    return bblob_

def form_bblob_recursive(bblob_, bblob, checked_ids):

    fsearch = 0  # there are blobs to check for inclusion in bblob

    for blob in bblob.blob_: # search blob' derBlob's blobs to get potential border clustering blob
        if (blob.DerBlob.mB > 0) and (blob.id not in checked_ids):  # positive mB
            for derBlob in blob.derBlob_:
                # if blob is in bblob.blob_, but derBlob._blob is not in bblob_blob_
                # so if sum of derBlob._blob's mB with blob's mB > 0 , pack the derBlob._blob into bblob.blob_
                if (derBlob._blob not in bblob.blob_) and (derBlob._blob.DerBlob.mB + blob.DerBlob.mB >0):
                    accum_bblob(bblob, derBlob._blob)
                    checked_ids.append(blob.id)
                    fsearch = 1

    if fsearch:
        form_bblob_recursive(bblob_, bblob, checked_ids)

    bblob_.append(bblob) # pack bblob after scanning all possible derBlobs

def accum_derBlob(blob, derBlob):

    blob.DerBlob.accumulate(**{param:getattr(derBlob, param) for param in blob.DerBlob.numeric_params})
    blob.derBlob_.append(derBlob)

def accum_bblob(bblob, blob):

    # accumulate derBlob
    bblob.DerBlob.accumulate(**{param:getattr(blob.DerBlob, param) for param in bblob.DerBlob.numeric_params})

    if blob not in bblob.blob_:
    # isn't that checked before calling accum_bblob?
        bblob.blob_.append(blob) # pack blob into bblob.blob_

    for derBlob in blob.derBlob_: # pack adjacent blobs of blob into bblob
        if derBlob._blob not in bblob.blob_:
            bblob.blob_.append(derBlob._blob)

'''
    cross-comp among sub_blobs in nested sub_layers:
    _sub_layer = bblob.sub_layer[0]
    for sub_layer in bblob.sub_layer[1:]:
        for _sub_blob in _sub_layer:
            for sub_blob in sub_layer:
                comp_blob(_sub_blob, sub_blob)
        merge(_sub_layer, sub_layer)  # only for sub-blobs not combined into new bblobs by cross-comp
'''

def visualize_cluster_(bblob_, blob_, frame):

    colour_list = []  # list of colours:
    colour_list.append([200, 130, 1])  # blue
    colour_list.append([75, 25, 230])  # red
    colour_list.append([25, 255, 255])  # yellow
    colour_list.append([75, 180, 60])  # green
    colour_list.append([212, 190, 250])  # pink
    colour_list.append([240, 250, 70])  # cyan
    colour_list.append([48, 130, 245])  # orange
    colour_list.append([180, 30, 145])  # purple
    colour_list.append([40, 110, 175])  # brown
    colour_list.append([192, 192, 192])  # silver
    colour_list.append([255, 255, 0])  # blue2
    colour_list.append([34, 34, 178])  # red2
    colour_list.append([0, 215, 255])  # yellow2
    colour_list.append([50, 205, 50])  # green2
    colour_list.append([114, 128, 250])  # pink2
    colour_list.append([255, 255, 224])  # cyan2
    colour_list.append([0, 140, 255])  # orange2
    colour_list.append([204, 50, 153])  # purple2
    colour_list.append([63, 133, 205])  # brown2
    colour_list.append([128, 128, 128])  # silver2

    # initialization
    ysize, xsize = frame.dert__[0].shape
    blob_img = np.zeros((ysize, xsize,3)).astype('uint8')
    cluster_img = np.zeros((ysize, xsize,3)).astype('uint8')
    img_separator = np.zeros((ysize,3,3)).astype('uint8')

    # create mask
    blob_mask = np.zeros_like(frame.dert__[0]).astype('uint8')
    cluster_mask = np.zeros_like(frame.dert__[0]).astype('uint8')

    blob_colour_index = 1
    cluster_colour_index = 1

    cv2.namedWindow('blobs & clusters', cv2.WINDOW_NORMAL)

    # draw blobs
    for blob in blob_:
        cy0, cyn, cx0, cxn = blob.box
        blob_mask[cy0:cyn,cx0:cxn] += (~blob.mask__ * blob_colour_index).astype('uint8')
        blob_colour_index += 1

    # draw blob cluster of bblob
    for bblob in bblob_:
        for blob in bblob.blob_:
            cy0, cyn, cx0, cxn = blob.box
            blob_colour_index += 1
            cluster_mask[cy0:cyn,cx0:cxn] += (~blob.mask__ *cluster_colour_index).astype('uint8')

        # increase colour index
        cluster_colour_index += 1

        # insert colour of blobs
        for i in range(1,blob_colour_index):
            blob_img[np.where(blob_mask == i)] = colour_list[i % 10]

        # insert colour of clusters
        for i in range(1,cluster_colour_index):
            cluster_img[np.where(cluster_mask == i)] = colour_list[i % 10]

        # combine images for visualization
        img_concat = np.concatenate((blob_img, img_separator,
                                    cluster_img, img_separator), axis=1)

        # plot cluster of blob
        cv2.imshow('blobs & clusters',img_concat)
        cv2.resizeWindow('blobs & clusters', 1920, 720)
        cv2.waitKey(10)

    cv2.waitKey(0)
    cv2.destroyAllWindows()


from _pybgpstream import BGPStream, BGPRecord
from collections import defaultdict
from itertools import groupby
import networkx as nx
import IP2Location
import re

class DataCollector(object):

    as_prefix = defaultdict(set)

    def __init__(self, start, end, mode):
        self.start = start
        self.end = end
        self.mode = mode

        if mode == "IPv4":
            self.pattern = "([\w]*[.])+([\w]*)"
        elif mode == "IPv6":
            self.pattern = "([\w]*[:])+([\w]*)"
        else:
            raise ValueError("The mode can only be IPv4 or IPv6.")

    # method to catch data from BGPStream
    def get_data_graph(self):

        stream = BGPStream()
        rec = BGPRecord()
        stream.add_filter('record-type', 'ribs')
        stream.add_interval_filter(self.start, self.end)
        stream.start()

        as_graph = nx.Graph()

        while stream.get_next_record(rec):
            if rec.status == "valid":
                elem = rec.get_next_elem()
                while elem:
                    # the list is a list, which stores an as-path
                    as_path = [k for k, g in groupby(elem.fields['as-path'].split(" "))]
                    peer = str(elem.peer_asn)
                    # judge whether the as-path is legal
                    if len(as_path) > 1 and as_path[0] == peer:
                        if re.match(self.pattern, elem.fields['prefix']):
                            self.as_prefix[as_path[-1]].add(elem.fields['prefix'])
                            # add edges to the graph
                            for i in range(0, len(as_path) - 1):
                                as_graph.add_edge(as_path[i], as_path[i + 1])
                    elem = rec.get_next_elem()

        as_graph = self.add_geo_loc(as_graph)

        return as_graph

    def add_geo_loc(self, as_graph):
        as_prefix = self.as_prefix
        # use IP2Location database to get the link between AS and longtitude
        ip2_loc_obj = IP2Location.IP2Location()

        if self.mode == "IPv4":
            ip2_loc_obj.open("IP2LocationDBs/IP2LOCATION-LITE-DB5.BIN")
        elif self.mode == "IPv6":
            ip2_loc_obj.open("IP2LocationDBs/IP2LOCATION-LITE-DB5.IPV6.BIN")

        for key in as_prefix:
            # calculate the average longtitude of a set of prefixes for one asn
            sum_longtitude = 0.0
            for elem in as_prefix[key]:
                ip_adress = elem.split('/')
                try:
                    rec = ip2_loc_obj.get_all(ip_adress[0])
                except ValueError:
                    continue
                sum_longtitude += rec.longitude
            # set the longtitude as a attribute of the node in graph
            if len(as_prefix[key]) > 0:
                as_graph.add_node(key, longtitude=sum_longtitude / len(as_prefix[key]))
        return as_graph

    def get_as_prefix(self):
        return self.as_prefix


from collections import defaultdict, deque
from typing import List

class Solution:
    def canVisitAllRooms(self, rooms: List[List[int]]) -> bool:
        """
        rooms = [[1],[2],[3],[]] return True
        """
        roomMap = defaultdict(list)
        n = len(rooms)
        for i in range(n):
            for j in range(len(rooms[i])):
                roomMap[i].append(rooms[i][j])
        count = 0
        queue = deque()
        visited = [False for _ in range(n)]
        queue.append(0)
        while len(queue):
            current = queue.popleft()
            if visited[current]:
                continue
            visited[current]=True
            count+=1
            if count>=n:
                return True
            keys = roomMap[current]
            if len(keys)>0:
                for e in keys:
                    queue.append(e)
        return count>=n

if __name__ == '__main__':
    rooms = [[1], [2], [3], []]
    rooms1 = [[1, 3], [3, 0, 1], [2], [0]]
    print(Solution().canVisitAllRooms(rooms))
    print(Solution().canVisitAllRooms(rooms1))


import json

class RelationActionListLazy(In.core.lazy.HTMLObjectLazy):
	'''list of comments'''

	def __init__(self, data = None, items = None, **args):

		super().__init__(data, items, **args)

		# always set new id
		self.id = 'RelationActionListLazy'

		context = IN.context
		if context.request.ajax_lazy:

			db = IN.db
			connection = db.connection

			nabar_id = context.nabar.id

			if not nabar_id:
				return

			query = IN.db.select({
				'table' : ['relation.relation', 'r'],
				'columns' : ['r.id', 'rr.from_entity_id', 'r.type'],
				'join' : [
					#['inner join', 'relation.relation_member', 'rm', [
					#	['r.id = rm.relation_id']
					#]],
					['inner join', 'relation.relation_request', 'rr', [
						['r.id = rr.relation_id']
					]]
				],
				'where' : [
					['r.relation_status', IN.relater.RELATION_STATUS_REQUESTED],
					['r.status', '>=', 1],
					['rr.to_entity_type', 'Nabar'],
					['rr.to_entity_id', nabar_id]
				],
				'order' : {'rr.created' : 'desc'}
			})

			cursor = query.execute()

			__form_load = IN.former.load

			if cursor.rowcount > 0:

				for row in cursor:

					from_entity_id = row['from_entity_id']

					form = __form_load('RelationActionDirect', args = {
						'from_entity_type' : 'Nabar',
						'from_entity_id' : nabar_id,	# viewer
						'to_entity_type' : 'Nabar',
						'to_entity_id' : from_entity_id,# page
						'relation_type' : row['type']
					})

					self.add(form)

					#remaining = total - limit
					#if remaining > 0 and  last_id > 0:
						#self.add('TextDiv', {
							#'id' : '_'.join(('more-commens', parent_entity_type, str(parent_entity_id), str(self.parent_id))),
							#'value' : str(remaining) + ' more comments',
							#'css' : ['ajax i-text-center pointer i-panel-box i-panel-box-primary'],
							#'attributes' : {
								#'data-href' : ''.join(('/comment/more/!Content/', str(parent_entity_id), '/', str(last_id), '/', str(self.parent_id)))
							#},
							#'weight' : -1
						#})

@IN.register('RelationActionListLazy', type = 'Themer')
class RelationActionListLazyThemer(In.core.lazy.HTMLObjectLazyThemer):
	'''lazy themer'''

	def theme_attributes(self, obj, format, view_mode, args):

		obj.lazy_args['type'] = obj.__type__

		json_args = ''
		try:
			json_args = json.dumps(obj.lazy_args, skipkeys = True, ensure_ascii = False)
		except Exception as e:
			print(9999999999, obj.lazy_args, obj.__type__)
			raise e

		obj.attributes['data-args'] = json_args
		return super().theme_attributes(obj, format, view_mode, args)


from datetime import date
import re
import sys

from django.shortcuts import render
from django.http import JsonResponse
from django.utils.safestring import SafeString
from .lib.dwd import  Forecast, Plz, Monattage

# Create your view
def index(request):
    return render(request, 'wetter/index.html')

def daten(request, geo, monattag):
    if "ort" in request.GET:
        ort = re.split("\ \d", request.GET['ort'])[0]
    else:
        ort = ""

    p1 = geo.split(",")
    try:
        fc = Forecast(p1[0], p1[1], 100, monattag, 3)
        fc.get_aggregates()
        jstr = SafeString(fc.make_history())
        jstr = jstr.replace("None", "null")
        jstr = jstr.replace("'", '"')
        heute = date.today()
        stations = [(x.stationsname, x.bundesland, x.stationshoehe, x.distance)
                    for x in fc.stations]
        if len(set([x[1] for x in stations])) == 1:
            bundesland = "/" + stations[0][1]
        else:
            bundesland = ""

        return render(
            request, 'wetter/daten.html', {
                "aggr": fc.aggregates,
                "geoBreite": p1[0],
                "geoLaenge": p1[1],
                "monattag": str(fc.tag) + ". " + fc.get_month(fc.monat) + "",
                "monattatstr": str(fc.tag),
                "ort": ort,
                "bundesland": bundesland,
                "history": SafeString(jstr),
                "stationen": stations,
                "heutetag": heute.day,
                "heutemonat": fc.get_month(heute.month)[0:3],
                "heutejahr": heute.year
            })
    except:
        msg = "Entschuldigung, das hat leider nicht geklappt. \
        Wahrscheinlich gab es eine Problem mit der Bereitsstellung der Daten. \
        Versuche es bitte mit einen anderen Ort. "
        e = sys.exc_info()[0]

        return render(request, "wetter/index.html", {"error": msg,"err_info":e})

def plz(request):
    plz = Plz()
    data = plz.query(request.GET['query'])
    return JsonResponse(data, safe=False)

def monattage(request):
    monattage = Monattage()
    data = monattage.query(request.GET['query'])
    return JsonResponse(data, safe=False)


# -*- coding: utf-8 -*-

import datetime

from django.contrib.auth.models import User

from django.core.management.base import BaseCommand, CommandError
import sys
from random import randint

import logging
from django.contrib.contenttypes.models import ContentType
from lugati.products.models import Product, ProductPrice, Brand
import json
from django.core.files import File
from django.contrib.sites.models import Site
import psycopg2
from lugati.lugati_media.models import ThebloqImage
import requests
#from urllib.parse import urljoin
try:  # Python 3
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin
import lxml
from lxml.html import fromstring
from BeautifulSoup import BeautifulSoup

import urllib2
import json
import urllib
from lxml import html
from lxml import etree

logger = logging.getLogger(__name__)
from django.conf import settings
import os
from urlparse import urlparse

BASE_FB_ALBUM_NAME='Thebloq'

CUR_SITE = 10

DOMAIN = 'http://www.vigorcentre.ru'

class Command(BaseCommand):
    help = 'general procedures'

    def handle(self, *args, **options):
        logger.info('start import products: ' + str(datetime.datetime.now()))
        self.recompose_creations()

    def format_str(self, arg, delimiter=' '):
        res = ''
        for s in arg:
            res += (s + delimiter)
        return res.strip()

    def create_product(self, name, img_srcs, preview, description, cat, brand=None, item_link='', price = 0):
        product = Product()
        product.name = name
        product.preview = preview
        product.site = Site.objects.get(pk=CUR_SITE)
        product.parent_object = cat
        product.brand = brand
        product.description = description
        product.additional_info = item_link.strip()
        product.save()

        product.set_price(price)

        for img_src in img_srcs:
            tb_image = ThebloqImage()

            file_save_dir = 'static/upload'
            filename = urlparse(img_src).path.split('/')[-1]

            r = requests.get(img_src)
            with open(os.path.join(file_save_dir, filename),'wb') as f:
                f.write(r.content)

            tb_image.file = File(open(os.path.join(file_save_dir, filename)))
            tb_image.content_object = product
            tb_image.save()

    def get_description_price(self,base_link, product_link):
        r = requests.get(base_link+product_link)
        doc = html.fromstring(r.text)
        #desc = self.format_str(arg=doc.xpath('//*[@id="main"]/div/div[1]/div[2]/p/text()'), delimiter='\n')
        desc = etree.tostring(doc.xpath('//*[@id="main"]/div/div[1]/div[2]')[0], pretty_print=True)
        price = float(doc.xpath('//*[@id="main"]/div/div[1]/div[1]/div/p[1]/ins/strong/text()')[0])
        return desc, price

    def get_brand_pages(self, base_link, brand_link, cat, brand):

        r = requests.get(base_link + brand_link)
        doc = html.fromstring(r.text)
        pages = doc.xpath('//*[@id="pager"]/p/a')

        self.create_products(base_link=base_link, category_link=brand_link, cat=cat, brand=brand)
        for page in pages:
            self.create_products(base_link=base_link, category_link=page.attrib['href'], cat=cat, brand=brand)

    def get_brands(self, base_link, category_link, cat):
        r = requests.get(base_link+category_link)
        doc = html.fromstring(r.text)
        items = doc.xpath('//*[@id="vendors"]/p/span')

        for item in items:
            brand_name = item.text

            brand = None
            try:
                brand = Brand.objects.get(name=brand_name)
            except:
                brand = Brand()
                brand.name = brand_name
                brand.site = Site.objects.get(pk=CUR_SITE)
                brand.save()

            brand_link = item.attrib['rel']
            self.get_brand_pages(base_link=base_link, brand_link=brand_link, cat=cat, brand=brand)

    def create_products(self, base_link, category_link, cat, brand):
        r = requests.get(base_link+category_link)
        # doc = html.fromstring(r.text)
        # items = doc.xpath('//*[@id="leaders"]/div')
        #
        # for item in items:
        #
        #     name = self.format_str(item.xpath('div/p/a/text()'))
        #     item_link = item.xpath('div/p/a')[0].attrib['href']
        #
        #     img_srcs = []
        #     img_src = base_link+(item.xpath('div/p[2]/span/img')[0].attrib['src']).replace('/resources/catalog/small','/resources/catalog').strip()
        #     img_srcs.append(img_src)
        #
        #     preview = self.format_str(item.xpath('div/p[3]/text()'))
        #     description, price = self.get_description_price(base_link, item_link)
        #
        #     self.create_product(name=name, img_srcs=img_srcs, preview=preview, description=description, cat=cat, brand=brand, item_link=item_link, price=price)

    def get_prods_list(self, base_link):
        r = requests.get(DOMAIN+base_link)
        doc = html.fromstring(r.text)
        prods = doc.xpath('/html/body/div[5]/div/div[2]/div[2]/div/div[1]/a')
        for prod in prods:
            prod_link = prod.get('href')

    def get_sub_links(self, base_link):

        r = requests.get(DOMAIN+base_link)
        doc = html.fromstring(r.text)
        cats = doc.xpath('/html/body/div[5]/div/div[2]/div[2]/div/div[1]/div[1]/div')
        for cat in cats:
            href = cat.xpath('a')[0].get('href')
            self.get_prods_list(href)

    def get_links(self, base_link):

        r = requests.get(base_link)
        doc = html.fromstring(r.text)

        cats = doc.xpath('/html/body/div[5]/div/div[2]/div[2]/div/div/div')
        for cat in cats:
            href = cat.xpath('a')[0].get('href')
            self.get_sub_links(href)

    def recompose_creations(self):
        logger.info('deleting')
        products = Product.objects.filter(site=None)
        for product in products:
            product.get_images().delete()
            product.get_prices().delete()
        products.delete()
        products = Product.objects.filter(site=Site.objects.get(pk=CUR_SITE))
        for product in products:
            product.get_images().delete()
            product.get_prices().delete()
        products.delete()
        Brand.objects.filter(site=Site.objects.get(pk=CUR_SITE)).delete()

        self.get_links('http://www.vigorcentre.ru/oborudovanie/')


'''
Created on Feb 19, 2017

@author: jjordening
'''

import cv2
import numpy as np

def minValImage(arr, channel=0):
    """
    Determines the minimum value of arr in a channel
    """
    return np.min(np.min(np.min(arr[:, :, :, channel], axis=1), axis=1), axis=0)

def maxValImage(arr, channel=0):
    """
    Determines the maximium value of arr in a channel
    """
    return np.max(np.max(np.max(arr[:, :, :, channel], axis=0), axis=0), axis=0)

def addGrayLayer(image):
    """
    Adds a gray layer as a fourth channel to an image

    Input:
        image

    Output:
        an array of images with the channels RGBGray
    """
    gray = cv2.cvtColor(image, cv2.CV_HLS2GRAY)
    return np.concatenate((image, gray.reshape((image.shape[0], image.shape[1], 1))), axis=2)

def addGradientLayer(image, sobel_kernel=3, magThresh=(0, 255), dirThresh=(0, np.pi/2)):
    """
    Adds a gray layer as a fourth channel to an image

    Input:
        image

    Output:
        an array of images with the channels RGBGray
    """
    sobelx = cv2.Sobel(image[:, :, 1], cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(image[:, :, 1], cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # Calculate the gradient magnitude
    gradMag = np.sqrt(sobelx**2 + sobely**2)
    absGradDir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    # Rescale to 8 bit
    scaleFactor = np.max(gradMag)/255
    gradMag = (gradMag/scaleFactor).astype(np.uint8)
    # Create a binary image of ones where thresholds are met, zeros otherwise
    binaryOutput = np.zeros_like(gradMag)
    binaryOutput[(gradMag >= magThresh[0]) & (gradMag <= magThresh[1]) &
                 (absGradDir >= dirThresh[0]) & (absGradDir <= dirThresh[1])] = 1
    # Return the binary image
    return np.concatenate((image, binaryOutput.reshape((image.shape[0], image.shape[1], 1))) , axis=2)

def applyNormalisation(image):
    """
        Applies a normalisation to an image with the channels RGBGray.
        It applies a CLAHE normalisation to the gray layer and then normalises the
        values such, that they have a mean of 0 and a deviation of 1

        Input:
            image

        Output:
            an array of images with the channels RGBGray
    """
    #clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
    #image[:,:,3] = clahe.apply(image[:,:,3])
    return image / 255.

def convertRGBToHSV(img):
    temp = np.zeros(img.shape)
    for i in range(img.shape[0]):
        for j in range(img.shape[1]):
            r = img[i, j, 0]
            g = img[i, j, 1]
            b = img[i, j, 2]
            maximum = np.max([r, g, b])
            minimum = np.min([r, g, b])
            if maximum == minimum:
                temp[i, j, 0] = 0
            elif maximum == r:
                temp[i, j, 0] = 1./6.*(g - b)/(maximum - minimum)
            elif maximum == g:
                temp[i, j, 0] = 1./6.*(2 + (b - r)/(maximum - minimum))
            else:
                temp[i, j, 0] = 1./6.*(4+(r - g) / (maximum - minimum))
            if temp[i, j, 0] < 0 :
                temp[i, j, 0] += 1.
            temp[i, j, 1] = 0 if maximum == 0 else (maximum-minimum)/maximum
            temp[i, j, 2] = maximum
    return temp

def convertHSVToRGB(img):
    temp = np.zeros(img.shape)
    for i in range(img.shape[0]):
        for j in range(img.shape[1]):
            f = img[i, j, 0]*6.
            hi = int(f)
            f -= hi
            p = img[i, j, 2]*(1. - img[i, j, 1])
            q = img[i, j, 2]*(1. - f*img[i, j, 1])
            t = img[i, j, 2]*(1. - (1.-f)*img[i, j, 1])
            if hi == 0 or hi == 6:
                temp[i, j] = [img[i, j, 2], t, p]
            elif hi == 1:
                temp[i, j] = [q, img[i, j, 2], p]
            elif hi == 2:
                temp[i, j] = [p, img[i, j, 2], t]
            elif hi == 3:
                temp[i, j] = [p, q, img[i, j, 2]]
            elif hi == 4:
                temp[i, j] = [t, p, img[i, j, 2]]
            else:
                temp[i, j] = [img[i, j, 2], p, q]
    return temp

def augmentImageAndLabel(image, label, depth=None):
    image = lightImage(image)
    shiftHor = np.random.randint(-30,31)
    shiftVer = np.random.randint(-30,31)
    image = shiftImg(image, shiftHor, shiftVer)
    image = blurImage(image)
    label = shiftImg(label, shiftHor, shiftVer)
    np.clip(image, 0, 255, image)
    if depth is not None:
        depth = shiftImg(depth, shiftHor, shiftVer)
        return image, label, depth
    return image, label

def augmentImage(image, label):
    #rot = int(np.random.rand()**2*90 -45)
    rot = int(np.random.rand()**2*60 - 30)
    image = rotateImage(image, rot)
    # Add a part of the rotated angle, as it is counted counter-clockwise.
    # If you turn counter-clockwise, this looks like the car would be more left
    # and needs to drive to the right -> add some angle
    # divide it by the maximum of the steering angle in deg ->25
    label[0] -= 3*rot/(30)
    label[1] -= .2*rot/20
    #label[2] += .1*rot/10
    shiftHor = np.random.randint(-40,41)
    shiftVer = np.random.randint(-5,6)
    image = shiftImg(image, shiftHor, shiftVer)
    #label[0] *= (1-shiftVer/100)
    label[0] += 2.5*shiftHor/(40)
    label[1] -= .3*shiftHor/40
    #label[2] += .1*shiftHor/40steering
    label[0] = min(max(label[0], -5),5)
    label[1] = min(max(label[1], -1),1)
    label[2] = min(max(label[2], -1),1)
    #image = lightImage(blurImage(image))
    return image, label

def blurImage(img):
    # Blur image with random kernel
    kernel_size = np.random.randint(1, 5)
    if kernel_size % 2 != 1:
        kernel_size += 1
    return cv2.GaussianBlur(img, (kernel_size, kernel_size), 0)

def lightImage(img):
    # HSV brightness transform
    img = cv2.cvtColor(img,cv2.COLOR_RGB2HSV)
    brightness = np.random.uniform(0.8,1.1)
    img[:,:,2] = img[:,:,2]*brightness
    np.clip(img, 0, 255, img)
    return cv2.cvtColor(img,cv2.COLOR_HSV2RGB)

def preprocessImage(image):
    """
    This function represents the default preprocessing for
    an image to prepare them for the network
    """
    #image = cv2.resize(image, (320, 160))[::-1]
    #image = image[image.shape[0]*1//5:image.shape[0]*7//8,:,:]
    #image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
    #image = cv2.convertScaleAbs(image, alpha=(1))
    #image = addGradientLayer(image, 7, (100,255), (0, np.pi/2))
    image = applyNormalisation(image)
    return image

def preprocessImages(arr):
    """
    This function represents the default preprocessing for
    images to prepare them for the network
    """
    return np.array([preprocessImage(image) for image in arr])

def perspectiveTransform(img, M):
    h, w = img.shape[:2]
    return cv2.warpPerspective(img, M, (w,h), flags=cv2.INTER_LINEAR)

def shiftImg(arr, horizontal, vertical):
    if len(arr.shape) == 2:
        if(horizontal > 0):
            arr = np.concatenate((arr[horizontal:,:],np.zeros((horizontal,arr.shape[1]))), axis=0)
        elif(horizontal < 0):
            arr = np.concatenate((np.zeros((np.abs(horizontal),arr.shape[1])), arr[:horizontal,:]), axis=0)
        if(vertical > 0):
            arr = np.concatenate((arr[:,vertical:],np.zeros((arr.shape[0],vertical))), axis=1)
        elif(vertical < 0):
            arr = np.concatenate((np.zeros((arr.shape[0], np.abs(vertical))), arr[:,:vertical]), axis=1)

    else:
        if(horizontal > 0):
            arr = np.concatenate((arr[horizontal:], np.zeros((horizontal, arr.shape[1], arr.shape[2]))), axis=0)
        elif(horizontal < 0):
            arr = np.concatenate((np.zeros((np.abs(horizontal), arr.shape[1], arr.shape[2])), arr[:horizontal]), axis=0)
        if(vertical > 0):
            arr = np.concatenate((arr[:, vertical:], np.zeros((arr.shape[0], vertical, arr.shape[2]))), axis=1)
        elif(vertical < 0):
            arr = np.concatenate((np.zeros((arr.shape[0], np.abs(vertical), arr.shape[2])), arr[:,:vertical]), axis=1)
    return arr

"""def shiftImg(arr, horizontal, vertical):

        This function shifts an image horizontally and vertically
        Input:
            horizontal - amplitude of shift in pixels (positive to the right
            negative to the left)
            vertical - aplitude of the ishift in pixels (positive downwards
            negative upwards)

    M = np.float32([[1,0,1*horizontal],[0,1,vertical]])
    return cv2.warpAffine(arr,M,(arr.shape[1], arr.shape[0]))"""

def mirrorImage(img):
    """
        This function mirrors the handed image around the y-axis
    """
    return img[:,::-1]

def rotateImage(img, angle):
    """
        Rotates image around the point in the middle of the bottom of the picture by
        angle degrees.
    """
    rotation = cv2.getRotationMatrix2D((img.shape[0], img.shape[1]//2), angle, 1)
    return cv2.warpAffine(img, rotation, (img.shape[1], img.shape[0]))

def rotateImages(arr, angles):
    """
        Rotates multiple images by the given angles.
    """
    arr = [rotateImage(img, angle) for img, angle in zip(arr, angles)]
    return arr


from django.shortcuts import render, redirect, reverse
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
import os
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.conf import settings
import numpy as np
from PIL import Image
import json
import math
import requests
from django.conf import settings

# Create your views here.
@login_required
def home(request):
    return render(request, 'doctors/home.html')

@login_required
def appointments(request):
    return redirect('calendar')

@login_required
def predict(request):
    if request.method == 'GET':
        return render(request, 'doctors/predict.html')
    else:
        print('entered else')
        print("file recieved")
        form = request.FILES
        print(form['file'])

        file_object = request.FILES["file"]
        file_name = str(file_object)
        print(f'[INFO] File Name: {file_name}')
        path = os.path.join('doctors/static/doctors/', file_name)
        with open(path, 'wb+') as f:
            for chunk in file_object.chunks():
                f.write(chunk)
        #images = os.listdir("path/to/images") #Can use glob as well
        url = 'https://skin-cancer-sample.herokuapp.com/'
        img = np.asarray(Image.open(path).resize((256,256)))
        # img.save(settings.MEDIA_URL + 'cancer_images/', file_name)
        #print(img)
        class NumpyEncoder(json.JSONEncoder):
            def default(self, obj):
                if isinstance(obj, np.ndarray):
                    return obj.tolist()
                return json.JSONEncoder.default(self, obj)
        json_dump = json.dumps({"image":img}, cls = NumpyEncoder)
        #print(json_dump)
        send_request = requests.post(url, json_dump)
        print(send_request)
        result = send_request.json()['results']
        print(result)
        index = np.argmax(result)
        print(result[0][index])

        cancers = {
            0: 'Actinic kerastose',
            1: 'Basal cell carcinoma',
            2: 'Benign keratosis-like lesions',
            3: 'Dermatofibroma',
            4: 'Melanocytic nevi',
            5: 'Melanoma',
            6: 'Vascular lessions'
        }
        path = 'doctors/' + file_name
        context = {
        "detected": cancers[index],
        "percentage": math.trunc(result[0][index]*100),
        "address": "Hyderabad, India",
        'images': path
        }

        return render(request, 'doctors/predict_res.html',context)


from string import ascii_lowercase
class Solution:
    def findWords(self, board, words):
        row = len(board)
        col = len(board[0])

        res = set()
        wdb = {}
        visited = None
        for c in ascii_lowercase:
            t = {w for w in words if w[0] == c}
            if len(t) > 0:
                wdb[c] = t

        def getNeighbours(r,c):
            n = []
            if r > 0 and not visited[r-1][c]:
                n.append([r-1, c])
            if c > 0 and not visited[r][c-1]:
                n.append([r, c-1])
            if r < len(visited) - 1 and not visited[r+1][c]:
                n.append([r+1, c])
            if c < len(visited[0]) - 1 and not visited[r][c+1]:
                n.append([r,c+1])
            return n

        def findWords(r, c, cw):
            lst = wdb[cw[0]]
            if cw in lst:
                res.add(cw)

            nb = getNeighbours(r, c)
            for n in nb:
                x,y = n
                nc = board[x][y]

                if any(w.startswith(cw+nc) for w in lst):
                    visited[x][y] = True
                    findWords(x,y, cw+nc)
                    visited[x][y] = False

        for i in range(row):
            for j in range(col):
                if board[i][j] in wdb:
                    visited = [[False]*col for _ in range(row)]
                    visited[i][j] = True
                    findWords(i, j, board[i][j])
        return res

if __name__ == '__main__':
    sol = Solution()

    board = [
        ['o','a','a','n'],
        ['e','t','a','e'],
        ['i','h','k','r'],
        ['i','f','l','v']
    ]
    words = ["oath","pea","eat","rain"]
    r = sol.findWords(board, words)
    print(r)


#
# Copyright (c) 2020-2021 Arm Limited and Contributors. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
import pathlib

from unittest import mock

import pytest

from mbed_tools.devices._internal.file_parser import OnlineId, read_device_files

class TestReadDeviceFiles:
    def test_finds_daplink_compatible_device_files(self, tmp_path):
        details = pathlib.Path(tmp_path, "details.txt")
        htm = pathlib.Path(tmp_path, "mbed.htm")
        htm.write_text("code=2222")
        details.write_text("Version: 2")

        info = read_device_files([tmp_path])

        assert info.product_code is not None
        assert info.interface_details is not None

    def test_finds_jlink_device_files(self, tmp_path):
        segger_html = pathlib.Path(tmp_path, "Segger.html")
        board_html = pathlib.Path(tmp_path, "Board.html")
        segger_html.write_text(build_segger_html())
        board_html.write_text(build_board_html())

        info = read_device_files([tmp_path])

        assert info.online_id is not None
        assert info.interface_details is not None

    def test_warns_if_no_device_files_found(self, caplog, tmp_path):
        read_device_files([tmp_path])

        assert str(tmp_path) in caplog.text

    def test_skips_hidden_files(self, caplog, tmp_path):
        auth = "101000000000000000000002F7F35E602eeb0bb9b632205c51f6c357aeee7bc9"
        file_contents = (
            '<meta http-equiv="refresh" '
            f'content="0; url=http://mbed.org/start?auth={auth}&loader=11972&firmware=16457&configuration=4" />'
        )
        pathlib.Path(tmp_path, "._MBED.HTM").write_text(file_contents)
        pathlib.Path(tmp_path, "._DETAILS.TXT").write_text("Version: 2222")

        assert read_device_files([tmp_path]).product_code is None
        assert not read_device_files([tmp_path]).interface_details

    def test_handles_os_error_with_warning(self, tmp_path, caplog, monkeypatch):
        bad_htm = pathlib.Path(tmp_path, "MBED.HTM")
        bad_htm.touch()
        monkeypatch.setattr(pathlib.Path, "read_text", mock.Mock(side_effect=OSError))

        read_device_files([tmp_path])
        assert str(bad_htm) in caplog.text

class TestExtractProductCodeFromHtm:
    def test_reads_product_code_from_code_attribute(self, tmp_path):
        code = "02400201B80ECE4A45F033F2"
        file_contents = f'<meta http-equiv="refresh" content="0; url=http://mbed.org/device/?code={code}"/>'
        pathlib.Path(tmp_path, "MBED.HTM").write_text(file_contents)

        assert read_device_files([tmp_path]).product_code == code[:4]

    def test_reads_product_code_from_auth_attribute(self, tmp_path):
        auth = "101000000000000000000002F7F35E602eeb0bb9b632205c51f6c357aeee7bc9"
        file_contents = (
            '<meta http-equiv="refresh" '
            f'content="0; url=http://mbed.org/start?auth={auth}&loader=11972&firmware=16457&configuration=4" />'
        )
        pathlib.Path(tmp_path, "MBED.HTM").write_text(file_contents)

        assert read_device_files([tmp_path]).product_code == auth[:4]

    def test_none_if_no_product_code(self, tmp_path):
        file_contents = '<meta http-equiv="refresh" content="0; url=http://mbed.org/config" />'
        pathlib.Path(tmp_path, "MBED.HTM").write_text(file_contents)

        assert read_device_files([tmp_path]).product_code is None

    def test_extracts_first_product_code_found(self, tmp_path):
        auth = "101000000000000000000002F7F35E602eeb0bb9b632205c51f6c357aeee7bc9"
        file_contents_1 = (
            '<meta http-equiv="refresh" '
            f'content="0; url=http://mbed.org/start?auth={auth}&loader=11972&firmware=16457&configuration=4" />'
        )
        code = "02400201B80ECE4A45F033F2"
        file_contents_2 = f'<meta http-equiv="refresh" content="0; url=http://mbed.org/device/?code={code}"/>'
        directory_1 = pathlib.Path(tmp_path, "test-1")
        directory_1.mkdir()
        directory_2 = pathlib.Path(tmp_path, "test-2")
        directory_2.mkdir()
        pathlib.Path(directory_1, "mbed.htm").write_text(file_contents_1)
        pathlib.Path(directory_2, "mbed.htm").write_text(file_contents_2)

        result = read_device_files([directory_1, directory_2])

        assert result.product_code == auth[:4]

class TestExtractOnlineIDFromHTM:
    def test_reads_online_id_from_url(self, tmp_path):
        url = "https://os.mbed.com/platforms/THIS-IS_a_SLUG_123/"
        file_contents = f"window.location.replace({url});"
        pathlib.Path(tmp_path, "MBED.HTM").write_text(file_contents)

        assert read_device_files([tmp_path]).online_id == OnlineId(target_type="platform", slug="THIS-IS_a_SLUG_123")

    def test_none_if_not_found(self, tmp_path):
        file_contents = "window.location.replace(https://os.mbed.com/about);"
        pathlib.Path(tmp_path, "MBED.HTM").write_text(file_contents)

        assert read_device_files([tmp_path]).online_id is None

class TestExtractsJlinkData:
    def test_reads_board_slug(self, tmp_path):
        board_html = pathlib.Path(tmp_path, "Board.html")
        board_html.write_text(build_board_html("http://test.com/test/slug"))

        info = read_device_files([tmp_path])

        assert info.online_id == OnlineId(target_type="jlink", slug="slug")

    def test_reads_board_slug_ignore_extension(self, tmp_path):
        board_html = pathlib.Path(tmp_path, "Board.html")
        board_html.write_text(build_board_html("http://test.com/slug.html"))

        info = read_device_files([tmp_path])

        assert info.online_id == OnlineId(target_type="jlink", slug="slug")

    def test_id_none_if_no_board_slug(self, tmp_path):
        board_html = pathlib.Path(tmp_path, "Board.html")
        board_html.write_text(build_board_html("http://test.com"))

        info = read_device_files([tmp_path])

        assert info.online_id is None

    def test_reads_segger_slug(self, tmp_path):
        segger_html = pathlib.Path(tmp_path, "Segger.html")
        segger_html.write_text(build_segger_html("variant"))

        info = read_device_files([tmp_path])

        assert info.interface_details.get("Version") == "variant"

    def test_interface_empty_if_not_found(self, tmp_path):
        board_html = pathlib.Path(tmp_path, "Board.html")
        board_html.write_text("")

        info = read_device_files([tmp_path])

        assert info.interface_details == {}

# Helpers to build test data
def build_short_details_txt(version="0226", build="Aug 24 2015 17:06:30", commit_sha="27a2367", local_mods="Yes"):
    return (
        f"""Version: {version}
Build:   {build}
Git Commit SHA: {commit_sha}
Git Local mods: {local_mods}
""",
        {"Version": version, "Build": build, "Git Commit SHA": commit_sha, "Git Local mods": local_mods},
    )

def build_long_details_txt(
    interface_version="0226",
    uid="0240000029164e45002f0012706e0006f301000097969900",
    hif_id="97969900",
    auto_reset="0",
    auto_allow="0",
    daplink_mode="Interface",
    commit_sha="c76899838",
    local_mods="0",
    usb_ifaces="MSD, CDC, HID",
    iface_crc="0x26764ebf",
):
    return (
        f"""# DAPLink Firmware - see https://mbed.com/daplink
Unique ID: {uid}
HIF ID: {hif_id}
Auto Reset: {auto_reset}
Automation allowed: {auto_allow}
Daplink Mode: {daplink_mode}
Interface Version: {interface_version}
Git SHA: {commit_sha}
Local Mods: {local_mods}
USB Interfaces: {usb_ifaces}
Interface CRC: {iface_crc}
""",
        {
            "Unique ID": uid,
            "HIF ID": hif_id,
            "Auto Reset": auto_reset,
            "Automation allowed": auto_allow,
            "Daplink Mode": daplink_mode,
            "Version": interface_version,
            "Git SHA": commit_sha,
            "Local Mods": local_mods,
            "USB Interfaces": usb_ifaces,
            "Interface CRC": iface_crc,
        },
    )

def build_segger_html(model="j-link-ob"):
    return (
        """<meta http-equiv="refresh" content="0;"""
        f"""url=https://www.segger.com/products/debug-probes/j-link/models/{model}/"/>"""
        """"""
    )

def build_board_html(target_url="http://www.nxp.com/FRDM-K64F"):
    return f"""<meta http-equiv="refresh" content="0; url={target_url}"/>"""

class TestReadDetailsTxt:
    @pytest.mark.parametrize(
        "content, expected",
        (
            build_short_details_txt(),
            build_short_details_txt(version="0777", commit_sha="99789s", local_mods="No"),
            build_long_details_txt(),
            ("", {}),
            ("\n", {}),
            ("blablablablaandbla", {}),
            ("blablabla\nblaandbla\nversion : 2\n\n", {"version": "2"}),
        ),
        ids=("short", "short2", "long", "empty", "newline", "nosep", "multiline"),
    )
    def test_parses_details_txt(self, content, expected, tmp_path):
        details_file_path = pathlib.Path(tmp_path, "DETAILS.txt")
        details_file_path.write_text(content)

        result = read_device_files([tmp_path]).interface_details
        assert result == expected


import preprocess
import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
from sklearn.ensemble import GradientBoostingClassifier

#test phase
X_train,X_test,Y_train,Y_test = preprocess.load_data()
"""
#test test everywhere yay

model = GradientBoostingClassifier()

model.fit(X_train,Y_train)
y_predict = model.predict_proba(X_test)

print log_loss(Y_test,y_predict)
"""

#deploy
#loading data using numerai example
np.random.seed(0)
training_data = pd.read_csv('numerai_training_data.csv', header=0)
prediction_data = pd.read_csv('numerai_tournament_data.csv', header=0)
Y = training_data['target']
X = training_data.drop('target', axis=1)
t_id = prediction_data['t_id']
x_prediction = prediction_data.drop('t_id', axis=1)

model = GradientBoostingClassifier()
model.fit(X,Y)
y_prediction = model.predict_proba(x_prediction)
results = y_prediction[:, 1]
results_df = pd.DataFrame(data={'probability':results})
joined = pd.DataFrame(t_id).join(results_df)

print("Writing predictions from Oracle Lamp to CSV File...")
# Save the predictions out to a CSV file
joined.to_csv("predictions-gbm.csv", index=False)


# -*- coding: utf-8 -*-
from odoo import models, fields, api
from datetime import datetime, date

class Pointing(models.Model):
    _name = 'btp_pointing.pointing'

    @api.one
    @api.depends('construction_site_id.name', 'date')
    def _compute_name(self):
        if self.construction_site_id and self.date:
            self.name = "Pointages " + (self.construction_site_id.name).lower() + " du " + datetime.strptime(self.date,
                                                                                '%Y-%m-%d').strftime(
                '%d-%m-%Y')

    name = fields.Char(string="Nom", compute='_compute_name', store=True)
    date = fields.Date(string='Date', required=True, default=date.today())
    construction_site_id = fields.Many2one('construction_site.construction_site', string='Chantier', required=True)
    stage_ids = fields.One2many('btp_pointing.pointing.stage', 'pointing_id',
                                string="Étapes de pointages")

    _sql_constraints = [
        ('name_uniq', 'unique(name)',
         "Le nom doit être unique.\n Veuillez corriger pour continuer!"),
    ]


def skip_dyck(mot,indice):
    compteur_dyck = -1
    for k in range(indice-1,-1,-1):
        if compteur_dyck == 0:
            return k+1
        if mot[k] == "[" or mot[k] == "]":
            compteur_dyck += (-1)**(mot[k] == "]")
    return "Impossible"

def predecesseur(mot, indice, alphabet):
    if indice == 0:
        return "Impossible"
    if mot[indice - 1] in alphabet:
        return indice - 1
    elif mot[indice - 1] == "]":
        j = skip_dyck(mot,indice - 1)
        return predecesseur(mot, j, alphabet)
    else:
        return predecesseur(mot, indice - 1, alphabet)


import os

import torch

import coloredlogs
import logging
import numpy as np
from torch.utils.tensorboard import SummaryWriter

from models.seq_baseline import SeqBaselineModel

logger = logging.getLogger('BaselineLog')
coloredlogs.install(logger=logger, level='DEBUG',
                    fmt='%(asctime)s - %(name)s - %(levelname)s'
                        ' - %(message)s')
tensorboard_writer = SummaryWriter(log_dir='runs/Baseline')

class Baseline:
    def __init__(self, config):
        self.base_path = config['base_path']
        self.stamp = config['stamp']
        self.updates = config['num_updates']
        self.epochs = config['num_meta_epochs']
        self.early_stopping = config['early_stopping']
        self.stopping_threshold = config.get('stopping_threshold', 1e-3)

        if 'seq' in config['meta_model']:
            self.baseline_model = SeqBaselineModel(config)

        logger.info('Baseline instantiated')

    def training(self, train_episodes, val_episodes):
        best_loss = float('inf')
        best_f1 = 0
        patience = 0
        model_path = os.path.join(self.base_path, 'saved_models', 'Baseline-{}.h5'.format(self.stamp))
        logger.info('Model name: Baseline-{}.h5'.format(self.stamp))
        for epoch in range(self.epochs):
            logger.info('Starting epoch {}'.format(epoch + 1))
            losses, accuracies, precisions, recalls, f1s = self.baseline_model(train_episodes, self.updates)
            avg_loss = np.mean(losses)
            avg_accuracy = np.mean(accuracies)
            avg_precision = np.mean(precisions)
            avg_recall = np.mean(recalls)
            avg_f1 = np.mean(f1s)

            logger.info('Meta train epoch {}: Avg loss = {:.5f}, avg accuracy = {:.5f}, avg precision = {:.5f}, '
                        'avg recall = {:.5f}, avg F1 score = {:.5f}'.format(epoch + 1, avg_loss, avg_accuracy,
                                                                            avg_precision, avg_recall, avg_f1))
            tensorboard_writer.add_scalar('Loss/train', avg_loss, global_step=epoch + 1)

            losses, accuracies, precisions, recalls, f1s = self.baseline_model(val_episodes, self.updates, testing=True)
            avg_loss = np.mean(losses)
            avg_accuracy = np.mean(accuracies)
            avg_precision = np.mean(precisions)
            avg_recall = np.mean(recalls)
            avg_f1 = np.mean(f1s)

            logger.info('Meta val epoch {}: Avg loss = {:.5f}, avg accuracy = {:.5f}, avg precision = {:.5f}, '
                        'avg recall = {:.5f}, avg F1 score = {:.5f}'.format(epoch + 1, avg_loss, avg_accuracy,
                                                                            avg_precision, avg_recall, avg_f1))
            tensorboard_writer.add_scalar('Loss/val', avg_loss, global_step=epoch + 1)

            if avg_f1 > best_f1 + self.stopping_threshold:
                patience = 0
                best_loss = avg_loss
                best_f1 = avg_f1
                torch.save(self.baseline_model.learner.state_dict(), model_path)
                logger.info('Saving the model since the F1 improved')
            else:
                patience += 1
                logger.info('F1 did not improve')
                if patience == self.early_stopping:
                    break

            # Log params and grads into tensorboard
            for name, param in self.baseline_model.named_parameters():
                if param.requires_grad and param.grad is not None:
                    tensorboard_writer.add_histogram('Params/' + name, param.data.view(-1),
                                                     global_step=epoch + 1)
                    tensorboard_writer.add_histogram('Grads/' + name, param.grad.data.view(-1),
                                                     global_step=epoch + 1)

        self.baseline_model.learner.load_state_dict(torch.load(model_path))
        return best_f1

    def testing(self, test_episodes):
        logger.info('---------- Baseline testing starts here ----------')
        episode_accuracies, episode_precisions, episode_recalls, episode_f1s = [], [], [], []
        for episode in test_episodes:
            loss, accuracy, precision, recall, f1_score = self.baseline_model([episode], updates=self.updates, testing=True)
            loss = loss[0]
            accuracy, precision, recall, f1_score = accuracy[0], precision[0], recall[0], f1_score[0]

            episode_accuracies.append(accuracy)
            episode_precisions.append(precision)
            episode_recalls.append(recall)
            episode_f1s.append(f1_score)

        logger.info('Avg meta-testing metrics: Accuracy = {:.5f}, precision = {:.5f}, recall = {:.5f}, '
                    'F1 score = {:.5f}'.format(np.mean(episode_accuracies),
                                               np.mean(episode_precisions),
                                               np.mean(episode_recalls),
                                               np.mean(episode_f1s)))

        return np.mean(episode_f1s)


#!/usr/bin/env python
u"""
plot_AIS_grid_4maps.py
Written by Tyler Sutterley (05/2023)
Creates 3 GMT-like plots of the Antarctic Ice Sheet
on a polar stereographic south (3031) projection

Grounded ice and islands from Eric Rignot grounded ice image
Glacial drainage basins updated from Rignot et al. (2012) and IMBIE-2 (2016)
2003-2004 MODIS mosaic of Antarctica (MOA) from NSIDC (Haran et al., 2013)
    https://nsidc.org/data/moa/

PYTHON DEPENDENCIES:
    numpy: Scientific Computing Tools For Python
        http://www.numpy.org
        http://www.scipy.org/NumPy_for_Matlab_Users
    scipy: Scientific Tools for Python
        http://www.scipy.org/
    netCDF4: Python interface to the netCDF C library
        https://unidata.github.io/netcdf4-python/netCDF4/index.html
    h5py: Python interface for Hierarchal Data Format 5 (HDF5)
        http://h5py.org
    matplotlib: Python 2D plotting library
        http://matplotlib.org/
        https://github.com/matplotlib/matplotlib
    cartopy: Python package designed for geospatial data processing
        https://scitools.org.uk/cartopy
    pyshp: Python read/write support for ESRI Shapefile format
        https://github.com/GeospatialPython/pyshp
    gdal: Pythonic interface to the Geospatial Data Abstraction Library (GDAL)
        https://pypi.python.org/pypi/GDAL/

UPDATE HISTORY:
    Updated 05/2023: use pathlib to define and operate on paths
        added option to set the input variable names or column order
    Updated 03/2023: switch from parameter files to argparse arguments
        updated inputs to spatial from_ascii function
    Updated 07/2022: place some imports behind try/except statements
    Updated 05/2022: use argparse descriptions within documentation
        use mask, shift grid and interpolation functions from tools
    Updated 12/2021: added color palette table (cpt) file reader from tools
    Updated 05/2021: define int/float precision to prevent deprecation warning
    Updated 10/2020: using spatial utilities for reading
        using argparse to set command-line parameters
    Forked 01/2020 from plot_AIS_grid_3maps.py
    Updated 10/2019: changing Y/N flags to True/False
    Updated 09/2019: added parameter for specifying if netCDF4 or HDF5
    Written 09/2019
"""
from __future__ import print_function

import sys
import os
import copy
import logging
import pathlib
import argparse
import warnings
import traceback
import numpy as np
import gravity_toolkit as gravtk

# attempt imports
try:
    import cartopy.crs as ccrs
except ModuleNotFoundError:
    warnings.filterwarnings("module")
    warnings.warn("cartopy not available", ImportWarning)
try:
    import matplotlib
    import matplotlib.pyplot as plt
    import matplotlib.cm as cm
    import matplotlib.colors as colors
    import matplotlib.ticker as ticker
    import matplotlib.offsetbox as offsetbox
    matplotlib.rcParams['axes.linewidth'] = 2.0
    matplotlib.rcParams['font.family'] = 'sans-serif'
    matplotlib.rcParams['font.sans-serif'] = ['Helvetica']
    matplotlib.rcParams['mathtext.default'] = 'regular'
except ModuleNotFoundError:
    warnings.filterwarnings("module")
    warnings.warn("matplotlib not available", ImportWarning)
try:
    import osgeo.gdal
except ModuleNotFoundError:
    warnings.filterwarnings("module")
    warnings.warn("GDAL not available", ImportWarning)
try:
    import shapefile
except ModuleNotFoundError:
    warnings.filterwarnings("module")
    warnings.warn("shapefile not available", ImportWarning)
# ignore warnings
warnings.filterwarnings("ignore")

# Antarctic 2012 basins
# region directory, filename, title and data type
region_dir = ['masks','Rignot_ANT']
region_title = ['AAp','ApB','BC','CCp','CpD','DDp','DpE','EEp','EpFp',
    'FpG','GH','HHp','HpI','IIpp','IppJ','JJpp','JppK','KKp','KpA']
# regional filenames
region_filename = 'basin_{0}_index.ascii'
# regional datatypes
region_dtype = {'names':('lat','lon'),'formats':('f','f')}

# IMBIE-2 Drainage basins
IMBIE_basin_file = ['masks','ANT_Basins_IMBIE2_v1.6','ANT_Basins_IMBIE2_v1.6.shp']
# basin titles within shapefile to extract
IMBIE_title = ('A-Ap','Ap-B','B-C','C-Cp','Cp-D','D-Dp','Dp-E','E-Ep','Ep-F',
    'F-Fp','F-G','G-H','H-Hp','Hp-I','I-Ipp','Ipp-J','J-Jpp','Jpp-K','K-A')

# background image mosaics
# MODIS mosaic of Antarctica
image_file = ['MOA','moa750_2004_hp1_v1.1.tif']

# Coastlines for antarctica (islands)
coast_file = ['masks','IceBoundaries_Antarctica_v02',
    'ant_ice_sheet_islands_v2.shp']

# Antarctica (AIS)
# x and y limit (modified from Bamber 1km DEM)
xlimits = np.array([-3100000,3100000])
ylimits = np.array([-2600000,2600000])
# cartopy transform for polar stereographic south
try:
    projection = ccrs.Stereographic(central_longitude=0.0,
        central_latitude=-90.0,true_scale_latitude=-71.0)
except (NameError,ValueError) as exc:
    pass

# PURPOSE: keep track of threads
def info(args):
    logging.info(pathlib.Path(sys.argv[0]).name)
    logging.info(args)
    logging.info(f'module name: {__name__}')
    if hasattr(os, 'getppid'):
        logging.info(f'parent process: {os.getppid():d}')
    logging.info(f'process id: {os.getpid():d}')

# PURPOSE: plot Rignot 2012 drainage basin polylines
def plot_rignot_basins(ax, base_dir):
    region_directory = base_dir.joinpath(*region_dir)
    # for each region
    for reg in region_title:
        # read the regional polylines
        region_file = region_directory.joinpath(region_filename.format(reg))
        region_ll = np.loadtxt(region_file, dtype=region_dtype)
        # converting region lat/lon into plot coordinates
        points = projection.transform_points(ccrs.PlateCarree(),
            region_ll['lon'], region_ll['lat'])
        ax.plot(points[:,0], points[:,1], color='k', transform=projection)

# PURPOSE: plot Antarctic drainage basins from IMBIE2 (Mouginot)
def plot_IMBIE2_basins(ax, base_dir):
    # read drainage basin polylines from shapefile (using splat operator)
    basin_shapefile = base_dir.joinpath(*IMBIE_basin_file)
    logging.debug(str(basin_shapefile))
    shape_input = shapefile.Reader(str(basin_shapefile))
    shape_entities = shape_input.shapes()
    shape_attributes = shape_input.records()
    # find record index for region by iterating through shape attributes
    # no islands or large regions
    i=[i for i,a in enumerate(shape_attributes) if a[1] in IMBIE_title]
    # for each valid shape entity
    for indice in i:
        # extract Polar-Stereographic coordinates for record
        points = np.array(shape_entities[indice].points)
        # IMBIE-2 basins can have multiple parts
        parts = shape_entities[indice].parts
        parts.append(len(points))
        for p1,p2 in zip(parts[:-1],parts[1:]):
            ax.plot(points[p1:p2,0], points[p1:p2,1], c='k',
                transform=projection)

# PURPOSE: plot Antarctic drainage sub-basins from IMBIE-2 (Mouginot)
def plot_IMBIE2_subbasins(ax, base_dir):
    # read drainage basin polylines from shapefile (using splat operator)
    IMBIE_subbasin_file = ['Basins_20Oct2016_v1.7','Basins_v1.7.shp']
    basin_shapefile = base_dir.joinpath('masks',*IMBIE_subbasin_file)
    logging.debug(str(basin_shapefile))
    shape_input = shapefile.Reader(str(basin_shapefile))
    shape_entities = shape_input.shapes()
    shape_attributes = shape_input.records()
    # iterate through shape entities and attributes
    indices = [i for i,a in enumerate(shape_attributes) if (a[1] != 'Islands')]
    for i in indices:
        # extract Polar-Stereographic coordinates for record
        points = np.array(shape_entities[i].points)
        # IMBIE-2 basins can have multiple parts
        parts = shape_entities[i].parts
        parts.append(len(points))
        for p1,p2 in zip(parts[:-1],parts[1:]):
            ax.plot(points[p1:p2,0], points[p1:p2,1], c='k',
                transform=projection)

# PURPOSE: plot Antarctic grounded ice delineation
def plot_grounded_ice(ax, base_dir, START=1):
    coast_shapefile = base_dir.joinpath(*coast_file)
    logging.debug(str(coast_shapefile))
    shape_input = shapefile.Reader(str(coast_shapefile))
    shape_entities = shape_input.shapes()
    shape_attributes = shape_input.records()
    i = [i for i,e in enumerate(shape_entities) if (np.ndim(e.points) > 1)]
    for indice in i[START:]:
        # extract Polar-Stereographic coordinates for record
        points = np.array(shape_entities[indice].points)
        ax.plot(points[:,0], points[:,1], c='k',
            transform=projection)

# PURPOSE: plot MODIS mosaic of Antarctica as background image
def plot_image_mosaic(ax, base_dir, MASKED=True):
    # read MODIS mosaic of Antarctica
    image_geotiff_file = base_dir.joinpath(*image_file)
    logging.debug(str(image_geotiff_file))
    ds = osgeo.gdal.Open(str(image_geotiff_file))
    # get dimensions
    xsize = ds.RasterXSize
    ysize = ds.RasterYSize
    # get geotiff info
    info_geotiff = ds.GetGeoTransform()
    # calculate image extents
    xmin = info_geotiff[0]
    ymax = info_geotiff[3]
    xmax = xmin + (xsize-1)*info_geotiff[1]
    ymin = ymax + (ysize-1)*info_geotiff[5]
    # read as grayscale image
    mosaic = np.ma.array(ds.ReadAsArray())
    # mask image mosaic
    if MASKED:
        # mask invalid values
        mosaic.fill_value = 0
        # create mask array for bad values
        mosaic.mask = (mosaic.data == mosaic.fill_value)
    # image extents
    extents=(xmin,xmax,ymin,ymax)
    # dataset range
    vmin, vmax = (0, 16386)
    # create color map with transparent bad points
    image_cmap = copy.copy(cm.gist_gray)
    image_cmap.set_bad(alpha=0.0)
    # nearest to not interpolate image
    im = ax.imshow(mosaic, interpolation='nearest', extent=extents,
        cmap=image_cmap, vmin=vmin, vmax=vmax, origin='upper',
        transform=projection)
    im.set_rasterized(True)
    # close the dataset
    ds = None

# PURPOSE: add a plot scale
def add_plot_scale(ax,X,Y,dx,dy,masked,fc1='w',fc2='k'):
    if masked:
        x1,x2,y1,y2 = [X-0.1*dx,X+1.2*dx,Y-2.5*dy,Y+3.2*dy]
        ax.fill([x1,x2,x2,x1,x1], [y1,y1,y2,y2,y1], fc1, zorder=4)
    for i,c in enumerate([fc1,fc2,fc1,fc2]):
        x1,x2,y1,y2 = [X+0.25*i*dx,X+0.25*(i+1)*dx,Y,Y+dy]
        ax.fill([x1,x2,x2,x1,x1], [y1,y1,y2,y2,y1], c, zorder=5)
    ax.plot([X,X+dx,X+dx,X,X], [Y,Y,Y+dy,Y+dy,Y], fc2, zorder=6)
    for i in range(3):
        ax.plot([X+0.5*i*dx,X+0.5*i*dx], [Y,Y-0.5*dy], fc2, zorder=6)
        ax.text(X+0.5*i*dx, Y-0.9*dy, '{0:0.0f}'.format(0.5*i*dx/1e3),
            ha='center', va='top', fontsize=10, color=fc2, zorder=6)
    ax.text(X+0.5*dx, Y+1.3*dy, 'km', ha='center', va='bottom',
        fontsize=10, color=fc2, zorder=6)

# plot grid program
def plot_grid(base_dir, FILENAMES,
    DATAFORM=None,
    VARIABLES=[],
    MASK=None,
    INTERPOLATION=None,
    DDEG=None,
    INTERVAL=None,
    SCALE_FACTOR=1.0,
    REMOVE_FILE=None,
    COLOR_MAP=None,
    CPT_FILE=None,
    PLOT_RANGE=None,
    BOUNDARY=None,
    ALPHA=1.0,
    CONTOURS=False,
    CONTOUR_RANGE=None,
    MEAN_CONTOUR=False,
    TITLES=None,
    LABELS=None,
    CBEXTEND=None,
    CBTITLE=None,
    CBUNITS=None,
    CBFORMAT=None,
    BASEMAP=False,
    BASIN_TYPE=None,
    DRAW_GRID_LINES=False,
    GRID=None,
    DRAW_SCALE=False,
    FIGURE_FILE=None,
    FIGURE_FORMAT=None,
    FIGURE_DPI=None,
    MODE=0o775):

    # extend list if a single format was entered for all files
    if len(DATAFORM) < len(FILENAMES):
        DATAFORM = DATAFORM*len(FILENAMES)

    # read CPT or use color map
    if CPT_FILE is not None:
        # cpt file
        cmap = gravtk.tools.from_cpt(CPT_FILE)
    else:
        # colormap
        cmap = copy.copy(cm.get_cmap(COLOR_MAP))

    # if using MODIS mosaic of Antarctica as basemap
    if BASEMAP:
        # transparent color map for bad values
        cmap.set_bad(alpha=0.0)
    else:
        # grey color map for bad values
        cmap.set_bad('w',0.5)

    # set transparency ALPHA
    if BOUNDARY is None:
        # contours
        levels = np.arange(PLOT_RANGE[0], PLOT_RANGE[1]+PLOT_RANGE[2],
            PLOT_RANGE[2])
        norm = colors.Normalize(vmin=PLOT_RANGE[0], vmax=PLOT_RANGE[1])
    else:
        # boundary between contours
        levels = np.array(BOUNDARY, dtype=np.float64)
        norm = colors.BoundaryNorm(BOUNDARY, ncolors=256)

    # convert degree spacing and interval parameters
    # Grid spacing
    dlon,dlat = (DDEG[0],DDEG[0]) if (len(DDEG) == 1) else (DDEG[0],DDEG[1])
    # Grid dimensions
    if (INTERVAL == 1):# (0:360, 90:-90)
        nlon = np.int64((360.0/dlon)+1.0)
        nlat = np.int64((180.0/dlat)+1.0)
    elif (INTERVAL == 2):# degree spacing/2
        nlon = np.int64((360.0/dlon))
        nlat = np.int64((180.0/dlat))

    # interpolation method for image background using transform_scalar
    if (INTERPOLATION == 'nearest'):
        order = 0
    elif (INTERPOLATION == 'bilinear'):
        order = 1
    elif (INTERPOLATION == 'cubic'):
        order = 3

    # create masked array if missing values
    if MASK is not None:
        # Read Land-Sea Mask of specified input file
        # 0=Ocean, 1=Land, 2=Lake, 3=Small Island, 4=Ice Shelf
        # Open the land-sea NetCDF file for reading
        landsea = gravtk.spatial().from_netCDF4(MASK,
            date=False, varname='LSMASK')
        # create land function
        nth,nphi = landsea.shape
        mask = np.zeros((nth,nphi),dtype=bool)
        # combine land and island levels for land function
        indx,indy = np.nonzero((landsea.data >= 1) & (landsea.data <= 3))
        mask[indx,indy] = True

    # remove a spatial field from each input map
    if REMOVE_FILE is not None:
        REMOVE = gravtk.spatial().from_netCDF4(REMOVE_FILE,
            date=False).data[:,:]
    else:
        REMOVE = 0.0

    # image extents
    ax = {}
    # setup polar stereographic maps
    fig, ((ax[0],ax[1]),(ax[2],ax[3])) = plt.subplots(num=1, nrows=2, ncols=2,
        figsize=(6,6.3), subplot_kw=dict(projection=projection))
    # WGS84 Ellipsoid parameters
    a_axis = 6378137.0# [m] semimajor axis of the ellipsoid
    flat = 1.0/298.257223563# flattening of the ellipsoid
    # (4pi/3)R^3 = (4pi/3)(a^2)b = (4pi/3)(a^3)(1 -f)
    rad_e = a_axis*(1.0 - flat)**(1.0/3.0)

    for i,ax1 in ax.items():
       # plot image of MODIS mosaic of Antarctica as base layer
        if BASEMAP:
            # plot MODIS mosaic of Antarctica
            plot_image_mosaic(ax1, base_dir)

        # input ascii/netCDF4/HDF5 file
        if (DATAFORM[i] == 'ascii'):
            # ascii (.txt)
            dinput = gravtk.spatial().from_ascii(FILENAMES[i], date=False,
                columns=VARIABLES, spacing=[dlon,dlat], nlat=nlat, nlon=nlon)
        elif (DATAFORM[i] == 'netCDF4'):
            # netCDF4 (.nc)
            field_mapping = gravtk.spatial().default_field_mapping(VARIABLES)
            dinput = gravtk.spatial().from_netCDF4(FILENAMES[i], date=False,
                field_mapping=field_mapping)
        elif (DATAFORM[i] == 'HDF5'):
            # HDF5 (.H5)
            field_mapping = gravtk.spatial().default_field_mapping(VARIABLES)
            dinput = gravtk.spatial().from_HDF5(FILENAMES[i], date=False,
                field_mapping=field_mapping)

        # remove offset and scale to units
        if (REMOVE != 0.0) or (SCALE_FACTOR != 1.0):
            dinput = dinput.offset(-REMOVE).scale(SCALE_FACTOR)

        # update masked values
        if MASK is not None:
            dinput.replace_invalid(fill_value=dinput.fill_value, mask=mask)

        # if dlat is negative
        if (np.sign(dlat) == -1):
            dinput = dinput.flip(axis=0)

        # calculate image coordinates
        mx = np.int64((xlimits[1]-xlimits[0])/1000.)+1
        my = np.int64((ylimits[1]-ylimits[0])/1000.)+1
        X = np.linspace(xlimits[0],xlimits[1],mx)
        Y = np.linspace(ylimits[0],ylimits[1],my)
        gridx,gridy = np.meshgrid(X,Y)
        # create mesh lon/lat
        points = ccrs.PlateCarree().transform_points(projection,
            gridx.flatten(), gridy.flatten())
        lonsin = points[:,0].reshape(my,mx)
        latsin = points[:,1].reshape(my,mx)

        # interpolate to image coordinates
        if (INTERVAL == 1) and (np.max(dinput.lon) > 180):# (0:360, 90:-90)
            shift_data,lon180 = gravtk.tools.shift_grid(180.0,
                dinput.data,dinput.lon)
            shift_mask,lon180 = gravtk.tools.shift_grid(180.0,
                dinput.mask,dinput.lon)
            img = gravtk.tools.interp_grid(shift_data,lon180,
                dinput.lat,lonsin,latsin,order)
            msk = gravtk.tools.interp_grid(shift_mask,lon180,
                dinput.lat,lonsin,latsin,order)
        elif (INTERVAL == 2) and (np.max(dinput.lon) > 180):# DDEG/2
            shift_data,lon180 = gravtk.tools.shift_grid(180.0+dlon,
                dinput.data,dinput.lon)
            shift_mask,lon180 = gravtk.tools.shift_grid(180.0+dlon,
                dinput.mask,dinput.lon)
            img = gravtk.tools.interp_grid(shift_data,
                lon180,dinput.lat,lonsin,latsin,order)
            msk = gravtk.tools.interp_grid(shift_mask,
                lon180,dinput.lat,lonsin,latsin,order)
        else:# -180:180 or modification of there of
            img = gravtk.tools.interp_grid(dinput.data,
                dinput.lon,dinput.lat,lonsin,latsin,order)
            msk = gravtk.tools.interp_grid(dinput.mask,
                dinput.lon,dinput.lat,lonsin,latsin,order)
        # create masked array of image
        img = np.ma.array(img, mask=msk.astype(bool))

        # plot only grounded points
        if MASK is not None:
            img = gravtk.tools.mask_oceans(lonsin,latsin,
                data=img,order=order,iceshelves=False)
        # plot image with transparency using normalization
        im = ax1.imshow(img, interpolation='nearest', cmap=cmap,
            extent=(xlimits[0],xlimits[1],ylimits[0],ylimits[1]),
            norm=norm, alpha=ALPHA, origin='lower', transform=projection)

        # create mesh lon/lat
        lon, lat = np.meshgrid(dinput.lon,dinput.lat)
        data = dinput.to_masked_array()
        # plot line contours
        if CONTOURS:
            clevs = np.arange(CONTOUR_RANGE[0],
                CONTOUR_RANGE[1] + CONTOUR_RANGE[2],
                CONTOUR_RANGE[2])
            # remove 0 (will plot in red)
            reduce_clevs = clevs[np.nonzero(clevs)]
            # plot contours
            ax1.contour(lon,lat,data,reduce_clevs,colors='0.2',
                linestyles='solid',transform=ccrs.PlateCarree())
            ax1.contour(lon,lat,data,[0],colors='red',linestyles='solid',
                linewidths=1.5,transform=ccrs.PlateCarree())

        # plot line contour for global average
        if MEAN_CONTOUR and CONTOURS:
            # calculate areas of each grid cell
            dphi,dth = (dlon*np.pi/180.0,dlat*np.pi/180.0)
            indy,indx = np.nonzero(np.logical_not(data.mask))
            area = (rad_e**2)*dth*dphi*np.cos(lat[indy,indx]*np.pi/180.0)
            # calculate average
            ave=np.sum(area*data[indy,indx])/np.sum(area)
            # plot line contour of global average
            ax1.contour(lon,lat,data,[ave],colors='blue',linestyles='solid',
                linewidths=1.5,transform=ccrs.PlateCarree())

        # add basins based on BASIN_TYPE (Rignot 2012, IMBIE-2, IMBIE-2 subbasins)
        if (BASIN_TYPE == 'Rignot'):
            plot_rignot_basins(ax1, base_dir)
            start_indice = 1
        elif (BASIN_TYPE == 'IMBIE-2'):
            plot_IMBIE2_basins(ax1, base_dir)
            start_indice = 1
        elif (BASIN_TYPE == 'IMBIE-2_subbasin'):
            plot_IMBIE2_subbasins(ax1, base_dir)
            start_indice = 1
        else:
            start_indice = 0
        # add glaciers/ice caps/islands
        plot_grounded_ice(ax1, base_dir, START=start_indice)

        # draw lat/lon grid lines
        if DRAW_GRID_LINES:
            # meridian and parallel grid spacing
            llx,lly = (GRID[0],GRID[0]) if (len(GRID) == 1) else (GRID[0],GRID[1])
            grid_meridians = np.arange(-180, 180 + llx, llx)
            grid_parallels = np.arange(-90, 90 + lly, lly)
            gl = ax1.gridlines(crs=ccrs.PlateCarree(), draw_labels=False,
                linewidth=0.1, color='0.25', linestyle='-')
            gl.xlocator = ticker.FixedLocator(grid_meridians)
            gl.ylocator = ticker.FixedLocator(grid_parallels)

        # add title for each subplot
        if TITLES is not None:
            TITLE = ' '.join(TITLES[i].split('_'))
            ax1.set_title(TITLE.replace('-',u'\u2013'), fontsize=14)
            ax1.title.set_y(0.995)
        # Add figure label for each subplot
        if LABELS is not None:
            at = offsetbox.AnchoredText(LABELS[i],
                loc=2, pad=0, borderpad=0.25, frameon=True,
                prop=dict(size=18,weight='bold',color='k'))
            at.patch.set_boxstyle("Square,pad=0.1")
            at.patch.set_edgecolor("white")
            ax1.axes.add_artist(at)

        # x and y limits, axis = equal
        ax1.set_xlim(xlimits)
        ax1.set_ylim(ylimits)
        ax1.set_aspect('equal', adjustable='box')
        # no ticks on the x and y axes
        ax1.get_xaxis().set_ticks([])
        ax1.get_yaxis().set_ticks([])
        # stronger linewidth on frame
        ax1.spines['geo'].set_linewidth(2.0)
        ax1.spines['geo'].set_zorder(10)
        ax1.spines['geo'].set_capstyle('projecting')

    # draw map scale to corners of axis
    if DRAW_SCALE:
        add_plot_scale(ax[2],-292e4,-215e4,1600e3,140e3,False)

    # Add colorbar
    # Add an axes at position rect [left, bottom, width, height]
    cbar_ax = fig.add_axes([0.085, 0.095, 0.83, 0.035])
    # extend = add extension triangles to upper and lower bounds
    # options: neither, both, min, max
    cbar = fig.colorbar(im, cax=cbar_ax, extend=CBEXTEND,
        extendfrac=0.0375, drawedges=False, orientation='horizontal')
    # rasterized colorbar to remove lines
    cbar.solids.set_rasterized(True)
    # Add label to the colorbar
    cbar.ax.set_title(CBTITLE, fontsize=14, rotation=0, y=-1.65, va='top')
    cbar.ax.set_xlabel(CBUNITS, fontsize=14, rotation=0, va='center')
    cbar.ax.xaxis.set_label_coords(1.08, 0.5)
    # Set the tick levels for the colorbar
    cbar.set_ticks(levels)
    cbar.set_ticklabels([CBFORMAT.format(ct) for ct in levels])
    # ticks lines all the way across
    cbar.ax.tick_params(which='both', width=1, length=16, labelsize=14,
        direction='in')

    # adjust subplots within figure
    fig.subplots_adjust(left=0.01, right=0.99, bottom=0.14, top=0.97,
        hspace=0.05, wspace=0.05)
    # create output directory if non-existent
    FIGURE_FILE.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
    # save to file
    logging.info(str(FIGURE_FILE))
    plt.savefig(FIGURE_FILE,
        metadata={'Title':pathlib.Path(sys.argv[0]).name},
        dpi=FIGURE_DPI, format=FIGURE_FORMAT)
    plt.clf()
    # change the permissions mode
    FIGURE_FILE.chmod(mode=MODE)

# PURPOSE: create argument parser
def arguments():
    parser = argparse.ArgumentParser(
        description="""Creates 4 GMT-like plots of the Antarctic ice sheet
            on a polar stereographic south (EPSG 3031) projection
            """,
        fromfile_prefix_chars="@"
    )
    parser.convert_arg_line_to_args = gravtk.utilities.convert_arg_line_to_args
    # command line parameters
    parser.add_argument('infile', nargs=4,
        type=pathlib.Path,
        help='Input grid files')
    # working data directory
    parser.add_argument('--directory','-D',
        type=pathlib.Path, default=pathlib.Path.cwd(),
        help='Working data directory')
    # Input data format (ascii, netCDF4, HDF5)
    parser.add_argument('--format','-F',
        type=str, nargs='+',
        default='netCDF4', choices=['ascii','netCDF4','HDF5'],
        help='Input data format')
    # variable names (for ascii names of columns)
    parser.add_argument('--variables','-v',
        type=str, nargs='+', default=['lon','lat','z'],
        help='Variable names of data in input file')
    # land-sea mask
    lsmask = gravtk.utilities.get_data_path(['data','landsea_hd.nc'])
    parser.add_argument('--mask',
        type=pathlib.Path, default=lsmask,
        help='Land-sea mask')
    # output grid parameters
    parser.add_argument('--spacing',
        type=float, nargs='+', default=[0.5,0.5], metavar=('dlon','dlat'),
        help='Spatial resolution of input data')
    parser.add_argument('--interval',
        type=int, default=2, choices=[1,2],
        help=('Input grid interval (1: global, 2: centered global)'))
    # Interpolation method
    parser.add_argument('--interpolation','-I',
        type=str, default='bilinear', choices=['nearest','bilinear','cubic'],
        help='Interpolation method')
    # scale factor
    parser.add_argument('--scale-factor','-s',
        type=float, default=1.0,
        help='Multiplicative scale factor for converting to plot units')
    # plot range
    parser.add_argument('--plot-range','-R',
        type=float, nargs=3, metavar=('MIN','MAX','STEP'),
        help='Plot range and step size for normalization')
    parser.add_argument('--boundary','-B',
        type=float, nargs='+',
        help='Plot boundary for normalization')
    # color palette table or named color map
    try:
        cmap_set = set(cm.datad.keys()) | set(cm.cmaps_listed.keys())
    except (ValueError, NameError) as exc:
        cmap_set = []
    parser.add_argument('--colormap','-m',
        metavar='COLORMAP', type=str, default='viridis',
        choices=sorted(cmap_set),
        help='Named Matplotlib colormap')
    parser.add_argument('--cpt-file','-c',
        type=pathlib.Path,
        help='Input Color Palette Table (.cpt) file')
    # color map alpha
    parser.add_argument('--alpha','-a',
        type=float, default=1.0,
        help='Named Matplotlib colormap')
    # plot contour parameters
    parser.add_argument('--plot-contours',
        default=False, action='store_true',
        help='Plot contours')
    parser.add_argument('--contour-range',
        type=float, nargs=3, metavar=('MIN','MAX','STEP'),
        help='Contour range and step size')
    parser.add_argument('--mean-contour',
        default=False, action='store_true',
        help='Plot contours for mean of dataset')
    # title and label
    parser.add_argument('--plot-title', nargs=4,
        type=str, help='Plot title')
    parser.add_argument('--plot-label', nargs=4,
        type=str, help='Plot label')
    # colorbar parameters
    parser.add_argument('--cbextend',
        type=str, default='both',
        choices=['neither', 'both', 'min', 'max'],
        help='Add extension triangles to colorbar')
    parser.add_argument('--cbtitle',
        type=str, default='',
        help='Title label for colorbar')
    parser.add_argument('--cbunits',
        type=str, default='',
        help='Units label for colorbar')
    parser.add_argument('--cbformat',
        type=str, default='{0:3.0f}',
        help='Tick format for colorbar')
    # additional parameters
    parser.add_argument('--basemap',
        default=False, action='store_true',
        help='Add background basemap image')
    parser.add_argument('--basin-type',
        type=str, default='',
        help='Add delineations for glacier drainage basins')
    parser.add_argument('--draw-grid-lines',
        default=False, action='store_true',
        help='Add map grid lines')
    parser.add_argument('--grid-lines',
        type=float, nargs='+', default=(15,15),
        help='Input grid spacing for meridians and parallels')
    parser.add_argument('--draw-scale',
        default=False, action='store_true',
        help='Add map scale bar')
    # output file, format and dpi
    parser.add_argument('--figure-file','-O',
        type=pathlib.Path,
        help='Output figure file')
    parser.add_argument('--figure-format','-f',
        type=str, default='png', choices=('pdf','png','jpg','svg'),
        help='Output figure format')
    parser.add_argument('--figure-dpi','-d',
        type=int, default=180,
        help='Output figure resolution in dots per inch (dpi)')
    # print information about each input and output file
    parser.add_argument('--verbose','-V',
        action='count', default=0,
        help='Verbose output of run')
    # permissions mode of the local directories and files (number in octal)
    parser.add_argument('--mode','-M',
        type=lambda x: int(x,base=8), default=0o775,
        help='Permissions mode of output files')
    # return the parser
    return parser

# This is the main part of the program that calls the individual functions
def main():
    # Read the system arguments listed after the program
    parser = arguments()
    args,_ = parser.parse_known_args()

    # create logger
    loglevels = [logging.CRITICAL, logging.INFO, logging.DEBUG]
    logging.basicConfig(level=loglevels[args.verbose])

    # try to run the analysis with listed parameters
    try:
        info(args)
        # run plot program with parameters
        plot_grid(args.directory, args.infile,
            DATAFORM=args.format,
            VARIABLES=args.variables,
            DDEG=args.spacing,
            INTERVAL=args.interval,
            INTERPOLATION=args.interpolation,
            SCALE_FACTOR=args.scale_factor,
            CPT_FILE=args.cpt_file,
            COLOR_MAP=args.colormap,
            PLOT_RANGE=args.plot_range,
            BOUNDARY=args.boundary,
            ALPHA=args.alpha,
            CONTOURS=args.plot_contours,
            CONTOUR_RANGE=args.contour_range,
            MEAN_CONTOUR=args.mean_contour,
            TITLES=args.plot_title,
            LABELS=args.plot_label,
            CBEXTEND=args.cbextend,
            CBTITLE=args.cbtitle,
            CBUNITS=args.cbunits,
            CBFORMAT=args.cbformat,
            BASEMAP=args.basemap,
            BASIN_TYPE=args.basin_type,
            DRAW_GRID_LINES=args.draw_grid_lines,
            GRID=args.grid_lines,
            DRAW_SCALE=args.draw_scale,
            FIGURE_FILE=args.figure_file,
            FIGURE_FORMAT=args.figure_format,
            FIGURE_DPI=args.figure_dpi,
            MODE=args.mode)
    except Exception as exc:
        # if there has been an error exception
        # print the type, value, and stack trace of the
        # current exception being handled
        logging.critical(f'process id {os.getpid():d} failed')
        logging.error(traceback.format_exc())

# run main program
if __name__ == '__main__':
    main()