import os

from google.protobuf import message
from google.protobuf import text_format
from tensorflow.core.protobuf import saved_model_pb2
from tensorflow.python.lib.io import file_io
from tensorflow.python.saved_model import constants
from tensorflow.python.util import compat

def read_saved_model(saved_model_dir):
  """Reads the saved_model.pb or saved_model.pbtxt file containing `SavedModel`.

  Args:
    saved_model_dir: Directory containing the SavedModel file.

  Returns:
    A `SavedModel` protocol buffer.

  Raises:
    IOError: If the file does not exist, or cannot be successfully parsed.
  """
  # Build the path to the SavedModel in pbtxt format.
  path_to_pbtxt = os.path.join(
      compat.as_bytes(saved_model_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PBTXT))
  # Build the path to the SavedModel in pb format.
  path_to_pb = os.path.join(
      compat.as_bytes(saved_model_dir),
      compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))

  # Ensure that the SavedModel exists at either path.
  if not file_io.file_exists(path_to_pbtxt) and not file_io.file_exists(
      path_to_pb):
    raise IOError("SavedModel file does not exist at: %s" % saved_model_dir)

  # Parse the SavedModel protocol buffer.
  saved_model = saved_model_pb2.SavedModel()
  if file_io.file_exists(path_to_pb):
    with file_io.FileIO(path_to_pb, "rb") as f:
      file_content = f.read()
    try:
      saved_model.ParseFromString(file_content)
      return saved_model
    except message.DecodeError as e:
      raise IOError("Cannot parse proto file %s: %s." % (path_to_pb, str(e)))
  elif file_io.file_exists(path_to_pbtxt):
    with file_io.FileIO(path_to_pbtxt, "rb") as f:
      file_content = f.read()
    try:
      text_format.Merge(file_content.decode("utf-8"), saved_model)
      return saved_model
    except text_format.ParseError as e:
      raise IOError("Cannot parse pbtxt file %s: %s." % (path_to_pbtxt, str(e)))
  else:
    raise IOError("SavedModel file does not exist at: %s/{%s|%s}" %
                  (saved_model_dir, constants.SAVED_MODEL_FILENAME_PBTXT,
                   constants.SAVED_MODEL_FILENAME_PB))

def get_saved_model_tag_sets(saved_model_dir):
  """Retrieves all the tag-sets available in the SavedModel.

  Args:
    saved_model_dir: Directory containing the SavedModel.

  Returns:
    List of all tag-sets in the SavedModel, where a tag-set is represented as a
    list of strings.
  """
  saved_model = read_saved_model(saved_model_dir)
  all_tags = []
  for meta_graph_def in saved_model.meta_graphs:
    all_tags.append(list(meta_graph_def.meta_info_def.tags))
  return all_tags

def get_meta_graph_def(saved_model_dir, tag_set):
  """Gets MetaGraphDef from SavedModel.

  Returns the MetaGraphDef for the given tag-set and SavedModel directory.

  Args:
    saved_model_dir: Directory containing the SavedModel to inspect.
    tag_set: Group of tag(s) of the MetaGraphDef to load, in string format,
        separated by ','. The empty string tag is ignored so that passing ''
        means the empty tag set. For tag-set contains multiple tags, all tags
        must be passed in.

  Raises:
    RuntimeError: An error when the given tag-set does not exist in the
        SavedModel.

  Returns:
    A MetaGraphDef corresponding to the tag-set.
  """
  saved_model = read_saved_model(saved_model_dir)
  # Note: Discard empty tags so that "" can mean the empty tag set.
  set_of_tags = set([tag for tag in tag_set.split(",") if tag])

  valid_tags = []
  for meta_graph_def in saved_model.meta_graphs:
    meta_graph_tags = set(meta_graph_def.meta_info_def.tags)
    if meta_graph_tags == set_of_tags:
      return meta_graph_def
    else:
      valid_tags.append(",".join(meta_graph_tags))

  raise RuntimeError(
      f"MetaGraphDef associated with tag-set {tag_set} could not be found in "
      f"the SavedModel. Please use one of the following tag-sets: {valid_tags}")


from .models import Book, Author, Seller, Genre, Format
from django.contrib.auth.models import User
from .fetch import isbn, author

def db_loader():
   data = author()
   for i in range(len(data)):
      try:
         d = data[i]
         u1 = User.objects.all()[0]
         s1 = Seller.objects.all()[0]
         try:
            a = Author.objects.get(name__contains=d['author'])
            print(d['author'],'-',d['title'])
         except:
            print(f"Invalid data!!")
            continue

         book = Book.objects.create(user=u1, author=a, seller=s1, title=d['title'], ISBN=d['ISBN'], image=d['image'], price=d['price'],
                                    discount=d['discount'], language=d['language'], publisher=d['publisher'],
                                    publication_date=d['publication_date'], pages=d['pages'], origin=d['origin'],
                                    age_group=d['age_group'], dimensions=d['dimensions'], description=d['description'],
                                    rating=d['rating'], num_reviews=d['num_reviews'], count_in_stock=d['count_in_stock'])

         for genre in d['genres']:
            try:
               g = Genre.objects.get(genre__contains=genre)
            except:
               g = Genre.objects.create(genre=genre)
            book.genres.add(g)

         for format, link in d['formats'].items():
            Format.objects.create(format=format, link=link, book=book)
      except:
         print(f"Can't load in DB!! Invalid data")


# -*- encoding: utf-8 -*-
'''
@project :   LeetCode
@File    :   minimumLengthEncoding.py
@Contact :   9824373@qq.com
@Desc    :
                给定一个单词列表，我们将这个列表编码成一个索引字符串 S 与一个索引列表 A。

                例如，如果这个列表是 ["time", "me", "bell"]，我们就可以将其表示为 S = "time#bell#" 和 indexes = [0, 2, 5]。

                对于每一个索引，我们可以通过从字符串 S 中索引的位置开始读取字符串，直到 "#" 结束，来恢复我们之前的单词列表。

                那么成功对给定单词列表进行编码的最小字符串长度是多少呢？

                示例：

                输入: words = ["time", "me", "bell"]
                输出: 10
                说明: S = "time#bell#" ， indexes = [0, 2, 5] 。

                提示：

                1 <= words.length <= 2000
                1 <= words[i].length <= 7
                每个单词都是小写字母 。

                来源：力扣（LeetCode）
                链接：https://leetcode-cn.com/problems/short-encoding-of-words

@Modify Time                @Author     @Version    @Desciption
------------                -------     --------    -----------
2020-03-28     zhan        1.0         None
'''
from typing import List
import collections
from functools import reduce

class Solution:
    def minimumLengthEncoding(self, words: List[str]) -> int:
        # return self.minimumLengthEncoding_sort(words)
        return self.minimumLengthEncoding_trie(words)

    def minimumLengthEncoding_sort(self, words: List[str]) -> int:
        n = len(words)
        words.sort(key=lambda word:word[::-1])

        S = ''
        i = 0
        while i < len(words):
            while i < len(words)-1 and len(words[i]) <= len(words[i+1]) and words[i] == words[i+1][-len(words[i]):]:
                i +=1
            S += words[i]
            S += '#'
            i += 1

        return len(S)

    def minimumLengthEncoding_trie(self, words: List[str]) -> int:
        words = list(set(words))  # remove duplicates
        # Trie is a nested dictionary with nodes created
        # when fetched entries are missing
        Trie = lambda: collections.defaultdict(Trie)
        trie = Trie()

        # reduce(..., S, trie) is trie[S[0]][S[1]][S[2]][...][S[S.length - 1]]
        # nodes = []
        # for word in words:
        #     v = reduce(dict.__getitem__, word[::-1], trie)
        #     nodes.append(v)
        # 构建 trie ，同时对于前缀，返回非控制，对于取到叶节点，返回空值
        nodes = [reduce(dict.__getitem__, word[::-1], trie)
                 for word in words]

        # Add word to the answer if it's node has no neighbors
        return sum(len(word) + 1
                   for i, word in enumerate(words)
                   if len(nodes[i]) == 0)

if __name__ == '__main__':
    words = ["me", "time", "bell"]

    ans = Solution().minimumLengthEncoding(words)
    print(ans)


import http.server
import io
import cgi
import json
import os
import posixpath
import re
import threading
import traceback
import urllib.request, urllib.parse, urllib.error
import urllib.parse

from avnav_store import AVNStore
from avnav_util import AVNUtil, AVNLog, AVNDownload
from avnav_websocket import HTTPWebSocketsHandler
from avnav_worker import AVNWorker

class WebSocketHandler(object):
  def __init__(self,handler):
    """

    @type handler: AVNHTTPHandler
    """
    self.handler=handler
    self.connected=False
  def send_message(self,message):
    if self.connected:
      self.handler.send_message(message)
  def on_ws_message(self, message):
    """Override this handler to process incoming websocket messages."""
    pass
  def on_ws_connected(self):
    self.connected=True
  def on_ws_closed(self):
    self.connected=False

class Encoder(json.JSONEncoder):
  '''
  allow our objects to have a "serialize" method
  to make them json encodable in a generic manner
  '''
  def default(self, o):
    if hasattr(o,'serialize'):
      return o.serialize()
    return super(Encoder, self).default(o)

class AVNHTTPHandler(HTTPWebSocketsHandler):
  wsHandler: WebSocketHandler
  protocol_version = "HTTP/1.1" #necessary for websockets!
  def __init__(self,request,client_address,server):
    #allow write buffering
    #see https://lautaportti.wordpress.com/2011/04/01/basehttprequesthandler-wastes-tcp-packets/
    self.wbufsize=-1
    self.id=None
    self.getRequestParam=AVNUtil.getHttpRequestParam
    threading.current_thread().setName("HTTPHandler")
    AVNLog.ld("receiver thread started",client_address)
    http.server.SimpleHTTPRequestHandler.__init__(self, request, client_address, server)
    self.wsHandler=None

  def log_message(self, format, *args):
    AVNLog.debug(format,*args)

  def allow_ws(self):
    (path,query) = self.server.pathQueryFromUrl(self.path)
    try:
      #handlers will either return
      #True if already done
      #None if no mapping found (404)
      #a path to be sent
      self.wsHandler=self.server.getWebSocketsHandler(path,query,handler=self)
      return True
    except Exception as e:
      return False

  def on_ws_message(self, message):
    if self.wsHandler is None:
      raise Exception("no websocket handler")
    self.wsHandler.on_ws_message(message)

  def on_ws_connected(self):
    if self.wsHandler is None:
      raise Exception("no websocket handler")
    self.wsHandler.on_ws_connected()

  def on_ws_closed(self):
    if self.wsHandler is None:
      raise Exception("no websocket handler")
    self.wsHandler.on_ws_closed()

  def do_POST(self):
    maxlen=5000000
    (path,sep,query) = self.path.partition('?')
    if not path.startswith(self.server.navurl):
      self.send_error(404, "unsupported post url")
      return
    try:
      ctype, pdict = cgi.parse_header(self.headers.get('content-type'))
      if ctype == 'multipart/form-data':
        postvars = cgi.parse_multipart(self.rfile, pdict)
      elif ctype == 'application/x-www-form-urlencoded':
        length = int(self.headers.get('content-length'))
        if length > maxlen:
          raise Exception("too much data"+str(length))
        postvars = cgi.parse_qs(self.rfile.read(length).decode('utf-8'), keep_blank_values=1)
      elif ctype == 'application/json':
        length = int(self.headers.get('content-length'))
        if length > maxlen:
          raise Exception("too much data"+str(length))
        postvars = { '_json':self.rfile.read(length).decode('utf-8')}
      else:
        postvars = {}
      requestParam=urllib.parse.parse_qs(query,True)
      requestParam.update(postvars)
      self.handleNavRequest(path,requestParam)
    except Exception as e:
      txt=traceback.format_exc()
      AVNLog.ld("unable to process request for ",path,query,txt)
      self.send_response(500,txt)
      self.end_headers()
      return

  def getMimeType(self,path):
    base, ext = posixpath.splitext(path)
    if ext in self.server.overwrite_map:
      ctype = self.server.overwrite_map[ext]
    else:
      ctype = self.guess_type(path)
    return ctype

  #overwrite this from SimpleHTTPRequestHandler
  def send_head(self):
    path=self.translate_path(self.path)
    if path is None:
      return
    """Common code for GET and HEAD commands.

    This sends the response code and MIME headers.

    Return value is either a file object (which has to be copied
    to the outputfile by the caller unless the command was HEAD,
    and must be closed by the caller under all circumstances), or
    None, in which case the caller has nothing further to do.

    """

    f = None
    if os.path.isdir(path):
        if not self.path.endswith('/'):
            # redirect browser - doing basically what apache does
            self.send_response(301)
            self.send_header("Location", self.path + "/")
            self.end_headers()
            self.close_connection=True
            return None
        for index in "index.html", "index.htm":
            index = os.path.join(path, index)
            if os.path.exists(index):
                path = index
                break
        else:
            return self.list_directory(path)

    ctype = self.getMimeType(path)
    try:
        # Always read in binary mode. Opening files in text mode may cause
        # newline translations, making the actual size of the content
        # transmitted *less* than the content-length!
        f = open(path, 'rb')
    except IOError:
        self.send_error(404, "File not found")
        return None
    self.send_response(200)
    self.send_header("Content-type", ctype)
    fs = os.fstat(f.fileno())
    self.send_header("Content-Length", str(fs[6]))
    if path.endswith(".js") or path.endswith(".css"):
      self.send_header("cache-control","private, max-age=0, no-cache")
    self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
    self.end_headers()
    return f

  def getPageRoot(self):
    path = self.server.getStringParam('index')
    return re.sub("/[^/]*$", "", path)

  #overwrite this from SimpleHTTPRequestHandler
  def translate_path(self, path):
      """Translate a /-separated PATH to the local filename syntax.

      Components that mean special things to the local file system
      (e.g. drive or directory names) are ignored.  (XXX They should
      probably be diagnosed.)

      """
      # abandon query parameters
      (path,query) = self.server.pathQueryFromUrl(path)
      try:
        #handlers will either return
        #True if already done
        #None if no mapping found (404)
        #a path to be sent
        extPath=self.server.tryExternalMappings(path,query,handler=self)
      except Exception as e:
        self.send_error(404,str(e))
        return None
      if isinstance(extPath, AVNDownload):
        self.writeFromDownload(extPath)
        return None
      if extPath == True:
        return None
      if extPath is not None:
        return extPath
      if path.startswith(self.server.navurl):
        requestParam=urllib.parse.parse_qs(query,True)
        self.handleNavRequest(path,requestParam)
        return None
      if path=="" or path=="/":
        path=self.server.getStringParam('index')
        self.send_response(301)
        self.send_header("Location", path)
        self.end_headers()
        self.close_connection=True
        return None

      return self.server.plainUrlToPath(path)

  #send a json encoded response
  def sendNavResponse(self,rtj,requestParam=None):
    if not rtj is None:
      self.send_response(200)
      if requestParam is not None and not requestParam.get('callback') is None:
        rtj="%s(%s);"%(requestParam.get('callback'),rtj)
        self.send_header("Content-type", "text/javascript")
      else:
        self.send_header("Content-type", "application/json")
      wbytes=rtj.encode('utf-8')
      self.send_header("Content-Length", str(len(wbytes)))
      self.send_header("Last-Modified", self.date_time_string())
      self.send_header("Cache-Control", "no-store")
      self.end_headers()
      self.wfile.write(wbytes)
      AVNLog.ld("nav response",rtj)
    else:
      raise Exception("empty response")

  def sendJsFile(self,filename,baseUrl,addCode=None):
    '''
    send a js file that we encapsulate into an anonymus function
    @param filename:
    @return:
    '''
    PREFIX=("try{(\nfunction(){\nvar AVNAV_BASE_URL=\"%s\";\n"%urllib.parse.quote(baseUrl)).encode('utf-8')
    SUFFIX="\n})();\n}catch(e){\nwindow.avnav.api.showToast(e.message+\"\\n\"+(e.stack||e));\n }\n".encode('utf-8')
    if not os.path.exists(filename):
      self.send_error(404,"File not found")
      return
    self.send_response(200)
    flen=os.path.getsize(filename)
    dlen=flen+len(PREFIX)+len(SUFFIX)
    if addCode is not None:
      addCode=addCode.encode('utf-8')
      dlen+=len(addCode)
    self.send_header("Content-type", "text/javascript")
    self.send_header("Content-Length", str(dlen))
    self.send_header("Last-Modified", self.date_time_string())
    self.end_headers()
    self.wfile.write(PREFIX)
    if addCode is not None:
      self.wfile.write(addCode)
    fh=open(filename,"rb")
    self.writeStream(flen,fh)
    self.wfile.write(SUFFIX)
    return True
  #handle a navigational query
  #request parameters:
  #request=gps&filter=TPV&bbox=54.531,13.014,54.799,13.255
  #request: gps,status,...
  #filter is a key of the map in the form prefix-suffix

  def handleNavRequest(self,path,requestParam):
    #check if we have something behind the navurl
    #treat this as a filename and set it ion the request parameter
    fname=path[(len(self.server.navurl)+1):]
    if fname is not None and fname != "":
      fname=fname.split('?',1)[0]
      if fname != "":
        if requestParam.get('filename') is None:
          requestParam['filename']=[fname.encode('utf-8')]
    requestType=requestParam.get('request')
    if requestType is None:
      requestType='gps'
    else:
      requestType=requestType[0]
    AVNLog.ld('navrequest',requestParam)
    try:
      rtj=None
      if requestType=='gps' or requestType=='self':
        rtj=self.handleGpsRequest(requestParam)
      elif requestType=='nmeaStatus':
        rtj=self.handleNmeaStatus(requestParam)
      elif requestType=='ais':
        rtj=self.handleAISRequest(requestParam)
      elif requestType=='status':
        rtj=self.handleStatusRequest(requestParam)
      elif requestType=='debuglevel' or requestType=='loglevel':
        rtj=self.handleDebugLevelRequest(requestParam)
      elif requestType=='currentloglevel':
        rtj=self.handleCurrentLevelRequest(requestParam)
      elif requestType=='listdir' or requestType == 'list':
        rtj=self.handleListDir(requestParam)
      elif requestType=='download':
        #download requests are special
        # the dow not return json...
        self.handleDownloadRequest(requestParam)
        return
      elif requestType=='upload':
        try:
          rtj=self.handleUploadRequest(requestParam)
        except Exception as e:
          AVNLog.error("upload error: %s",str(e))
          rtj=json.dumps({'status':str(e)},cls=Encoder)
      elif requestType=='delete':
        rtj=self.handleDeleteRequest(requestParam)

      elif requestType=='capabilities':
        rtj=self.handleCapabilityRequest(requestParam)
      elif requestType=='api':
        #new handling for dedicated requests for some handler
        type=self.getRequestParam(requestParam,'type')
        rtj=self.handleSpecificRequest(requestParam,type)
      else:
        #legacy: have the api type as requestType
        rtj=self.handleSpecificRequest(requestParam,requestType)
      self.sendNavResponse(rtj,requestParam)
    except Exception as e:
          text=str(e)
          rtj=json.dumps(AVNUtil.getReturnData(error=text,stack=traceback.format_exc()),cls=Encoder)
          self.sendNavResponse(rtj,requestParam)
          return

  def handleSpecificRequest(self,requestParam,rtype):
    """
    a request that is specific to a particular handler
    @param requestParam:
    @param rtype: the request type
    @return: json
    """
    if rtype is None:
      raise Exception("missing parameter type for api request")
    handler = self.server.getRequestHandler('api', rtype)
    if handler is None:
      raise Exception("no handler found for request %s", rtype)
    rtj = handler.handleApiRequest('api', rtype, requestParam,handler=self)
    if isinstance(rtj, dict) or isinstance(rtj, list):
      rtj = json.dumps(rtj,cls=Encoder)
    return rtj

  #return AIS targets
  #parameter: lat,lon,distance (in NM) - limit to this distance
  def handleAISRequest(self,requestParam):
    rt=self.server.navdata.getAisData()
    lat=None
    lon=None
    lat1=None
    lon1=None
    dist=None
    try:
      lat=float(self.getRequestParam(requestParam, 'lat'))
      lon=float(self.getRequestParam(requestParam, 'lon'))
      dist=float(self.getRequestParam(requestParam, 'distance')) #distance in NM
      rq=self.getRequestParam(requestParam,'lat1')
      if rq is not None:
        lat1=float(rq)
      rq=self.getRequestParam(requestParam,'lon1')
      if rq is not None:
        lon1=float(rq)
    except:
      pass
    frt=[]
    if not lat is None and not lon is None and not dist is None:
      dest=(lat,lon)
      AVNLog.debug("limiting AIS to lat=%f,lon=%f,dist=%f",lat,lon,dist)
      dest1=None
      if lat1 is not None and lon1 is not None:
        dest1=(lat1,lon1)
        AVNLog.debug("additional AIS range lat=%f,lon=%f,dist=%f",lat1,lon1,dist)
      for entry in rt:
        try:
          fentry=entry
          mdist=AVNUtil.distance((fentry.get('lat'),fentry.get('lon')), dest)
          inRange=False
          if mdist<=dist:
            inRange=True
          else:
            if dest1 is not None:
              mdist=AVNUtil.distance((fentry.get('lat'),fentry.get('lon')), dest1)
              if mdist<=dist:
                inRange=True
          if inRange:
            fentry['distance']=mdist*AVNUtil.NM #have this in m
            frt.append(fentry)
          else:
            AVNLog.debug("filtering out %s due to distance %f",str(fentry['mmsi']),mdist)
        except:
          AVNLog.debug("unable to convert ais data: %s",traceback.format_exc())
    else:
      for entry in rt:
        try:
          frt.append(entry)
        except Exception as e:
          AVNLog.debug("unable to convert ais data: %s",traceback.format_exc())
    return json.dumps(frt,cls=Encoder)

  def handleGpsRequest(self,requestParam):
    rtv=self.server.navdata.getDataByPrefix(AVNStore.BASE_KEY_GPS)
    return json.dumps(rtv,cls=Encoder)

  def handleNmeaStatus(self, requestParam):
    rtv = self.server.navdata.getDataByPrefix(AVNStore.BASE_KEY_GPS)
    # we depend the status on the mode: no mode - red (i.e. not connected), mode: 1- yellow, mode 2+lat+lon - green
    status = "red"
    if rtv.get("lat") is not None and rtv.get('lon') is not None:
      status = "green"
    info = self.server.navdata.getSingleValue(AVNStore.BASE_KEY_GPS + ".lat",includeInfo=True)  # we just want the last source of position
    src='unknown'
    if info is not None:
      src=info.source
    satInview = rtv.get('satInview')
    if satInview is None:
      satInview=0
    satUsed   = rtv.get('satUsed')
    if satUsed is None:
      satUsed=0
    statusNmea = {"status": status, "source": src, "info": "Sat %d visible/%d used" % (int(satInview), int(satUsed))}

    status = "red"
    numAis = self.server.navdata.getAisCounter()
    if numAis > 0:
      status = "green"
    src = self.server.navdata.getLastAisSource()
    statusAis = {"status": status, "source": src, "info": "%d targets" % (numAis)}
    rt = {"status": "OK","data":{"nmea": statusNmea, "ais": statusAis}}
    return json.dumps(rt,cls=Encoder)

  def handleStatusRequest(self,requestParam):
    rt=[]
    allHandlers=AVNWorker.getAllHandlers(True)
    #find for each type the first id
    typIds={}
    for h in allHandlers:
      type=h.getConfigName()
      if typIds.get(type) is None:
        typIds[type]=h.getId()
    #get the sorted list of typeIds
    typeList=sorted(typIds.keys(),key=lambda k: typIds[k])
    for type in typeList:
      for handler in AVNWorker.getAllHandlers(True):
        if handler.getConfigName() != type: continue
        entry={'configname':handler.getConfigName(),
            'config': handler.getParam(),
            'name':handler.getStatusName(),
            'info':handler.getInfo(),
            'disabled': handler.isDisabled(),
            'properties':handler.getStatusProperties() if not handler.isDisabled() else {},
            'canDelete':handler.canDeleteHandler(),
            'canEdit': handler.canEdit(),
            'id':handler.getId()
              }
        rt.append(entry)
    return json.dumps({'handler':rt},cls=Encoder)

  def handleDebugLevelRequest(self,requestParam):
    rt={'status':'ERROR','info':'missing parameter'}
    level=self.getRequestParam(requestParam,'level')
    timeout = self.getRequestParam(requestParam, 'timeout',mantadory=False)
    if timeout is not None:
      timeout=float(timeout)
    filter=self.getRequestParam(requestParam,'filter')
    if not level is None:
      crt=AVNLog.changeLogLevelAndFilter(level,filter,timeout)
      if crt:
        rt['status']='OK'
        rt['info']='set loglevel to '+str(level)
      else:
        rt['info']="invalid level "+str(level)
    return json.dumps(rt,cls=Encoder)

  def handleCurrentLevelRequest(self,requestParam):
    (level,filter)=AVNLog.getCurrentLevelAndFilter()
    return json.dumps({'status':'OK','level':level,'filter':filter},cls=Encoder)

  def writeStream(self,bToSend,fh):
    maxread = 1000000
    while bToSend > 0:
      buf = fh.read(maxread if bToSend > maxread else bToSend)
      if buf is None or len(buf) == 0:
        raise Exception("no more data")
      self.wfile.write(buf)
      bToSend -= len(buf)
    fh.close()

  def writeData(self,data,mimeType):
    self.send_response(200)
    self.send_header("Content-type", mimeType)
    wbytes=None
    if type(data) == bytes:
      wbytes=data
    else:
      wbytes=data.encode('utf-8')
    self.send_header("Content-Length", str(len(wbytes)))
    self.send_header("Last-Modified", self.date_time_string())
    self.end_headers()
    self.wfile.write(wbytes)

  def writeFromDownload(self,download,filename=None,noattach=False):
    # type: (AVNDownload, str,bool) -> object or None
    self.send_response(200)
    size = download.getSize()
    if filename is not None and filename != "" and not noattach:
      self.send_header("Content-Disposition", "attachment; %s"%AVNDownload.fileToAttach(filename))
    self.send_header("Content-type", download.getMimeType(self))
    self.send_header("Content-Length", size)
    self.send_header("Last-Modified", self.date_time_string())
    self.end_headers()
    stream = None
    stream = download.getStream()
    self.writeStream(size, stream)

  #download requests
  #parameters:
  #   type
  #   type specific parameters
  #        route: name
  #   filename
  def handleDownloadRequest(self,requestParam):
    type=self.getRequestParam(requestParam,"type")
    try:
      handler=self.server.getRequestHandler('download',type)
      if handler is not None:
        AVNLog.debug("found handler %s to handle download %s"%(handler.getConfigName(), type))
        dl=handler.handleApiRequest('download',type,requestParam,handler=self)
        if dl is None:
          raise Exception("unable to download %s"%type)
        if isinstance(dl, AVNDownload):
          try:
            self.writeFromDownload(dl,
              filename=self.getRequestParam(requestParam, "filename")
              ,noattach=self.getRequestParam(requestParam, 'noattach') is not None)
          except:
            AVNLog.debug("error when downloading %s: %s",dl.filename,traceback.format_exc())
          return
        #legacy handling
        #TODO: removethis
        if dl.get('mimetype') is None:
          raise Exception("no mimetype")
        if dl.get('size') is None:
          raise Exception("no size")
        if dl.get("stream") is None:
          raise Exception("missing stream")
        self.send_response(200)
        fname = self.getRequestParam(requestParam, "filename")
        if fname is not None and fname != "" and self.getRequestParam(requestParam,'noattach') is None:
          self.send_header("Content-Disposition", 'attachment;%s'%AVNDownload.fileToAttach(fname))
        self.send_header("Content-type", dl['mimetype'])
        self.send_header("Content-Length", dl['size'])
        self.send_header("Last-Modified", self.date_time_string())
        self.end_headers()
        try:
          self.writeStream(dl['size'],dl['stream'])
        except Exception as e:
          try:
            dl['stream'].close()
          except:
            pass
        return

    except Exception as e:
      if self.getRequestParam(requestParam,'noattach') is None:
        #send some empty data
        data = io.BytesIO(("error: %s"%str(e)).encode(errors='ignore'))
        data.seek(0)
        self.send_response(200)
        self.send_header("Content-type", "application/octet-stream")
        self.end_headers()
        try:
          self.copyfile(data, self.wfile)
        finally:
          data.close()
          self.close_connection=True
      else:
        self.send_error(404,message=str(e),explain=traceback.format_exc(1))
      return
    self.send_error(404, "invalid download request %s"%type)

  #we use a special form of upload
  #where the file is completely unencoded in the input stream
  #returns json status
  def handleUploadRequest(self,requestParam):
    rlen=None
    try:
      rlen=self.headers.get("Content-Length")
      if rlen is None:
        raise Exception("Content-Length not set in upload request")
      self.connection.settimeout(30)
      type=self.getRequestParam(requestParam,"type")
      handler=self.server.getRequestHandler("upload",type)
      if handler is not None:
        AVNLog.debug("found handler for upload request %s:%s"%(type,handler.getConfigName()))
        rt=handler.handleApiRequest("upload",type,requestParam,rfile=self.rfile,flen=rlen,handler=self)
        return json.dumps(rt,cls=Encoder)
      else:
        raise Exception("invalid request %s",type)
    except Exception as e:
      return json.dumps({'status':str(e)},cls=Encoder)

  def handleListDir(self,requestParam):
    type=self.getRequestParam(requestParam,"type")
    if type is None:
      raise Exception("no type for listdir")
    handler=self.server.getRequestHandler('list',type)
    if handler is not None:
      AVNLog.debug("found handler for list request %s:%s" % (type, handler.getConfigName()))
      rt=handler.handleApiRequest('list',type,requestParam,handler=self)
      if rt is None:
        raise Exception("invalid list response")
      return json.dumps(rt,cls=Encoder)

    raise Exception("invalid type %s"%type)

  def handleDeleteRequest(self,requestParam):
    type=self.getRequestParam(requestParam,"type")
    if type is None:
      raise Exception("no type for delete")
    handler = self.server.getRequestHandler('delete', type)
    rt = {'status': 'OK'}
    if handler is not None:
      AVNLog.debug("found handler for delete request %s:%s" % (type, handler.getConfigName()))
      handler.handleApiRequest('delete', type, requestParam,handler=self)
      return json.dumps(rt,cls=Encoder)
    raise Exception("invalid type %s"%type)

  def handleCapabilityRequest(self,param):
    #see keys.jsx in the viewer at gui.capabilities
    rt={
      'addons':True,
      'uploadCharts':True,
      'plugins':True,
      'uploadRoute': True,
      'uploadLayout':True,
      'uploadUser':True,
      'uploadImages':True,
      'uploadImport': True,
      'uploadOverlays': True,
      'uploadTracks': True,
      'uploadSettings': True,
      'canConnect': True,
      'config': True,
      'debugLevel': True,
      'log': True,
      'remoteChannel': True,
      'fetchHead': True
    }
    return json.dumps({'status':'OK','data':rt},cls=Encoder)


#!/usr/bin/python3
"""
This python module returns the state id of a given state name from
hbtn_0e_6_usa using sqlalchemy ORM in interacting with the database
"""

from sqlalchemy.orm import sessionmaker
from sys import argv
from model_state import Base, State
from sqlalchemy import create_engine

if __name__ == '__main__':
    if (len(argv) != 5):
        exit()
    engine = create_engine('mysql+mysqldb://{}:{}@localhost:3306/{}'.format(
        argv[1], argv[2], argv[3]))
    Base.metadata.create_all(engine)

    stateSearchName = (argv[4],)

    # create a connection Session
    Session = sessionmaker(bind=engine)
    session = Session()

    # quering the database in search of a given state name
    states = session.query(State).filter_by(name=stateSearchName).first()

    # printing the result
    if states:
        print(states.id)
    else:
        print("Not found")


import json
from os import walk

try:
	File = open("Data.json", "r")
	LastDirUsed = json.load(File) #Loads the json file into a dictionary
	File.close()
except:
	LastDirUsed = {}

Data = {}

def DirIndexLoad():
	Results = []
	CurrentSet = ""
	dirPath = ""
	#input(LastDirUsed)

	while True:
		try:
			UserInput = input("Please enter the path to a folder with your JSON sets.\nYour last used folder was located at " + LastDirUsed["RecentDirPath"] + "\nIf you would like to use this folder, press enter")
		except:
			UserInput = input("Please enter the path to a folder with your JSON sets.")
		if UserInput == "":
			DirPath = LastDirUsed["RecentDirPath"]
		else:
			DirPath = UserInput

		#Saving the recently used directory path
		Data["RecentDirPath"] = DirPath
		File = open("Data.json", "w")
		json.dump(Data,File)
		File.close()

		for (DirPath, DirNames, FileNames) in walk(DirPath):
			Results.extend(FileNames)

		Count = 0
		IndexedResults = {}
		for x in Results:
			Count += 1
			print(str(Count) + ". " + x + "\n")
			IndexedResults[Count] = x

		UserInput = input("Type the number of the set you would like to load")

		try:
			File = open(DirPath + "\\" + IndexedResults[int(UserInput)], "r")
			InputDict = json.load(File) #Loads the json file into a dictionary
			File.close()
		except:
			print("Sorry, that file does not exist")
			continue
		else:
			break

	CurrentSet = IndexedResults[int(UserInput)]
	return (InputDict, CurrentSet, DirPath)


from django.db import models
from django.conf import settings

from notification.api import add_notification
import user_profile.models
import dashboard.models

class StudentManager(models.Manager):
    # get or create the referral codes for a user
    def create_student(self, user, school):
        student = user_profile.models.Student(
            user=user, school=school,
        )
        student.save()
        profile = user_profile.models.UserProfile(student=student)
        profile.save()
        roles = user_profile.models.UserRole(user=user)
        roles.save()

        # set default rss settings
        dashboard.models.RSSSetting.objects.restore_default_settings(student)
        return student

    def referral_reward(self, user, referrer):
        reward_points = settings.MCHP_PRICING['referral_reward']
        user.student.add_earned_points(reward_points)
        referrer_roles = user_profile.models.UserRole.objects.get_roles(referrer)
        if referrer_roles.rep:
            referrer.student.modify_balance(1)
            add_notification(
                referrer,
                user.username + " used your code. You've got money in the bank",
            )
        else:
            referrer.student.add_earned_points(reward_points)
            add_notification(
                referrer,
                user.username + " used your code. You made a cool {} points".format(reward_points),
            )

class UserRoleManager(models.Manager):
    def get_roles(self, user):
        roles = user_profile.models.UserRole.objects.filter(user=user)
        if roles.exists():
            return roles[0]
        else:
            roles = user_profile.models.UserRole(user=user)
            roles.save()
            return roles

class OneTimeEventManager(models.Manager):
    @staticmethod
    def get_event(name):
        if not name:
            return None

        return user_profile.models.OneTimeEvent.objects.get_or_create(
            name=name
        )[0]

class OneTimeFlagManager(models.Manager):
    @staticmethod
    def clear_flags(student):
        user_profile.models.OneTimeFlag.objects.filter(
            student=student
        ).delete()

    @staticmethod
    def set_flag(student, event):
        flag, created = user_profile.models.OneTimeFlag.objects.get_or_create(
            student=student,
            event=event,
        )
        return created

    @staticmethod
    def get_flags(student):
        return user_profile.models.OneTimeFlag.objects.filter(
            student=student
        )

    @staticmethod
    def get_flag(student, event_name):
        # usually this mean the user is not logged in, so don't show any of these
        if not student:
            return (True,)

        event = user_profile.models.OneTimeEvent.objects.filter(
            name=event_name
        )

        flag = user_profile.models.OneTimeFlag.objects.filter(
            student=student,
            event=event,
        )
        return flag.exists()


from absl.testing import parameterized
import numpy as np
import tensorflow as tf

from tensorflow.python.keras import keras_parameterized  # pylint: disable=g-direct-tensorflow-import
from official.nlp.modeling.networks import bert_encoder

# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It
# guarantees forward compatibility of this code for the V2 switchover.
@keras_parameterized.run_all_keras_modes
class BertEncoderTest(keras_parameterized.TestCase):

  def tearDown(self):
    super(BertEncoderTest, self).tearDown()
    tf.keras.mixed_precision.experimental.set_policy("float32")

  def test_network_creation(self):
    hidden_size = 32
    sequence_length = 21
    # Create a small BertEncoder for testing.
    test_network = bert_encoder.BertEncoder(
        vocab_size=100,
        hidden_size=hidden_size,
        num_attention_heads=2,
        num_layers=3)
    # Create the inputs (note that the first dimension is implicit).
    word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    data, pooled = test_network([word_ids, mask, type_ids])

    self.assertIsInstance(test_network.transformer_layers, list)
    self.assertLen(test_network.transformer_layers, 3)
    self.assertIsInstance(test_network.pooler_layer, tf.keras.layers.Dense)

    expected_data_shape = [None, sequence_length, hidden_size]
    expected_pooled_shape = [None, hidden_size]
    self.assertAllEqual(expected_data_shape, data.shape.as_list())
    self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list())

    # The default output dtype is float32.
    self.assertAllEqual(tf.float32, data.dtype)
    self.assertAllEqual(tf.float32, pooled.dtype)

    test_network_dict = bert_encoder.BertEncoder(
        vocab_size=100,
        hidden_size=hidden_size,
        num_attention_heads=2,
        num_layers=3,
        dict_outputs=True)
    # Create the inputs (note that the first dimension is implicit).
    inputs = dict(
        input_word_ids=word_ids, input_mask=mask, input_type_ids=type_ids)
    _ = test_network_dict(inputs)

    test_network_dict.set_weights(test_network.get_weights())
    batch_size = 2
    vocab_size = 100
    num_types = 2
    word_id_data = np.random.randint(
        vocab_size, size=(batch_size, sequence_length))
    mask_data = np.random.randint(2, size=(batch_size, sequence_length))
    type_id_data = np.random.randint(
        num_types, size=(batch_size, sequence_length))
    list_outputs = test_network([word_id_data, mask_data, type_id_data])
    dict_outputs = test_network_dict(
        dict(
            input_word_ids=word_id_data,
            input_mask=mask_data,
            input_type_ids=type_id_data))
    self.assertAllEqual(list_outputs[0], dict_outputs["sequence_output"])
    self.assertAllEqual(list_outputs[1], dict_outputs["pooled_output"])

  def test_all_encoder_outputs_network_creation(self):
    hidden_size = 32
    sequence_length = 21
    # Create a small BertEncoder for testing.
    test_network = bert_encoder.BertEncoder(
        vocab_size=100,
        hidden_size=hidden_size,
        num_attention_heads=2,
        num_layers=3,
        return_all_encoder_outputs=True)
    # Create the inputs (note that the first dimension is implicit).
    word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    all_encoder_outputs, pooled = test_network([word_ids, mask, type_ids])

    expected_data_shape = [None, sequence_length, hidden_size]
    expected_pooled_shape = [None, hidden_size]
    self.assertLen(all_encoder_outputs, 3)
    for data in all_encoder_outputs:
      self.assertAllEqual(expected_data_shape, data.shape.as_list())
    self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list())

    # The default output dtype is float32.
    self.assertAllEqual(tf.float32, all_encoder_outputs[-1].dtype)
    self.assertAllEqual(tf.float32, pooled.dtype)

  def test_network_creation_with_float16_dtype(self):
    hidden_size = 32
    sequence_length = 21
    tf.keras.mixed_precision.experimental.set_policy("mixed_float16")
    # Create a small BertEncoder for testing.
    test_network = bert_encoder.BertEncoder(
        vocab_size=100,
        hidden_size=hidden_size,
        num_attention_heads=2,
        num_layers=3)
    # Create the inputs (note that the first dimension is implicit).
    word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    data, pooled = test_network([word_ids, mask, type_ids])

    expected_data_shape = [None, sequence_length, hidden_size]
    expected_pooled_shape = [None, hidden_size]
    self.assertAllEqual(expected_data_shape, data.shape.as_list())
    self.assertAllEqual(expected_pooled_shape, pooled.shape.as_list())

    # If float_dtype is set to float16, the data output is float32 (from a layer
    # norm) and pool output should be float16.
    self.assertAllEqual(tf.float32, data.dtype)
    self.assertAllEqual(tf.float16, pooled.dtype)

  @parameterized.named_parameters(
      ("all_sequence", None, 21),
      ("output_range", 1, 1),
  )
  def test_network_invocation(self, output_range, out_seq_len):
    hidden_size = 32
    sequence_length = 21
    vocab_size = 57
    num_types = 7
    # Create a small BertEncoder for testing.
    test_network = bert_encoder.BertEncoder(
        vocab_size=vocab_size,
        hidden_size=hidden_size,
        num_attention_heads=2,
        num_layers=3,
        type_vocab_size=num_types,
        output_range=output_range)
    # Create the inputs (note that the first dimension is implicit).
    word_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    mask = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    type_ids = tf.keras.Input(shape=(sequence_length,), dtype=tf.int32)
    data, pooled = test_network([word_ids, mask, type_ids])

    # Create a model based off of this network:
    model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled])

    # Invoke the model. We can't validate the output data here (the model is too
    # complex) but this will catch structural runtime errors.
    batch_size = 3
    word_id_data = np.random.randint(
        vocab_size, size=(batch_size, sequence_length))
    mask_data = np.random.randint(2, size=(batch_size, sequence_length))
    type_id_data = np.random.randint(
        num_types, size=(batch_size, sequence_length))
    outputs = model.predict([word_id_data, mask_data, type_id_data])
    self.assertEqual(outputs[0].shape[1], out_seq_len)

    # Creates a BertEncoder with max_sequence_length != sequence_length
    max_sequence_length = 128
    test_network = bert_encoder.BertEncoder(
        vocab_size=vocab_size,
        hidden_size=hidden_size,
        max_sequence_length=max_sequence_length,
        num_attention_heads=2,
        num_layers=3,
        type_vocab_size=num_types)
    data, pooled = test_network([word_ids, mask, type_ids])
    model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled])
    outputs = model.predict([word_id_data, mask_data, type_id_data])
    self.assertEqual(outputs[0].shape[1], sequence_length)

    # Creates a BertEncoder with embedding_width != hidden_size
    test_network = bert_encoder.BertEncoder(
        vocab_size=vocab_size,
        hidden_size=hidden_size,
        max_sequence_length=max_sequence_length,
        num_attention_heads=2,
        num_layers=3,
        type_vocab_size=num_types,
        embedding_width=16)
    data, pooled = test_network([word_ids, mask, type_ids])
    model = tf.keras.Model([word_ids, mask, type_ids], [data, pooled])
    outputs = model.predict([word_id_data, mask_data, type_id_data])
    self.assertEqual(outputs[0].shape[-1], hidden_size)
    self.assertTrue(hasattr(test_network, "_embedding_projection"))

  def test_serialize_deserialize(self):
    # Create a network object that sets all of its config options.
    kwargs = dict(
        vocab_size=100,
        hidden_size=32,
        num_layers=3,
        num_attention_heads=2,
        max_sequence_length=21,
        type_vocab_size=12,
        intermediate_size=1223,
        activation="relu",
        dropout_rate=0.05,
        attention_dropout_rate=0.22,
        initializer="glorot_uniform",
        return_all_encoder_outputs=False,
        output_range=-1,
        embedding_width=16,
        dict_outputs=True,
        embedding_layer=None)
    network = bert_encoder.BertEncoder(**kwargs)
    expected_config = dict(kwargs)
    expected_config["activation"] = tf.keras.activations.serialize(
        tf.keras.activations.get(expected_config["activation"]))
    expected_config["initializer"] = tf.keras.initializers.serialize(
        tf.keras.initializers.get(expected_config["initializer"]))

    self.assertEqual(network.get_config(), expected_config)
    # Create another network object from the first object's config.
    new_network = bert_encoder.BertEncoder.from_config(network.get_config())

    # Validate that the config can be forced to JSON.
    _ = network.to_json()

    # If the serialization was successful, the new config should match the old.
    self.assertAllEqual(network.get_config(), new_network.get_config())

    # Tests model saving/loading.
    model_path = self.get_temp_dir() + "/model"
    network.save(model_path)
    _ = tf.keras.models.load_model(model_path)

if __name__ == "__main__":
  tf.test.main()


###################################################################################################
# this class checks the quality of the calculated SST according to several tests and returns an array with quality flags
# description              ¦  bit ¦ value ¦
#                          ¦      ¦       ¦
# GROSS IR/Valid VIS (day) ¦  01  ¦     1 ¦
# VIS cloud (day)          ¦  02  ¦     2 ¦
# NIR/VIS ratio (day)      ¦  03  ¦     4 ¦
# IR cloud      (night)    ¦  04  ¦     8 ¦
# Low Stratus   (night)    ¦  05  ¦    16 ¦
# SST LE -5°C OR GE 35°C   ¦  06  ¦    32 ¦
# land/sea mask            ¦  07  ¦    64 ¦
# cloud mask               ¦  08  ¦   128 ¦
# Exclude singel pixels    ¦  09  ¦   256 ¦
# Spatial STDV > 3°K       ¦  10  ¦   512 ¦
# Spatial STDV > 1.5°K     ¦  11  ¦  1024 ¦
# SZA > 55°                ¦  12  ¦  2048 ¦
# SZA > 45°                ¦  13  ¦  4096 ¦
# Glint Angle < 36°        ¦  14  ¦  8192 ¦
# Error in QFlag proc.     ¦  15  ¦ 32767 ¦
###################################################################################################

import numpy
from scipy import ndimage

class QualityCheck:
    def __init__(self, lswt, rf1, rf2, bt3, bt4, bt5, sat_za, sun_za, rel_az, lwm, cmsk, height, width, sattype):
        self.qfilter = {'cmask': 7, 'r1lim': 0.005, 'r2lim': 0.1, 'r21lim': 1.0, 'T4lim': 263.15,
                        'lswtLim': [268.15, 308.15], 'vzaLim': 55., 'georefRel': 1.8, 'georefRel2': 1.8, 'std1': 3.,
                        'std2': 1.5, 'VZA2': 45., 'sunglint': 36., 'ircldtshld': 1.0, 'strattshld': -0.6,
                        'cloudTEST': 0}
        self.org_dim = lswt.shape
        self.lswt = numpy.reshape(lswt, (height, width))
        self.rf1 = numpy.reshape(rf1, (height, width))
        self.rf2 = numpy.reshape(rf2, (height, width))
        self.bt3 = numpy.reshape(bt3, (height, width))
        self.bt4 = numpy.reshape(bt4, (height, width))
        self.bt5 = numpy.reshape(bt5, (height, width))

        self.sat_za = numpy.reshape(sat_za, (height, width))
        self.sun_za = numpy.reshape(sun_za, (height, width))
        # Relative azimuth angle at the point for the view angle between the ray to the satellite and the ray to the Sun
        self.rel_az = numpy.reshape(rel_az, (height, width))
        if lwm is not None:
            self.lwm = numpy.reshape(lwm, (height, width))
        else:
            self.lwm = None
        if cmsk is not None:
            self.cmsk = numpy.reshape(cmsk, (height, width))
        else:
            self.cmsk = None

        self.height = height
        self.width = width

        self.sattype = sattype

    def check_quality(self):
        quality_flag = numpy.zeros(self.lswt.shape)

        ###############################################################
        # GROSS IR TEST / VALID VIS TEST # if the channel 4 temperature is less than  #10° then
        # do not compute a SST (BOM)
        ###############################################################
        quality_flag += numpy.logical_and(
            numpy.logical_or(self.bt4 <= self.qfilter['T4lim'], self.rf1 < self.qfilter['r1lim']), self.sun_za < 90) * 1

        ###############################################################
        # VISIBLE CLOUD THRESHOLD TEST # reflectance higher than 0.1 then
        # do not compute a SST (BoM)
        ###############################################################
        quality_flag += numpy.logical_and(self.rf2 >= self.qfilter['r2lim'], self.sun_za < 90) * 2

        ###############################################################
        # NIR/VISIBLE RATIO TEST # ration higher than 1.0 then
        # do not compute a SST (BoM)
        ###############################################################
        if numpy.nanmax(self.sun_za) < 90:
            r21 = self.rf2 / self.rf1
            quality_flag += (r21 > self.qfilter['r21lim']) * 4

        ###############################################################
        # NIGHTTIME IR CLOUD TEST # if a calculated channel 4 temperature based on the channel 5 value
        # (channel 5 temp * 1.0439 # 11.49) differs from the actual channel 4 temperature by more than
        # 1.0° C then do not compute a SST (BoM)
        ###############################################################

        # TODO, GL, this is masking out the LAKES !!!
        syn_ch4 = (self.bt5 * 1.0439) - 11.49
        quality_flag += numpy.logical_and(numpy.absolute(syn_ch4 - self.bt4) >= self.qfilter['ircldtshld'],
                                          self.sun_za > 85) * 8

        ###############################################################
        # NIGHTTIME LOW STRATUS TEST # the difference obtained when subtracting channel 3
        # temperature from channel 5 temperature must be LE #0.6° C. (BoM)
        ###############################################################
        quality_flag += numpy.logical_and((self.bt5 - self.bt3) > self.qfilter['strattshld'], self.bt3 > 100) * 16

        ###############################################################
        # Creating final product, limiting data to #5°C to +35°C
        ###############################################################
        quality_flag += numpy.logical_or(self.lswt <= self.qfilter['lswtLim'][0],
                                         self.lswt >= self.qfilter['lswtLim'][1]) * 32

        ###############################################################
        # Applying Sea and Lake Mask
        ###############################################################
        if self.lwm is not None:
            quality_flag += (self.lwm == 0) * 64
        else:
            quality_flag += 64

        ###############################################################
        # Applying Cloud Mask
        ###############################################################

        # if self.qfilter['cloudTEST'] == 0:
        if self.cmsk is not None:
            if self.sattype == 'AVHRR':
                quality_flag += numpy.logical_or(self.cmsk >= self.qfilter['cmask'], self.cmsk == 0) * 128
            else:
                quality_flag += (self.cmsk != 0) * 128
        else:
            quality_flag += 128

        #####################################
        # EXPERIMENTEL Cloud Mask alternative TEST
        #####################################
        # if self.qfilter['cloudTEST'] == 1:
        #     if numpy.nanmax(self.sun_za) < 80:  # daytime check
        #         cmsk_alt = numpy.logical_not(numpy.logical_or(
        #             numpy.logical_and(numpy.logical_and((self.rf1 + self.rf2) <= 1.2, self.bt5 >= 265),
        #                               (self.rf1 + self.rf2) <= 0.8), self.bt5 >= 285))
        #         cmsk_alt = numpy.logical_or(numpy.logical_or(cmsk_alt, self.cmsk >= self.qfilter['cmask']),
        #                                     self.cmsk == 0)
        #         quality_flag += cmsk_alt * 128
        #     else:
        #         cmsk_alt = numpy.logical_or(
        #             numpy.logical_or(self.bt4 < 243.15, (self.bt5 - self.bt3) > self.qfilter['strattshld']),
        #             numpy.absolute(((self.bt5 * 1.0439) - 11.49) - self.bt4) <= self.qfilter['ircldtshld'])
        #         cmsk_alt = numpy.logical_or(numpy.logical_or(cmsk_alt, self.cmsk >= self.qfilter['cmask']),
        #                                     self.cmsk == 0)
        #         quality_flag += cmsk_alt * 128
        #
        ###############################################################
        # Valid pixels only those which are not completely surrounded by nonvaliddata
        # according to Schwab et al 1999
        ###############################################################
        # the nxn window total not including central pixel
        n = 3
        if quality_flag.ndim == 1:
            raise ArrayDimensionException(quality_flag.ndim)
        else:
            kernel = numpy.ones((n, n))
        # check which pix are good so far
        good_pix = (quality_flag == 0).astype(int)
        # count good neighbors
        neighbors = ndimage.convolve(good_pix, kernel)
        # use only those pix with two or more neighbors
        quality_flag += (neighbors < 2) * 256

        ###############################################################
        # Valid pixels only those which sourrounding pixels have a SDEV from lower 3°C
        # according to Schwab et al 1999
        ###############################################################
        tmp_lswt = self.lswt
        bad_pix = (good_pix == 0).nonzero()
        if bad_pix[0].size != 0:
            tmp_lswt[bad_pix] = 0
        lswt_stdv = self.spatial_stddev(tmp_lswt, 3)

        quality_flag += numpy.logical_and(numpy.isfinite(lswt_stdv), lswt_stdv > self.qfilter['std1']) * 512
        quality_flag += numpy.logical_and(numpy.isfinite(lswt_stdv), lswt_stdv > self.qfilter['std2']) * 1024

        ###############################################################
        # Use only pixels for VZA LE 45 deg.
        ###############################################################
        quality_flag += (self.sat_za > self.qfilter['vzaLim']) * 2048
        quality_flag += (self.sat_za > self.qfilter['VZA2']) * 4096

        ###############################################################
        # Calculate glint angle
        ###############################################################
        glint = numpy.arcsin(numpy.sin(numpy.radians(self.sat_za)) * numpy.sin(numpy.radians(self.sun_za)) * numpy.cos(
            numpy.radians(self.rel_az)) + numpy.cos(numpy.radians(self.sat_za)) * numpy.cos(numpy.radians(self.sun_za)))
        glint = numpy.degrees(glint)
        quality_flag += (glint < self.qfilter['sunglint']) * 8192
        return quality_flag

    def get_quality_flags(self, flag):
        quality_flags = numpy.ones(flag.shape)
        flag = numpy.uint16(flag)
        init_id = numpy.logical_not(numpy.bitwise_and(flag, 511))
        if numpy.sum(init_id) > 0:
            quality_flags[init_id.nonzero()] = 2  # Q1
            quality_flags += init_id * 2  # georef1 Q2
            q3_id = numpy.bitwise_and(init_id, numpy.logical_not(self.bit(flag, 11)))  # VZA1
            if numpy.sum(q3_id) > 0:
                quality_flags[q3_id.nonzero()] = 8  # Q3
            q4_id = numpy.bitwise_and(q3_id, numpy.logical_not(self.bit(flag, 12)))  # VZA2
            if numpy.sum(q4_id) > 0:
                quality_flags[q4_id.nonzero()] = 16  # Q4
            q5_id = numpy.bitwise_and(q4_id, numpy.logical_not(self.bit(flag, 13)))  # glint
            if numpy.sum(q5_id) > 0:
                quality_flags[q5_id.nonzero()] = 32  # Q5
            q6_id = numpy.bitwise_and(q5_id, numpy.logical_not(self.bit(flag, 10)))  # std2
            if numpy.sum(q6_id) > 0:
                quality_flags[q6_id.nonzero()] = 64  # Q6
        return quality_flags

    # Different version: quality levels have value 1 to 6, like in old processor
    # def get_quality_flags(self, flag):
    #     quality_flags = numpy.zeros(flag.shape)
    #     flag = numpy.uint16(flag)
    #     init_id = numpy.logical_not(numpy.bitwise_and(flag, 511))
    #     if numpy.sum(init_id) > 0:
    #         quality_flags[init_id.nonzero()] = 1  # Q1
    #         quality_flags += init_id  # georef1 Q2
    #         q3_id = numpy.bitwise_and(init_id, numpy.logical_not(self.bit(flag, 11)))  # VZA1
    #         if numpy.sum(q3_id) > 0:
    #             quality_flags[q3_id.nonzero()] += 1  # Q3
    #         q4_id = numpy.bitwise_and(q3_id, numpy.logical_not(self.bit(flag, 12)))  # VZA2
    #         if numpy.sum(q4_id) > 0:
    #             quality_flags[q4_id.nonzero()] += 1  # Q4
    #         q5_id = numpy.bitwise_and(q4_id, numpy.logical_not(self.bit(flag, 13)))  # glint
    #         if numpy.sum(q5_id) > 0:
    #             quality_flags[q5_id.nonzero()] += 1  # Q5
    #         q6_id = numpy.bitwise_and(q5_id, numpy.logical_not(self.bit(flag, 10)))  # std2
    #         if numpy.sum(q6_id) > 0:
    #             quality_flags[q6_id.nonzero()] += 1  # Q6
    #     return quality_flags

    @staticmethod
    def bit(value, shift):
        return numpy.bitwise_and(7, numpy.right_shift(value, shift))

    @staticmethod
    def spatial_stddev(in_data, n):
        numpy.set_printoptions(threshold=numpy.nan)
        # Initialize kernel for convolution operation
        if in_data.ndim == 1:
            raise ArrayDimensionException(in_data.ndim)
        else:
            kernel = numpy.ones((n, n))
        # Compute the square of each data point
        data_sq = ndimage.convolve(numpy.power(in_data, 2), kernel)  # square of each data point
        sub_sum = ndimage.convolve(in_data, kernel)  # sum of each n x n pixel window
        n_valid = ndimage.convolve((in_data != 0).astype(int), kernel)
        # number of valid points in n x n pixel window
        data_me = sub_sum / n_valid  # mean of n x n pixel around each pixel, invalid pixels ignored

        # Compute variance
        data_var = data_sq - 2 * data_me * sub_sum + n_valid * numpy.power(data_me, 2)
        data_var /= (n_valid - 1)

        # compute standarddeviation
        return numpy.sqrt(numpy.absolute(data_var))

class ArrayDimensionException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


import os
from typing import List
from matplotlib import pyplot as plt
import more_itertools
import numpy as np
import pandas as pd
import tensorflow as tf
from embedding_dataset import EmbeddingDataset
from embedding_generator import EmbeddingPair
from mongo_db_client import MongoDbClient, MongoDbPairDoc
from runnable import Runnable

class DataAnalysis(Runnable):
  def __init__(self, analysis_dir = '') -> None:
    self.analysis_dir = analysis_dir if len(analysis_dir) > 0 else os.path.join(os.path.abspath(os.curdir), 'analysis')

    if not os.path.isdir(self.analysis_dir):
      os.mkdir(self.analysis_dir)

  def run(self):
    mongo_db = MongoDbClient()

    pairs = list(mongo_db.get_pairs_collection().find({ "partition": "train", "language": "python" }).limit(50))
    positive_pairs_db = self.create_tf_dataset(pairs)
    train_dataset = self.generate_dataset_with_negative_samples(positive_pairs_db)

    self.embeddings_analysis(train_dataset)
    self.raw_data_analysis(train_dataset)

  def raw_data_analysis(self, train_dataset: tf.data.Dataset):
    ids = [pair['id'].decode('utf-8') for pair, target in train_dataset.as_numpy_iterator()]
    pairs: dict[str, MongoDbPairDoc] = { pair['id']: pair for pair in MongoDbClient().get_pairs_collection().find({ "id": { "$in": ids } }) }
    raw_data_path = os.path.join(self.analysis_dir, 'raw_data.csv')

    df_data = []
    sample = 0

    for [positive_emb_pair, pos_target], [negative_emb_pair, neg_target] in more_itertools.chunked(train_dataset.as_numpy_iterator(), 2):
      positive_pair = pairs[positive_emb_pair['id'].decode('utf-8')]
      negative_pair = pairs[negative_emb_pair['id'].decode('utf-8')]

      if positive_pair is None:
        raise ValueError(f"positive pair with id {positive_emb_pair['id']} was not found")

      if negative_pair is None:
        raise ValueError(f"negative pair with id {negative_emb_pair['id']} was not found")

      df_data.append({
        "sample": sample,
        "code": positive_pair['code_tokens'],
        "positive_comment": positive_pair['comment_tokens'],
        "negative_comment": negative_pair['comment_tokens'],
      })
      sample += 1

    df = pd.DataFrame(df_data)
    df.to_csv(raw_data_path)

  def embeddings_analysis(self, train_dataset):
    sample = 0
    for [positive, pos_target], [negative, neg_target] in more_itertools.chunked(train_dataset.as_numpy_iterator(), 2):
      self.save_plot_emb_pairs(positive, negative, f'sample-{sample}')
      sample += 1

  def save_raw_data(self, positive_pair: EmbeddingPair, negative_pair: EmbeddingPair, title: str):
    path = os.path.join(self.analysis_dir, f'{title}.csv')
    with open(path, 'w') as file:
      file.write(f"{positive_pair['code_embedding']},{positive_pair['comment_embedding']},{negative_pair['comment_embedding']}\n")

  def save_plot_emb_pairs(self, positive, negative, title: str):
    fig, axes = plt.subplots(2, 2, constrained_layout=True)
    code_graph, comment_graph, neg_comment_graph, diff_graph = axes[0][0], axes[0][1], axes[1][0], axes[1][1]

    comment_diff = np.absolute(positive['comment_embedding'] - negative['comment_embedding'])

    code_graph.plot(positive['code_embedding'], color='#14d0c5')
    code_graph.set_title('Code')

    comment_graph.plot(positive['comment_embedding'], color='#b1d697')
    comment_graph.set_title('Comment')

    neg_comment_graph.plot(negative['comment_embedding'], color='#6db474')
    neg_comment_graph.set_title('Negative comment')

    diff_graph.plot(comment_diff, color='#8db670')
    diff_graph.set_title('Comment Diff')

    fig.suptitle(title)

    fig_path = os.path.join(self.analysis_dir, title)
    fig.savefig(fig_path)
    plt.close(fig)

  def create_tf_dataset(self, pairs: List[MongoDbPairDoc]) -> tf.data.Dataset:
    def dataset_generator():
      with EmbeddingDataset('embeddings') as embedding_dataset:
        for pair in pairs:
          embedding = embedding_dataset.get(pair["id"])
          if embedding is None:
            raise ValueError(f"{pair['id']} from partition {pair['partition']} was not found")

          yield {
            "id": embedding['id'],
            "code_embedding": embedding["code_embedding"],
            "comment_embedding": embedding["comment_embedding"],
          }

    return tf.data.Dataset.from_generator(dataset_generator, output_types={
      "id": tf.string,
      "code_embedding": tf.float32,
      "comment_embedding": tf.float32,
    })

  def generate_dataset_with_negative_samples(self, positive_samples: tf.data.Dataset) -> tf.data.Dataset:
    negative_samples = positive_samples.shuffle(100)
    def generator():
      for positive_sample, negative_sample in zip(positive_samples, negative_samples):
        yield {
          "id": positive_sample['id'],
          "code_embedding": positive_sample['code_embedding'],
          "comment_embedding": positive_sample['comment_embedding']
        }, 1.0
        yield {
          "id": negative_sample['id'],
          "code_embedding": positive_sample['code_embedding'],
          "comment_embedding": negative_sample['comment_embedding']
        }, 0.0

    return tf.data.Dataset.from_generator(generator, output_types=({
      "id": tf.string,
      "code_embedding": tf.float32,
      "comment_embedding": tf.float32,
    }, tf.float32))


import requests
# If you are using a Jupyter notebook, uncomment the following line.
# %matplotlib inline
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from PIL import Image
from io import BytesIO
import os
import sys
import json
import glob
import time

indir = sys.argv[1]
outdir = sys.argv[2]

dirlist = []
dirlist = glob.glob(os.path.join(indir, '*.jpg'))

# Add your Computer Vision subscription key and endpoint to your environment variables.
if 'COMPUTER_VISION_SUBSCRIPTION_KEY' in os.environ:
    subscription_key = os.environ['COMPUTER_VISION_SUBSCRIPTION_KEY']
else:
    print("\nSet the COMPUTER_VISION_SUBSCRIPTION_KEY environment variable.\n**Restart your shell or IDE for changes to take effect.**")
    sys.exit()

if 'COMPUTER_VISION_ENDPOINT' in os.environ:
    endpoint = os.environ['COMPUTER_VISION_ENDPOINT']

ocr_url = endpoint + "vision/v2.0/ocr"

for image in dirlist:
    image_path = image
    # Read the image into a byte array
    image_data = open(image_path, "rb").read()

    headers = {'Ocp-Apim-Subscription-Key': subscription_key, 'Content-Type': 'application/octet-stream'}
    params = {'language': 'unk', 'detectOrientation': 'true'}
    #data = {'url': image_url}
    response = requests.post(ocr_url, headers=headers, params=params, data = image_data)
    response.raise_for_status()

    analysis = response.json()

    out = os.path.join(outdir, os.path.basename(image.rstrip('.jpg') + '.json'), )
    with open(out, 'w') as outfile:
        outfile.write(json.dumps(analysis))
        outfile.close()
    time.sleep(5)

# Extract the word bounding boxes and text.
# line_infos = [region["lines"] for region in analysis["regions"]]
# word_infos = []
# for line in line_infos:
#     for word_metadata in line:
#         for word_info in word_metadata["words"]:
#             word_infos.append(word_info)
# word_infos


"""City.py - Contains the City object"""

import os, random, datetime
import pygame
from Project import Project

class City(object):
    """The cities with projects to let"""

    def __init__(self, date, name="sometown", xy=(0, 0)):
        self.name = name
        self.location = xy
        self.projects = []

        # project characteristics - below should net to 1,000,000 cm/year in 4 projects
        self.project_probability = 8. / 365.  # probability of a project per day (i.e. 8/year avg)
        self.project_size = 550000.  # average project size
        self.project_growth = 500  # projects grow (shoal) at this rate on average
        self.project_unit = 4.5  # projects start at this unit rate

        self.image_loc = os.path.join('graphics', 'city_256x256.png')
        image = pygame.image.load(self.image_loc).convert()
        self.image_size = (32, 32)
        self.image = pygame.transform.scale(image, self.image_size)
        self.plane = None

        self.generate_project(date)

    def generate_project(self, date, probability=1):
        """generateProject - Add a project to the city
           probability is the probability of a project.
        """
        if random.random() <= probability:
            size = random.normalvariate(self.project_size, self.project_size * 0.25)
            size = int(size / 1000.) * 1000
            unit = random.normalvariate(self.project_unit, self.project_unit * 0.2)
            proj = Project(self, date, qty=size, unit=unit)
            self.projects.append(proj)
            return [(self, "Project %s advertised for tender in %s" % (proj.name, self.name))]
        else:
            return []

    def time_step(self, steptimesize, game):
        """update the city as time passes"""
        # eventually generate new projects here
        ts_messages = []
        # Check my projects
        for p in self.projects:
            ts_messages.extend(p.time_step(steptimesize, game))
        # generate any new projects
        ts_messages.extend(self.generate_project(game.date, self.project_probability * steptimesize))
        return ts_messages


import pickle
import numpy as np
from util import *
from os import listdir
from os.path import isfile, join
import bisect
from collections import deque
from torch import nn
import torch
from torch import optim
from torch.autograd import Variable
import pickle
import json
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
import torch.nn.functional as F
import os.path
import random

DATASET_PATH = '/home/james/Desktop/research/datasets/raw-data-95-100/'

class BasicBlockDilated(nn.Module):
    def __init__(self, inplanes, planes, stage=0, block=0, kernel_size=3, numerical_names=False,
                 stride=None, dilations=(1,1)):
        super(BasicBlockDilated, self).__init__()
        self.block = block
        if stride is None:
            if block!=0 or stage==0:
                stride=1
            else:
                stride=2

        self.conv1 = nn.Conv1d(inplanes, planes, kernel_size=kernel_size, stride=stride, padding=dilations[0],
                              bias=False, dilation=dilations[0])
        self.bn1 = nn.BatchNorm1d(planes)

        self.relu = nn.ReLU(inplace=True)

        self.conv2 = nn.Conv1d(planes, planes, kernel_size=kernel_size, padding=dilations[1],
                              bias=False, dilation=dilations[1])

        self.bn2 = nn.BatchNorm1d(planes)

        if block == 0:
            self.conv3 = nn.Conv1d(inplanes, planes, kernel_size=1, stride=stride, bias=False)

            self.bn3 = nn.BatchNorm1d(planes)

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        out = self.conv2(out)
        out = self.bn2(out)
        if self.block == 0:
            identity = self.conv3(x)
            identity = self.bn3(identity)
        out += identity
        out = self.relu(out)

        return out

class ResNet18(nn.Module):
    def __init__(self, blocks=None, block=None, numerical_names = None):
        super(ResNet18, self).__init__()

        inplanes = 1
        in_channels = 64

        if blocks is None:
            blocks = [2, 2, 2, 2]
        if block is None:
            block = BasicBlockDilated
        if numerical_names is None:
            numerical_names = [True] * len(blocks)

        self.conv1 = nn.Conv1d(inplanes, 64, kernel_size=7, stride=2, bias=False, padding=3)

        self.bn1 = nn.BatchNorm1d(64)

        self.relu = nn.ReLU(inplace=True)

        self.maxpool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)

        self.layers = []

        out_channels = 64

        self.dilated_layers = nn.Sequential(
        BasicBlockDilated(64, 64, 0, 0, dilations=(1,2), numerical_names = False),
        BasicBlockDilated(64, 64, 0, 1, dilations=(4,8), numerical_names = False),

        BasicBlockDilated(64, 128, 1, 0, dilations=(1,2), numerical_names = False),
        BasicBlockDilated(128, 128, 1, 1, dilations=(4,8), numerical_names = False),

        BasicBlockDilated(128, 256, 2, 0, dilations=(1,2), numerical_names = False),
        BasicBlockDilated(256, 256, 2, 1, dilations=(4,8), numerical_names = False),

        BasicBlockDilated(256, 512, 3, 0, dilations=(1,2), numerical_names = False),
        BasicBlockDilated(512, 512, 3, 1, dilations=(4,8), numerical_names = False)
        )

        self.avgpool = nn.AdaptiveAvgPool1d(1)

    def forward(self, x):

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)
        out = self.maxpool(out)
        out = self.dilated_layers(out)

        out = self.avgpool(out)

        return out

class CombinedModel(nn.Module):
    '''PyTorch implementation of Var-CNN WF attack'''
    def __init__(self, nb_classes):
        super(CombinedModel, self).__init__()
        self.resnet_time = ResNet18()
        self.resnet_dir = ResNet18()

        self.fc_met = nn.Linear(7, 32)
        self.bn_met = nn.BatchNorm1d(32)
        self.relu_met = nn.ReLU()

        self.fc = nn.Linear(1024, 1024)
        self.bn = nn.BatchNorm1d(1024)

        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(p=0.5)

        self.last_fc = nn.Linear(1024, nb_classes)

    def forward(self, dir_input, time_input):

        t = self.resnet_time(time_input)
        d = self.resnet_dir(dir_input)

        t = t.view((-1, 512))
        d = d.view((-1, 512))

        out = torch.cat((d, t), dim = 1)

        out = self.fc(out)
        out = self.bn(out)
        out = self.relu(out)
        out = self.dropout(out)

        out = self.last_fc(out)
        out = torch.softmax(out, dim = 1)

        return out

    def get_feature_vector(self, dir_input, time_input, num=50):
        t = self.resnet_time(time_input)
        d = self.resnet_dir(dir_input)

        t = t.view((-1, 512))
        d = d.view((-1, 512))

        out = torch.cat((d, t), dim = 1)

        out = self.fc(out)
        out = self.bn(out)

        return out[:num]

class AWF(nn.Module):
    '''PyTorch implementation of AWF attack (Rimmer)'''
    def __init__(self, nb_classes):
        super(AWF, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Dropout(0.25),
            nn.Conv1d(1, 32, 5, 1, 0),
            nn.ReLU(),
            nn.MaxPool1d(4),
        )

        self.conv2 = nn.Sequential(
            nn.Conv1d(32, 32, 5, 1, 0),
            nn.ReLU(),
            nn.MaxPool1d(4),
        )

        self.out = nn.Sequential(
            nn.Flatten(),
            nn.Linear(19936, 500),
            nn.ReLU(),
            nn.Linear(500, nb_classes)
        )

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        output = self.out(x)
        output = torch.softmax(output, dim=1)
        return output

class TestData(Dataset):
    def __init__(self, test_dir, test_time, test_metadata, test_labels, device):
        self.x_dir = test_dir
        self.x_time = test_time
        self.x_metadata = test_metadata
        self.y_test = test_labels
        self.device = device

    def __getitem__(self, index):
        x_dir_cuda = torch.from_numpy(self.x_dir[index]).reshape(1,10000).float().to(self.device)
        x_time_cuda = torch.from_numpy(self.x_time[index]).reshape(1,10000).float().to(self.device)
        x_metadata_cuda = torch.from_numpy(self.x_metadata[index]).float().to(self.device)
        y_test_cuda = torch.from_numpy(self.y_test[index]).float().to(self.device)

        return x_dir_cuda, x_time_cuda, x_metadata_cuda, y_test_cuda

    def __len__(self):
        return len(self.x_dir)

class ValidationData(Dataset):
    def __init__(self, val_dir, val_time, val_metadata, val_labels, device):
        self.x_dir = val_dir
        self.x_time = val_time
        self.x_metadata = val_metadata
        self.y_val = val_labels
        self.device = device

    def __getitem__(self, index):

        x_dir_cuda = torch.from_numpy(self.x_dir[index]).reshape(1,10000).float().to(self.device)
        x_time_cuda = torch.from_numpy(self.x_time[index]).reshape(1,10000).float().to(self.device)
        x_metadata_cuda = torch.from_numpy(self.x_metadata[index]).float().to(self.device)
        y_val_cuda = torch.from_numpy(self.y_val[index]).float().to(self.device)

        return x_dir_cuda, x_time_cuda, x_metadata_cuda, y_val_cuda

    def __len__(self):
        return len(self.x_dir)

class TrainData(Dataset):
    def __init__(self, train_dir, train_time, train_metadata, train_labels, device):
        self.x_dir = train_dir
        self.x_time = train_time
        self.x_metadata = train_metadata
        self.y_train = train_labels
        self.device = device

    def __getitem__(self, index):

        x_dir_cuda = torch.from_numpy(self.x_dir[index]).reshape(1,10000).float().to(self.device)
        x_time_cuda = torch.from_numpy(self.x_time[index]).reshape(1,10000).float().to(self.device)
        x_metadata_cuda = torch.from_numpy(self.x_metadata[index]).float().to(self.device)
        y_train_cuda = torch.from_numpy(self.y_train[index]).float().to(self.device)

        return x_dir_cuda, x_time_cuda, x_metadata_cuda, y_train_cuda

    def __len__(self):
        return len(self.x_dir)

class TorTraffic:
    '''Tor network traffic environment'''
    def __init__(self, step_size=.02, cutoff_time=50, cutoff_length=10000):
        self.current_trace = 0
        self.current_time = 0.0
        self.step_size = step_size
        self.cutoff_time = cutoff_time
        self.cutoff_length = cutoff_length
        self.defended_traces = deque(maxlen=1000)
        self.current_trace_data = ()
        self.accuracy_list = []
        self.trace_data = []
        self.action_size = 16

        #load data for Var-CNN model
        data_dir = 'sirinam_1000/'
        train_dir = np.load(data_dir + 'train_dir.npy')
        train_time = np.load(data_dir + 'train_time.npy')
        train_metadata = np.load(data_dir + 'train_metadata.npy')
        train_labels = np.load(data_dir + 'train_labels.npy')

        val_dir = np.load(data_dir + 'val_dir.npy')
        val_time = np.load(data_dir + 'val_time.npy')
        val_metadata = np.load(data_dir + 'val_metadata.npy')
        val_labels = np.load(data_dir + 'val_labels.npy')

        test_dir = np.load(data_dir + 'test_dir.npy')
        test_time = np.load(data_dir + 'test_time.npy')
        test_metadata = np.load(data_dir + 'test_metadata.npy')
        test_labels = np.load(data_dir + 'test_labels.npy')

        device = torch.device("cuda")
        self.device = device

        nb_classes = 95
        self.model = CombinedModel(nb_classes).to(device)

        train_dataset = TrainData(train_dir, train_time, train_metadata, train_labels, device)
        self.train_loader = DataLoader(train_dataset, batch_size=50, shuffle=True)

        val_dataset = ValidationData(val_dir, val_time, val_metadata, val_labels, device)
        self.val_loader = DataLoader(val_dataset, batch_size=50, shuffle=True)

        test_dataset = TestData(test_dir, test_time, test_metadata, test_labels, device)
        self.test_loader = DataLoader(test_dataset, batch_size=50, shuffle=False)

        #load data for use by RL agent
        for i in range(5000):
            direction = train_dir[i].flatten()
            time = train_time[i].flatten()
            label = train_labels[i]

            #abs_time = np.zeros(self.cutoff_length, dtype=np.float32)
            abs_time = []
            current_time = 0.0
            for i in range(10000):
                if(direction[i] == 0):
                    break
                current_time += time[i]
                abs_time.append(current_time)

            abs_time = np.array(abs_time)

            self.trace_data.append((direction, time, abs_time, label))

    def TrainEpoch(self, optimizer):
        '''Trains attack model for one epoch'''
        train_correct = 0
        total = 0
        for batch_idx, (data_dir, data_time, data_met, target) in enumerate(self.train_loader):
            optimizer.zero_grad()
            output = self.model(data_dir, data_time)

            pred = output.argmax(dim=1, keepdim=True)
            t = target.argmax(dim=1, keepdim=True)
            train_correct += pred.eq(t.view_as(pred)).sum().item()
            total += len(pred)

            loss = F.binary_cross_entropy(output, target)
            loss.backward()
            optimizer.step()
            if batch_idx % 50 == 0:
                print("Loss: {:0.6f}".format(loss.item()))

        print("Train accuracy: {}".format(float(train_correct)/total))

    def TrainAttack(self):
        '''Trains attack (to be used for feature extraction)'''

        if os.path.exists('var_cnn_opt.pth'):
            print('Loading Model')
            self.model = torch.load('var_cnn_opt.pth')
            return

        for i in range(20):
            optimizer = torch.optim.Adam(self.model.parameters(), lr=.001)
            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=np.sqrt(0.1), cooldown=0, min_lr=1e-5)
            self.TrainEpoch(optimizer)

            correct = 0
            total = 0
            for batch_idx, (data_dir, data_time, data_met, target) in enumerate(self.val_loader):
                output = self.model(data_dir, data_time)
                pred = output.argmax(dim=1, keepdim=True)
                t = target.argmax(dim=1, keepdim=True)
                correct += pred.eq(t.view_as(pred)).sum().item()
                total += len(pred)

            val_acc = float(correct) / total
            print('Validation Accuracy: {}'.format(val_acc))

            scheduler.step(val_acc)

        torch.save(self.model, 'var_cnn_opt.pth')

    def getMeanFE(self):
        '''Gets the mean feature extraction vector'''

        with torch.no_grad():
            fv_list = []
            sizing = []
            large_traces = []
            large_fv_list = []

            for batch_idx, (data_dir, data_time, data_met, target) in enumerate(self.val_loader):
                for i in range(50):
                    #may change cutoff later, but for now take traces with over 2500 packets
                    if np.count_nonzero(data_dir.cpu().clone().numpy()[i]) > 3500:
                        large_traces.append((data_dir.cpu().clone().numpy()[i], data_time.cpu().clone().numpy()[i], data_met.cpu().clone().numpy()[i]))

            large_traces = large_traces[:300]
            data_dir_list = np.asarray([x[0] for x in large_traces])
            data_time_list = np.asarray([x[1] for x in large_traces])
            data_met_list = np.asarray([x[2] for x in large_traces])

            for i in range(6):
                data_dir = torch.from_numpy(data_dir_list[i*50:(i+1)*50]).float().reshape(50,1,10000).float().to(self.device)
                data_time = torch.from_numpy(data_time_list[i*50:(i+1)*50]).float().reshape(50,1,10000).float().to(self.device)
                #data_met = torch.from_numpy(data_met_list[i*50:(i+1)*50]).float().reshape(50,7).float().to(self.device)

                self.model.eval()
                fv = self.model.get_feature_vector(data_dir, data_time)
                fv.cpu()
                large_fv_list.append(fv)

            tensor_list = torch.cat(large_fv_list)
            mean_list = torch.mean(tensor_list, 0)

        self.mean_fv = mean_list.cpu().clone().numpy()
        return mean_list.cpu().clone().numpy()

    def get_obs_vector(self):
        '''Returns a vector representing that traffic that has been sent at certain point in time'''

        direction, time_gap, abs_time, label = self.current_trace_data

        #show packets sent before current time
        index = np.searchsorted(abs_time, self.current_time)

        direction = direction[:index]
        time = time_gap[:index]

        direction = np.pad(direction, pad_width=(0, 10000 - len(direction)), mode='constant')
        time = np.pad(time, pad_width=(0, 10000 - len(time)), mode='constant')

        return (direction, time, label)

    def step(self, action):

        #get original feature extraction vector
        if(self.current_time == 0.0):
            dir_seq, time_seq, _,  _ = self.current_trace_data
            self.model.eval()
            fv = self.get_FE(dir_seq, time_seq)
            self.orig_distance = np.linalg.norm(self.mean_fv - fv)
            self.last_distance = self.orig_distance

        self.current_time += self.step_size
        website = self.trace_data[self.current_trace][3]

        #use action to insert packets
        direction, timing, abs_timing, label = self.current_trace_data
        if(action == 0):
            #don't pad
            pass
        elif(action <= 10):
            #add 1-10 download packets at current time
            for i in range(action):
                #get index from timing
                index = np.searchsorted(abs_timing, self.current_time)
                abs_timing = np.insert(abs_timing, index, self.current_time)[:10000]
                #check to make sure this is correct
                timing = np.insert(timing, index, self.current_time-timing[index-1])[:10000]
                direction = np.insert(direction, index, -1)[:10000]

        elif(action > 10 and action <= 15):
            #add 1-5 upload packets
            for i in range(action-10):
                index = np.searchsorted(abs_timing, self.current_time)
                abs_timing = np.insert(abs_timing, index, self.current_time)[:10000]
                timing = np.insert(timing, index, self.current_time-timing[index-1])[:10000]
                direction = np.insert(direction, index, 1)[:10000]

        self.current_trace_data = direction, timing, abs_timing, label

        if(self.current_time >= self.cutoff_time):
            is_done = True
            self.current_trace += 1
            self.current_time = 0.0
        else:
            is_done = False

        if(self.current_trace >= len(self.trace_data)):
            self.current_trace = 0

        return self.get_obs_vector(), is_done, website, self.current_trace

    def reset(self):
        self.current_time = 0.0

        self.current_trace_data = self.trace_data[self.current_trace]

        return self.get_obs_vector()

    def get_FE(self, dir_seq, time_seq):
        '''Gets the feature extractor for an individual trace'''

        with torch.no_grad():
            self.model.eval()
            dir_cuda = torch.from_numpy(dir_seq).reshape(1,1,10000).float().to(self.device)
            dir_cuda = torch.cat([dir_cuda]*50)
            time_cuda = torch.from_numpy(time_seq).reshape(1,1,10000).float().to(self.device)
            time_cuda = torch.cat([time_cuda]*50)

            fv = self.model.get_feature_vector(dir_cuda, time_cuda, num=1)
        '''
        with torch.no_grad():
            self.model.eval()
            fv = self.model(dir_seq, time_seq)
        '''

        return fv.cpu().clone().numpy()

    def get_rewards(self, states):
        '''Gets states, calculates feature vectors and rewards'''
        direction_list = []
        time_list = []
        for i in range(1,51):
            direction, time, label = states[i]
            direction_list.append(torch.from_numpy(direction).reshape(1,1,10000).float().to(self.device))
            time_list.append(torch.from_numpy(time).reshape(1,1,10000).float().to(self.device))

        dir_cuda = torch.cat(direction_list)
        time_cuda = torch.cat(time_list)

        with torch.no_grad():
            self.model.eval()
            fv = self.model.get_feature_vector(dir_cuda, time_cuda, num=50).cpu().clone().numpy()

        rewards = []
        for i in range(50):
            distance = np.linalg.norm(self.mean_fv - fv[i])
            reward = self.last_distance - distance
            self.last_distance = distance
            rewards.append(reward)

        return rewards

    def retrain_attack(self):
        #TODO
        pass

#env = TorTraffic()
#env.TrainAttack()
#env.getMeanFE()
#env.reset()
#env.step(3)


from django.db import models
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

class Article(models.Model):
    '''
    Attributes of a blog post
    '''

    title = models.CharField(
        max_length=100,
    )
    text = models.TextField(
        help_text='The text of the article (in Markdown).',
    )
    created = models.DateField(
        auto_now_add=True,
        help_text='Creation date of the article.',
    )
    modified = models.DateField(
        auto_now=True,
        help_text='Modification date of the article.',
    )
    author = models.ForeignKey(
        User,
        help_text='The author of the article.',
    )

    def __str__(self):
        return self.title

    def get_absolute_url(self):
#        return '/%i' % self.id
        return reverse('blog:detail', args=[self.id])

    class Meta:
        ordering = ('-created',)

class Comment(models.Model):
    '''
    A comment of an article
    '''

    text = models.TextField(
        help_text='The text of the comment.',
    )
    created = models.DateField(
        auto_now_add=True,
        help_text='Creation date of the comment.',
    )
    commenter = models.CharField(
        max_length=50,
        help_text='Author of the comment.'
    )
    article = models.ForeignKey(Article)

    def __str__(self):
        return 'By ' + self.commenter +  ' at ' + str(self.created)

    class Meta:
        ordering = ('-created',)


# standard imports
import logging

# external imports
from fastapi import APIRouter, Depends, status, Query
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session

# local imports
from api import dependencies
from api.api_v1.docs.openapi.api.routes.country import (add_country_endpoint_responses,
                                                        fetch_all_countries_endpoint_responses,
                                                        update_country_endpoint_responses)
from src.config import config
from src import crud
from src.schemas import Paginate
from src.schemas.country import CountryCreate, CountryUpdate
from tasks import sync_supported_languages

logg = logging.getLogger()
router = APIRouter(prefix='/countries')

@router.get("", responses=fetch_all_countries_endpoint_responses())
def fetch_all_countries(db: Session = Depends(dependencies.get_database),
                        limit: int = Query(
                            default=100,
                            title="limit",
                            description="the row to end at when filtering query results (pagination end)",
                        ),
                        skip: int = Query(
                            default=0,
                            title="skip",
                            description="the row to start from when filtering query results (pagination start)")
                        ) -> Paginate:
    """This function fetches all countries from the database.
    :param db: A database session object.
    :type db: Session
    :param limit: The number of rows to return.
    :type limit: int
    :param skip: The number of rows to skip.
    :type skip: int
    :return: A Paginate object.
    :rtype: Paginate
    """
    countries = crud.country.get_multi(db=db, limit=limit, skip=skip)
    return Paginate(values=countries, total=len(countries), prev_link="", next_link="")

@router.post("/add", responses=add_country_endpoint_responses())
def add_country(payload: CountryCreate, db: Session = Depends(dependencies.get_database)) -> JSONResponse:
    """This endpoint adds a country object to the database.
    :param db: A database session object.
    :type db: Session
    :param payload: Country object to be added to the database.
    :type payload: CountryCreate
    :return: A JSON response containing the country object that was added to the database.
    :rtype: JSONResponse
    """
    response = {}
    try:
        country = crud.country.create(db=db, obj_in=payload)
        response['message'] = 'Country successfully added.'
        response['status'] = 0
        response['data'] = jsonable_encoder(country)
        sync_supported_languages.apply_async(queue=config.get("WORKER_QUEUE"))
        return JSONResponse(content=response, status_code=status.HTTP_201_CREATED)
    except IntegrityError as error:
        logg.error(error)
        response['message'] = 'Country already exists.'
        response['status'] = 1
        return JSONResponse(content=response, status_code=status.HTTP_409_CONFLICT)

@router.patch("/update/{country_id}", responses=update_country_endpoint_responses())
def update_country(payload: CountryUpdate,
                   country_id: int,
                   db: Session = Depends(dependencies.get_database)) -> JSONResponse:
    """This endpoint updates a country object in the database.
    :param db: A database session object.
    :type db: Session
    :param payload: Country data to update.
    :type payload: CountryUpdate
    :param country_id: id of the country object to update.
    :type country_id: int
    :return: A JSON response containing the country object that was updated in the database.
    :rtype: JSONResponse
    """
    response = {}
    country = crud.country.get(db=db, object_id=country_id)
    if not country:
        response['message'] = 'Country not found.'
        response['status'] = 1
        return JSONResponse(content=response, status_code=status.HTTP_404_NOT_FOUND)
    try:
        updated_country = crud.country.update(db=db, db_obj=country, obj_in=payload)
        response['message'] = 'Country successfully updated.'
        response['status'] = 0
        response['data'] = jsonable_encoder(updated_country)
        return JSONResponse(content=response, status_code=status.HTTP_200_OK)
    except Exception as error:
        logg.error(error)
        response['message'] = 'Country not updated.'
        response['status'] = 1
        return JSONResponse(content=response, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)


from tkinter import *
import tkinter.messagebox
from functools import partial
import logic

class Application(Frame):
	def __init__(self, master):
		super(Application, self).__init__(master)
		self.grid()

		self.game_logic = logic.ConnectFourRules()

		self.img = PhotoImage(file="b.png")
		self.img_player = [PhotoImage(file='red.png'), PhotoImage(file='yellow.png')]

		modes = ['Connect ' + str(i) for i in range(3, 6)]								### LC

		# first main label contains: current score, turn info, reset button
		self.label_1 = Label(self, height=1).grid()
		self.label_score = Label(self.label_1, text='Wynik\n0:0')
		self.label_score.grid(row=0, column=0, columnspan=1)
		self.label_turn = Label(self.label_1, text='Tura\ngracza\n 1')
		self.label_turn.grid(row=0, column=1, columnspan=1)
		self.button_reset = Button(self.label_1, text='reset', command=lambda: self.reset())		### LAMBDA
		self.button_reset.grid(row=0, column=2, columnspan=1)
		self.game_mode = StringVar(self.label_1)
		self.game_mode.set(modes[1])
		self.w = OptionMenu(self.label_1, self.game_mode, *modes)
		self.w.grid(row=0, column=3, columnspan=2)
		self.button_apply = Button(self.label_1, text='zmień tryb', command=lambda: self.change_mode())		### LAMBDA
		self.button_apply.grid(row=0, column=5, columnspan=2)

		# second main label contains 7 clickable buttons
		self.label_2 = Label(self).grid()
		self.bttn = [Button(self.label_2) for i in range(7)]										### LC
		###
		self.bttn[0].config(image=self.img_player[0], command=lambda: self.new_move(0))				### LAMBDA
		self.bttn[0].grid(row=1, column=0)
		for col in range(1,7):
			self.bttn[col].config(image=self.img_player[0], command=partial(self.new_move, col))
			self.bttn[col].grid(row=1, column=col)

		# third main label contains 6x7 labels representing fields for coins in game
		self.label_3 = Label(self).grid()
		self.label_field = []
		# creating 6x7 labels
		for i in range(6):
			tmp = []
			for j in range(7):
				tmp.append(Label(self.label_3))
				tmp[j].config(image=self.img)
				tmp[j].grid(row=i + 2, column=j)
			self.label_field.append(tmp)

	# this method is called whenever user clicked on any of 7 buttons
	def new_move(self, clicked_col):
		# if move cannot be performed then do nothing
		if self.game_logic.column_not_full(clicked_col):
			self.update_fields(clicked_col, self.game_logic.calculate_row(clicked_col))
		else:
			tkinter.messagebox.showinfo(":(", "Nie zmieści się już :/")

	# this method is called when user picked correct column to put coin in
	def update_fields(self, col, row):
		self.game_logic.update_fields(row, col)
		self.label_field[row][col].config(image=self.img_player[self.game_logic.get_turn()])

		if self.game_logic.draw():
			tkinter.messagebox.showinfo("koniec", "remis")
			self.reset()

		elif not self.game_logic.check_end():
			self.game_logic.change_turn()
			for i in range(7):
				self.bttn[i].config(image=self.img_player[self.game_logic.get_turn()])
			self.label_turn.config(text='Tura\ngracza\n' + str(self.game_logic.get_turn() + 1))

		else:
			tkinter.messagebox.showinfo("koniec", "wygrał gracz " + str(self.game_logic.get_turn() + 1))
			self.game_logic.add_point()
			temp = self.game_logic.get_score()
			self.label_score.config(text='Wynik\n' + str(temp[0]) + ':' + str(temp[1]))

			self.reset()

	def reset(self):
		self.game_logic.reset()
		self.label_turn.config(text='Tura\ngracza\n' + str(self.game_logic.get_turn() + 1))

		for i in range(7):
			self.bttn[i].config(image=self.img_player[0])
		for i in range(6):
			for j in range(7):
				self.label_field[i][j].config(image=self.img)

	def change_mode(self):
		self.label_score.config(text='Wynik\n0:0')
		new_mode = str(self.game_mode.get())
		if new_mode == 'Connect 5':
			self.game_logic = logic.ConnectFiveRules()
		elif new_mode == 'Connect 4':
			self.game_logic = logic.ConnectFourRules()
		elif new_mode == 'Connect 3':
			self.game_logic = logic.ConnectThreeRules()
		self.reset()