import ctypes
import enum

class SysError(enum.IntEnum):
    EPERM = -1  # Operation not permitted */
    ENOENT = -2  # No such file or directory */
    ESRCH = -3  # No such process */
    EINTR = -4  # Interrupted system call */
    EIO = -5  # I/O error */
    ENXIO = -6  # No such device or address */
    E2BIG = -7  # Arg list too long */
    ENOEXEC = -8  # Exec format error */
    EBADF = -9  # Bad file number */
    ECHILD = -10  # No child processes */
    EAGAIN = -11  # Try again */
    ENOMEM = -12  # Out of memory */
    EACCES = -13  # Permission denied */
    EFAULT = -14  # Bad address */
    ENOTBLK = -15  # Block device required */
    EBUSY = -16  # Device or resource busy */
    EEXIST = -17  # File exists */
    EXDEV = -18  # Cross-device link */
    ENODEV = -19  # No such device */
    ENOTDIR = -20  # Not a directory */
    EISDIR = -21  # Is a directory */
    EINVAL = -22  # Invalid argument */
    ENFILE = -23  # File table overflow */
    EMFILE = -24  # Too many open files */
    ENOTTY = -25  # Not a typewriter */
    ETXTBSY = -26  # Text file busy */
    EFBIG = -27  # File too large */
    ENOSPC = -28  # No space left on device */
    ESPIPE = -29  # Illegal seek */
    EROFS = -30  # Read-only file system */
    EMLINK = -31  # Too many links */
    EPIPE = -32  # Broken pipe */
    EDOM = -33  # Math argument out of domain of func */
    ERANGE = -34  # Math result not representable */
    EDEADLK = -35  # Resource deadlock would occur */
    ENAMETOOLONG = -36  # File name too long */
    ENOLCK = -37  # No record locks available */
    ENOSYS = -38  # Function not implemented */
    ENOTEMPTY = -39  # Directory not empty */
    ELOOP = -40  # Too many symbolic links encountered */
    EWOULDBLOCK = EAGAIN  # Operation would block */
    ENOMSG = -42  # No message of desired type */
    EIDRM = -43  # Identifier removed */
    ECHRNG = -44  # Channel number out of range */
    EL2NSYNC = -45  # Level 2 not synchronized */
    EL3HLT = -46  # Level 3 halted */
    EL3RST = -47  # Level 3 reset */
    ELNRNG = -48  # Link number out of range */
    EUNATCH = -49  # Protocol driver not attached */
    ENOCSI = -50  # No CSI structure available */
    EL2HLT = -51  # Level 2 halted */
    EBADE = -52  # Invalid exchange */
    EBADR = -53  # Invalid request descriptor */
    EXFULL = -54  # Exchange full */
    ENOANO = -55  # No anode */
    EBADRQC = -56  # Invalid request code */
    EBADSLT = -57  # Invalid slot */

    EDEADLOCK = EDEADLK
    EBFONT = -59  # Bad font file format */
    ENOSTR = -60  # Device not a stream */
    ENODATA = -61  # No data available */
    ETIME = -62  # Timer expired */
    ENOSR = -63  # Out of streams resources */
    ENONET = -64  # Machine is not on the network */
    ENOPKG = -65  # Package not installed */
    EREMOTE = -66  # Object is remote */
    ENOLINK = -67  # Link has been severed */
    EADV = -68  # Advertise error */
    ESRMNT = -69  # Srmount error */
    ECOMM = -70  # Communication error on send */
    EPROTO = -71  # Protocol error */
    EMULTIHOP = -72  # Multihop attempted */
    EDOTDOT = -73  # RFS specific error */
    EBADMSG = -74  # Not a data message */
    EOVERFLOW = -75  # Value too large for defined data type */
    ENOTUNIQ = -76  # Name not unique on network */
    EBADFD = -77  # File descriptor in bad state */
    EREMCHG = -78  # Remote address changed */
    ELIBACC = -79  # Can not access a needed shared library */
    ELIBBAD = -80  # Accessing a corrupted shared library */
    ELIBSCN = -81  # .lib section in a.out corrupted */
    ELIBMAX = -82  # Attempting to link in too many shared libraries */
    ELIBEXEC = -83  # Cannot exec a shared library directly */
    EILSEQ = -84  # Illegal byte sequence */
    ERESTART = -85  # Interrupted system call should be restarted */
    ESTRPIPE = -86  # Streams pipe error */
    EUSERS = -87  # Too many users */
    ENOTSOCK = -88  # Socket operation on non-socket */
    EDESTADDRREQ = -89  # Destination address required */
    EMSGSIZE = -90  # Message too long */
    EPROTOTYPE = -91  # Protocol wrong type for socket */
    ENOPROTOOPT = -92  # Protocol not available */
    EPROTONOSUPPORT = -93  # Protocol not supported */
    ESOCKTNOSUPPORT = -94  # Socket type not supported */
    EOPNOTSUPP = -95  # Operation not supported on transport endpoint */
    EPFNOSUPPORT = -96  # Protocol family not supported */
    EAFNOSUPPORT = -97  # Address family not supported by protocol */
    EADDRINUSE = -98  # Address already in use */
    EADDRNOTAVAIL = -99  # Cannot assign requested address */
    ENETDOWN = -100  # Network is down */
    ENETUNREACH = -101  # Network is unreachable */
    ENETRESET = -102  # Network dropped connection because of reset */
    ECONNABORTED = -103  # Software caused connection abort */
    ECONNRESET = -104  # Connection reset by peer */
    ENOBUFS = -105  # No buffer space available */
    EISCONN = -106  # Transport endpoint is already connected */
    ENOTCONN = -107  # Transport endpoint is not connected */
    ESHUTDOWN = -108  # Cannot send after transport endpoint shutdown */
    ETOOMANYREFS = -109  # Too many references =  splice */
    ETIMEDOUT = -110  # Connection timed out */
    ECONNREFUSED = -111  # Connection refused */
    EHOSTDOWN = -112  # Host is down */
    EHOSTUNREACH = -113  # No route to host */
    EALREADY = -114  # Operation already in progress */
    EINPROGRESS = -115  # Operation now in progress */
    ESTALE = -116  # Stale NFS file handle */
    EUCLEAN = -117  # Structure needs cleaning */
    ENOTNAM = -118  # Not a XENIX named type file */
    ENAVAIL = -119  # No XENIX semaphores available */
    EISNAM = -120  # Is a named type file */
    EREMOTEIO = -121  # Remote I/O error */
    EDQUOT = -122  # Quota exceeded */

    ENOMEDIUM = -123  # No medium found */
    EMEDIUMTYPE = -124  # Wrong medium type */

ARCH_PRCTL_OPTIONS = {
    0x1001: "ARCH_SET_GS",
    0x1002: "ARCH_SET_FS",
    0x1003: "ARCH_GET_FS",
    0x1004: "ARCH_GET_GS",
}

FCNTL_COMMANDS = {
    0: "F_DUPFD",  # dup
    1: "F_GETFD",  # get close_on_exec
    2: "F_SETFD",  # set/clear close_on_exec
    3: "F_GETFL",  # get file->f_flags
    4: "F_SETFL",  # set file->f_flags
    5: "F_GETLK",
    6: "F_SETLK",
    7: "F_SETLKW",
    8: "F_SETOWN",  # for sockets.
    9: "F_GETOWN",  # for sockets.
    10: "F_SETSIG",  # for sockets.
    11: "F_GETSIG",  # for sockets.
    12: "F_GETLK64",  # using 'struct flock64'
    13: "F_SETLK64",
    14: "F_SETLKW64",
    15: "F_SETOWN_EX",
    16: "F_GETOWN_EX",
    17: "F_GETOWNER_UIDS",
}

# Used to determine what kind of socket is being used.
protocol_families = {
    1: "UNIX",  # /* Unix domain sockets 		*/
    2: "INET",  # /* Internet IP Protocol 	*/
    3: "AX25",  # /* Amateur Radio AX.25 		*/
    4: "IPX",  # /* Novell IPX 			*/
    5: "APPLETALK",  # /* AppleTalk DDP 		*/
    6: "NETROM",  # /* Amateur Radio NET/ROM 	*/
    7: "BRIDGE",  # /* Multiprotocol bridge 	*/
    8: "ATMPVC",  # /* ATM PVCs			*/
    9: "X25",  # /* Reserved for X.25 project 	*/
    10: "INET6",  # /* IP version 6			*/
    11: "ROSE",  # /* Amateur Radio X.25 PLP	*/
    12: "DECnet",  # /* Reserved for DECnet project	*/
    13: "NETBEUI",  # /* Reserved for 802.2LLC project*/
    14: "SECURITY",  # /* Security callback pseudo AF */
    15: "KEY",  # /* PF_KEY key management API */
    16: "NETLINK",  # /* Alias to emulate 4.4BSD */
    17: "PACKET",  # /* Packet family		*/
    18: "ASH",  # * Ash				*/
    19: "ECONET",  # /* Acorn Econet			*/
    20: "ATMSVC",  # /* ATM SVCs			*/
    21: "RDS",  # /* RDS sockets 			*/
    22: "SNA",  # /* Linux SNA Project (nutters!) */
    23: "IRDA",  # /* IRDA sockets			*/
    24: "PPPOX",  # /* PPPoX sockets		*/
    25: "WANPIPE",  # /* Wanpipe API Sockets */
    26: "LLC",  # /* Linux LLC			*/
    27: "IB",  # /* Native InfiniBand address	*/
    28: "MPLS",  # * MPLS */
    29: "CAN",  # /* Controller Area Network      */
    30: "TIPC",  # /* TIPC sockets			*/
    31: "BLUETOOTH",  # /* Bluetooth sockets 		*/
    32: "IUCV",  # /* IUCV sockets			*/
    33: "RXRPC",  # /* RxRPC sockets 		*/
    34: "ISDN",  # /* mISDN sockets 		*/
    35: "PHONET",  # /* Phonet sockets		*/
    36: "IEEE802154",  # /* IEEE802154 sockets		*/
    37: "CAIF",  # /* CAIF sockets			*/
    38: "ALG",  # /* Algorithm sockets		*/
    39: "NFC",  # /* NFC sockets			*/
    40: "VSOCK",  # * vSockets			*/
    41: "KCM",  # /* Kernel Connection Multiplexor*/
    42: "QIPCRTR",  # /* Qualcomm IPC Router          */
    43: "SMC",  # /* smc sockets: reserve number for
    44: "XDP",  # /* XDP sockets			*/
    45: "MAX",  # /* For now.. */
}

class SOCKADDR(ctypes.Structure):
    _fields_ = [
        ("sin_family", ctypes.c_ushort),
        ("sa_data", ctypes.c_char * 8),
    ]

class SOCKADDR_UN(ctypes.Structure):
    _fields_ = [
        ("sun_family", ctypes.c_ushort),
        ("sun_path", ctypes.c_char * 108),
    ]

class SOCKADDR_IN(ctypes.Structure):
    _fields_ = [
        ("sin_family", ctypes.c_ushort),
        ("sin_port", ctypes.c_ushort),
        ("sin_addr", ctypes.c_uint32),
        ("sin_zero", ctypes.c_char * 8),
    ]

class SOCKADDR_IN6(ctypes.Structure):
    _fields_ = [
        ("sin6_family", ctypes.c_ushort),
        ("sin6_port", ctypes.c_ushort),
        ("sin6_flowinfo", ctypes.c_uint32),
        ("sin6_addr", ctypes.c_uint64),
        ("sin6_scope_id", ctypes.c_uint32),
    ]

class SocketFamily(enum.IntEnum):
    """
    Socket Family:
    https://github.com/torvalds/linux/blob/master/include/linux/socket.h
    """

    AF_UNSPEC = 0
    AF_UNIX = 1  # /* Unix domain sockets */
    AF_LOCAL = 1  # /* POSIX name for AF_UNIX */
    AF_INET = 2  # /* Internet IP Protocol */
    AF_AX25 = 3  # /* Amateur Radio AX.25 */
    AF_IPX = 4  # /* Novell IPX */
    AF_APPLETALK = 5  # /* AppleTalk DDP */
    AF_NETROM = 6  # /* Amateur Radio NET/ROM */
    AF_BRIDGE = 7  # /* Multiprotocol bridge */
    AF_ATMPVC = 8  # /* ATM PVCs */
    AF_X25 = 9  # /* Reserved for X.25 project */
    AF_INET6 = 10  # /* IP version 6 */
    AF_ROSE = 11  # /* Amateur Radio X.25 PLP */
    AF_DECnet = 12  # /* Reserved for DECnet project */
    AF_NETBEUI = 13  # /* Reserved for 802.2LLC project*/
    AF_SECURITY = 14  # /* Security callback pseudo AF */
    AF_KEY = 15  # /* PF_KEY key management API */
    AF_NETLINK = 16
    AF_ROUTE = 16  # /* AF_NETLINK: Alias to emulate 4.4BSD */
    AF_PACKET = 17  # /* Packet family */
    AF_ASH = 18  # /* Ash */
    AF_ECONET = 19  # /* Acorn Econet */
    AF_ATMSVC = 20  # /* ATM SVCs */
    AF_RDS = 21  # /* RDS sockets */
    AF_SNA = 22  # /* Linux SNA Project (nutters!) */
    AF_IRDA = 23  # /* IRDA sockets */
    AF_PPPOX = 24  # /* PPPoX sockets */
    AF_WANPIPE = 25  # /* Wanpipe API Sockets */
    AF_LLC = 26  # /* Linux LLC */
    AF_IB = 27  # /* Native InfiniBand address */
    AF_MPLS = 28  # /* MPLS */
    AF_CAN = 29  # /* Controller Area Network */
    AF_TIPC = 30  # /* TIPC sockets */
    AF_BLUETOOTH = 31  # /* Bluetooth sockets */
    AF_IUCV = 32  # /* IUCV sockets */
    AF_RXRPC = 33  # /* RxRPC sockets */
    AF_ISDN = 34  # /* mISDN sockets */
    AF_PHONET = 35  # /* Phonet sockets */
    AF_IEEE802154 = 36  # /* IEEE802154 sockets */
    AF_CAIF = 37  # /* CAIF sockets */
    AF_ALG = 38  # /* Algorithm sockets */
    AF_NFC = 39  # /* NFC sockets */
    AF_VSOCK = 40  # /* vSockets */
    AF_KCM = 41  # /* Kernel Connection Multiplexor*/
    AF_QIPCRTR = 42  # /* Qualcomm IPC Router */
    AF_SMC = 43  # /* smc sockets: reserve for PF_SMC protocol family */
    AF_XDP = 44  # /* XDP sockets */
    AF_MAX = 45  # /* For now.. */

class CommonSocketType(enum.IntEnum):
    """
    Socket Types:
    https://github.com/torvalds/linux/blob/master/include/linux/net.h
    """

    # Mips defines SOCK_STREAM and SOCK_DGRAM differently.
    SOCK_STREAM = 1
    SOCK_DGRAM = 2

    SOCK_RAW = 3
    SOCK_RDM = 4
    SOCK_SEQPACKET = 5
    SOCK_DCCP = 6
    SOCK_PACKET = 10
    SOCK_MAX = 11

    SOCK_CLOEXEC = 0o2000000
    SOCK_NONBLOCK = 0o4000

class MipsSocketType(enum.IntEnum):
    # Mips is special :)
    # https://github.com/torvalds/linux/
    # blob/master/arch/mips/include/asm/socket.h

    SOCK_DGRAM = 1
    SOCK_STREAM = 2

    SOCK_RAW = 3
    SOCK_RDM = 4
    SOCK_SEQPACKET = 5
    SOCK_DCCP = 6
    SOCK_PACKET = 10
    SOCK_MAX = 11

    SOCK_CLOEXEC = 0o2000000
    SOCK_NONBLOCK = 0o4000

class SocketProtocol(enum.IntEnum):
    IPPROTO_IP = 0  # Dummy protocol for TCP.
    IPPROTO_ICMP = 1  # Internet Control Message Protocol.
    IPPROTO_IGMP = 2  # Internet Group Management Protocol.
    IPPROTO_IPIP = 4  # IPIP tunnels (older KA9Q tunnels use 94).
    IPPROTO_TCP = 6  # Transmission Control Protocol.
    IPPROTO_EGP = 8  # Exterior Gateway Protocol.
    IPPROTO_PUP = 12  # PUP protocol.
    IPPROTO_UDP = 17  # User Datagram Protocol.
    IPPROTO_IDP = 22  # XNS IDP protocol.
    IPPROTO_TP = 29  # SO Transport Protocol Class 4.
    IPPROTO_DCCP = 33  # Datagram Congestion Control Protocol.
    IPPROTO_IPV6 = 41  # IPv6 header.
    IPPROTO_RSVP = 46  # Reservation Protocol.
    IPPROTO_GRE = 47  # General Routing Encapsulation.
    IPPROTO_ESP = 50  # encapsulating security payload.
    IPPROTO_AH = 51  # authentication header.
    IPPROTO_MTP = 92  # Multicast Transport Protocol.
    IPPROTO_BEETPH = 94  # IP option pseudo header for BEET.
    IPPROTO_ENCAP = 98  # Encapsulation Header.
    IPPROTO_PIM = 103  # Protocol Independent Multicast.
    IPPROTO_COMP = 108  # Compression Header Protocol.
    IPPROTO_SCTP = 132  # Stream Control Transmission Protocol.
    IPPROTO_UDPLITE = 136  # UDP-Lite protocol.
    IPPROTO_MPLS = 137  # MPLS in IP.
    IPPROTO_RAW = 255  # Raw IP packets.

class SocketOptionsLevels(enum.IntEnum):
    IPPROTO_IP = 0
    SOL_SOCKET = 1
    IPPROTO_TCP = 6
    IPPROTO_IPV6 = 41

class SocketOptionsSocket(enum.IntEnum):
    SO_DEBUG = 1
    SO_REUSEADDR = 2
    SO_TYPE = 3
    SO_ERROR = 4
    SO_DONTROUTE = 5
    SO_BROADCAST = 6
    SO_SNDBUF = 7
    SO_RCVBUF = 8
    SO_SNDBUFFORCE = 32
    SO_RCVBUFFORCE = 33
    SO_KEEPALIVE = 9
    SO_OOBINLINE = 10
    SO_NO_CHECK = 11
    SO_PRIORITY = 12
    SO_LINGER = 13
    SO_BSDCOMPAT = 14
    SO_REUSEPORT = 15
    SO_PASSCRED = 16
    SO_PEERCRED = 17
    SO_RCVLOWAT = 18
    SO_SNDLOWAT = 19
    SO_RCVTIMEO = 20
    SO_SNDTIMEO = 21

    # Security levels - as per NRL IPv6 - don't actually do anything
    SO_SECURITY_AUTHENTICATION = 22
    SO_SECURITY_ENCRYPTION_TRANSPORT = 23
    SO_SECURITY_ENCRYPTION_NETWORK = 24
    SO_BINDTODEVICE = 25

    # Socket filtering
    SO_ATTACH_FILTER = 26
    SO_DETACH_FILTER = 27
    SO_PEERNAME = 28
    SO_TIMESTAMP = 29
    SO_ACCEPTCONN = 30
    SO_PEERSEC = 31
    SO_PASSSEC = 34
    SO_TIMESTAMPNS = 35
    SO_MARK = 36
    SO_TIMESTAMPING = 37
    SO_PROTOCOL = 38
    SO_DOMAIN = 39
    SO_RXQ_OVFL = 40
    SO_WIFI_STATUS = 41
    SO_PEEK_OFF = 42

    # Instruct lower device to use last 4-bytes of skb data as FCS
    SO_NOFCS = 43
    SO_LOCK_FILTER = 44
    SO_SELECT_ERR_QUEUE = 45
    SO_BUSY_POLL = 46
    SO_MAX_PACING_RATE = 47
    SO_BPF_EXTENSIONS = 48
    SO_INCOMING_CPU = 49
    SO_ATTACH_BPF = 50
    SO_ATTACH_REUSEPORT_CBPF = 51
    SO_ATTACH_REUSEPORT_EBPF = 52
    SO_CNX_ADVICE = 53
    SCM_TIMESTAMPING_OPT_STATS = 54
    SO_MEMINFO = 55
    SO_INCOMING_NAPI_ID = 56
    SO_COOKIE = 57
    SCM_TIMESTAMPING_PKTINFO = 58
    SO_PEERGROUPS = 59
    SO_ZEROCOPY = 60

class SocketOptionsIp(enum.IntEnum):
    IP_TOS = 1
    IP_TTL = 2
    IP_HDRINCL = 3
    IP_OPTIONS = 4
    IP_ROUTER_ALERT = 5
    IP_RECVOPTS = 6
    IP_RETOPTS = 7
    IP_PKTINFO = 8
    IP_PKTOPTIONS = 9
    IP_MTU_DISCOVER = 10
    IP_RECVERR = 11
    IP_RECVTTL = 12
    IP_RECVTOS = 13
    IP_MTU = 14
    IP_FREEBIND = 15
    IP_IPSEC_POLICY = 16
    IP_XFRM_POLICY = 17
    IP_PASSSEC = 18
    IP_TRANSPARENT = 19

    # TProxy original addresses
    IP_ORIGDSTADDR = 20

    IP_MINTTL = 21
    IP_NODEFRAG = 22
    IP_CHECKSUM = 23
    IP_BIND_ADDRESS_NO_PORT = 24
    IP_RECVFRAGSIZE = 25

    IP_MULTICAST_IF = 32
    IP_MULTICAST_TTL = 33
    IP_MULTICAST_LOOP = 34
    IP_ADD_MEMBERSHIP = 35
    IP_DROP_MEMBERSHIP = 36
    IP_UNBLOCK_SOURCE = 37
    IP_BLOCK_SOURCE = 38
    IP_ADD_SOURCE_MEMBERSHIP = 39
    IP_DROP_SOURCE_MEMBERSHIP = 40
    IP_MSFILTER = 41
    MCAST_JOIN_GROUP = 42
    MCAST_BLOCK_SOURCE = 43
    MCAST_UNBLOCK_SOURCE = 44
    MCAST_LEAVE_GROUP = 45
    MCAST_JOIN_SOURCE_GROUP = 46
    MCAST_LEAVE_SOURCE_GROUP = 47
    MCAST_MSFILTER = 48
    IP_MULTICAST_ALL = 49
    IP_UNICAST_IF = 50

class SocketOptionsTcp(enum.IntEnum):
    TCP_NODELAY = 1  # Turn off Nagle's algorithm.
    TCP_MAXSEG = 2  # Limit MSS
    TCP_CORK = 3  # Never send partially complete segments
    TCP_KEEPIDLE = 4  # Start keeplives after this period
    TCP_KEEPINTVL = 5  # Interval between keepalives
    TCP_KEEPCNT = 6  # Number of keepalives before death
    TCP_SYNCNT = 7  # Number of SYN retransmits
    TCP_LINGER2 = 8  # Life time of orphaned FIN-WAIT-2 state
    TCP_DEFER_ACCEPT = 9  # Wake up listener only when data arrive
    TCP_WINDOW_CLAMP = 10  # Bound advertised window
    TCP_INFO = 11  # Information about this connection.
    TCP_QUICKACK = 12  # Block/reenable quick acks
    TCP_CONGESTION = 13  # Congestion control algorithm
    TCP_MD5SIG = 14  # TCP MD5 Signature (RFC2385)
    TCP_THIN_LINEAR_TIMEOUTS = 16  # Use linear timeouts for thin streams*/
    TCP_THIN_DUPACK = 17  # Fast retrans. after = 1 dupack
    TCP_USER_TIMEOUT = 18  # How long for loss retry before timeout
    TCP_REPAIR = 19  # TCP sock is under repair right now
    TCP_REPAIR_QUEUE = 20
    TCP_QUEUE_SEQ = 21
    TCP_REPAIR_OPTIONS = 22
    TCP_FASTOPEN = 23  # Enable FastOpen on listeners
    TCP_TIMESTAMP = 24
    TCP_NOTSENT_LOWAT = 25  # limit number of unsent bytes in write queue
    TCP_CC_INFO = 26  # Get Congestion Control (optional) info
    TCP_SAVE_SYN = 27  # Record SYN headers for new connections
    TCP_SAVED_SYN = 28  # Get SYN headers recorded for connection
    TCP_REPAIR_WINDOW = 29  # Get/set window parameters
    TCP_FASTOPEN_CONNECT = 30  # Attempt FastOpen with connect
    TCP_ULP = 31  # Attach a ULP to a TCP connection
    TCP_MD5SIG_EXT = 32  # TCP MD5 Signature with extensions
    TCP_FASTOPEN_KEY = 33  # Set the key for Fast Open (cookie)
    TCP_FASTOPEN_NO_COOKIE = 34  # Enable TFO without a TFO cookie

class SocketOptionsIPV6(enum.IntEnum):
    IPV6_ADDRFORM = 1
    IPV6_2292PKTINFO = 2
    IPV6_2292HOPOPTS = 3
    IPV6_2292DSTOPTS = 4
    IPV6_2292RTHDR = 5
    IPV6_2292PKTOPTIONS = 6
    IPV6_CHECKSUM = 7
    IPV6_2292HOPLIMIT = 8
    IPV6_NEXTHOP = 9
    IPV6_AUTHHDR = 10
    IPV6_FLOWINFO = 11

    IPV6_UNICAST_HOPS = 16
    IPV6_MULTICAST_IF = 17
    IPV6_MULTICAST_HOPS = 18
    IPV6_MULTICAST_LOOP = 19
    IPV6_ADD_MEMBERSHIP = 20
    IPV6_DROP_MEMBERSHIP = 21
    IPV6_ROUTER_ALERT = 22
    IPV6_MTU_DISCOVER = 23
    IPV6_MTU = 24
    IPV6_RECVERR = 25
    IPV6_V6ONLY = 26
    IPV6_JOIN_ANYCAST = 27
    IPV6_LEAVE_ANYCAST = 28

    # Flowlabel
    IPV6_FLOWLABEL_MGR = 32
    IPV6_FLOWINFO_SEND = 33

    IPV6_IPSEC_POLICY = 34
    IPV6_XFRM_POLICY = 35
    IPV6_HDRINCL = 36

    IPV6_RECVPKTINFO = 49
    IPV6_PKTINFO = 50
    IPV6_RECVHOPLIMIT = 51
    IPV6_HOPLIMIT = 52
    IPV6_RECVHOPOPTS = 53
    IPV6_HOPOPTS = 54
    IPV6_RTHDRDSTOPTS = 55
    IPV6_RECVRTHDR = 56
    IPV6_RTHDR = 57
    IPV6_RECVDSTOPTS = 58
    IPV6_DSTOPTS = 59
    IPV6_RECVPATHMTU = 60
    IPV6_PATHMTU = 61
    IPV6_DONTFRAG = 62

    IPV6_RECVTCLASS = 66
    IPV6_TCLASS = 67

    IPV6_AUTOFLOWLABEL = 70
    # RFC5014: Source address selection
    IPV6_ADDR_PREFERENCES = 72

    # RFC5082: Generalized Ttl Security Mechanism
    IPV6_MINHOPCOUNT = 73

    IPV6_ORIGDSTADDR = 74
    IPV6_TRANSPARENT = 75
    IPV6_UNICAST_IF = 76
    IPV6_RECVFRAGSIZE = 77
    IPV6_FREEBIND = 78

class PRCTL(enum.IntEnum):
    """
    Process Control Types:
    https://github.com/torvalds/linux/blob/master/include/uapi/linux/prctl.h
    """

    PR_SET_PDEATHSIG = 1
    PR_GET_PDEATHSIG = 2
    PR_GET_DUMPABLE = 3
    PR_SET_DUMPABLE = 4
    PR_GET_UNALIGN = 5
    PR_SET_UNALIGN = 6
    PR_UNALIGN_NOPRINT = 1
    PR_UNALIGN_SIGBUS = 2
    PR_GET_KEEPCAPS = 7
    PR_SET_KEEPCAPS = 8
    PR_GET_FPEMU = 9
    PR_SET_FPEMU = 10
    PR_FPEMU_NOPRINT = 1
    PR_FPEMU_SIGFPE = 2
    PR_GET_FPEXC = 11
    PR_SET_FPEXC = 12
    PR_FP_EXC_SW_ENABLE = 0x80
    PR_FP_EXC_DIV = 0x010000
    PR_FP_EXC_OVF = 0x020000
    PR_FP_EXC_UND = 0x040000
    PR_FP_EXC_RES = 0x080000
    PR_FP_EXC_INV = 0x100000
    PR_FP_EXC_DISABLED = 0
    PR_FP_EXC_NONRECOV = 1
    PR_FP_EXC_ASYNC = 2
    PR_FP_EXC_PRECISE = 3
    PR_GET_TIMING = 13
    PR_SET_TIMING = 14
    PR_TIMING_STATISTICAL = 0
    PR_TIMING_TIMESTAMP = 1
    PR_SET_NAME = 15
    PR_GET_NAME = 16
    PR_GET_ENDIAN = 19
    PR_SET_ENDIAN = 20
    PR_ENDIAN_BIG = 0
    PR_ENDIAN_LITTLE = 1
    PR_ENDIAN_PPC_LITTLE = 2
    PR_GET_SECCOMP = 21
    PR_SET_SECCOMP = 22
    PR_CAPBSET_READ = 23
    PR_CAPBSET_DROP = 24
    PR_GET_TSC = 25
    PR_SET_TSC = 26
    PR_TSC_ENABLE = 1
    PR_TSC_SIGSEGV = 2
    PR_GET_SECUREBITS = 27
    PR_SET_SECUREBITS = 28
    PR_SET_TIMERSLACK = 29
    PR_GET_TIMERSLACK = 30
    PR_TASK_PERF_EVENTS_DISABLE = 31
    PR_TASK_PERF_EVENTS_ENABLE = 32
    PR_MCE_KILL = 33
    PR_MCE_KILL_CLEAR = 0
    PR_MCE_KILL_SET = 1
    PR_MCE_KILL_LATE = 0
    PR_MCE_KILL_EARLY = 1
    PR_MCE_KILL_DEFAULT = 2
    PR_MCE_KILL_GET = 34
    PR_SET_MM = 35
    PR_SET_MM_START_CODE = 1
    PR_SET_MM_END_CODE = 2
    PR_SET_MM_START_DATA = 3
    PR_SET_MM_END_DATA = 4
    PR_SET_MM_START_STACK = 5
    PR_SET_MM_START_BRK = 6
    PR_SET_MM_BRK = 7
    PR_SET_MM_ARG_START = 8
    PR_SET_MM_ARG_END = 9
    PR_SET_MM_ENV_START = 10
    PR_SET_MM_ENV_END = 11
    PR_SET_MM_AUXV = 12
    PR_SET_MM_EXE_FILE = 13
    PR_SET_MM_MAP = 14
    PR_SET_MM_MAP_SIZE = 15
    PR_SET_PTRACER = 0x59616D61
    # PR_SET_PTRACER_ANY          = ((unsigned long)-1)
    PR_SET_CHILD_SUBREAPER = 36
    PR_GET_CHILD_SUBREAPER = 37
    PR_SET_NO_NEW_PRIVS = 38
    PR_GET_NO_NEW_PRIVS = 39
    PR_GET_TID_ADDRESS = 40
    PR_SET_THP_DISABLE = 41
    PR_GET_THP_DISABLE = 42
    PR_MPX_ENABLE_MANAGEMENT = 43
    PR_MPX_DISABLE_MANAGEMENT = 44
    PR_SET_FP_MODE = 45
    PR_GET_FP_MODE = 46
    PR_FP_MODE_FR = 1
    PR_FP_MODE_FRE = 2
    PR_CAP_AMBIENT = 47
    PR_CAP_AMBIENT_IS_SET = 1
    PR_CAP_AMBIENT_RAISE = 2
    PR_CAP_AMBIENT_LOWER = 3
    PR_CAP_AMBIENT_CLEAR_ALL = 4
    PR_SVE_SET_VL = 50
    PR_SVE_SET_VL_ONEXEC = 0x40000
    PR_SVE_GET_VL = 51
    PR_SVE_VL_LEN_MASK = 0xFFFF
    PR_SVE_VL_INHERIT = 0x20000
    PR_GET_SPECULATION_CTRL = 52
    PR_SET_SPECULATION_CTRL = 53
    PR_SPEC_STORE_BYPASS = 0
    PR_SPEC_INDIRECT_BRANCH = 1
    PR_SPEC_NOT_AFFECTED = 0
    PR_SPEC_PRCTL = 0x00000001
    PR_SPEC_ENABLE = 0x00000002
    PR_SPEC_DISABLE = 0x00000004
    PR_SPEC_FORCE_DISABLE = 0x00000008
    PR_SPEC_DISABLE_NOEXEC = 0x00000010
    PR_PAC_RESET_KEYS = 54
    PR_PAC_APIAKEY = 0x00000001
    PR_PAC_APIBKEY = 0x00000002
    PR_PAC_APDAKEY = 0x00000004
    PR_PAC_APDBKEY = 0x00000008
    PR_PAC_APGAKEY = 0x00000010

class SIGINFO(ctypes.Structure):
    _fields_ = [
        ("si_signo", ctypes.c_int32),
        ("si_errno", ctypes.c_int32),
        ("si_code", ctypes.c_int32),
        ("si_trapno", ctypes.c_int32),
        ("si_pid", ctypes.c_int32),  # pid_t
    ]

"""
('sin6_family', ctypes.c_ushort),
('sin6_port', ctypes.c_ushort),
('sin6_flowinfo', ctypes.c_uint32),
('sin6_addr', ctypes.c_uint64),
('sin6_scope_id', ctypes.c_uint32),
"""

class SIGAction(enum.IntEnum):
    SI_USER = 0
    SI_KERNEL = 0x80
    SI_QUEUE = -1
    SI_TIMER = -2
    SI_MESGQ = -3
    SI_ASYNCIO = -4
    SI_SIGIO = -5
    SI_TKILL = -6
    SI_DETHREAD = -7
    SI_ASYNCNL = -60
    ILL_ILLOPC = 1
    ILL_ILLOPN = 2
    ILL_ILLADR = 3
    ILL_ILLTRP = 4
    ILL_PRVOPC = 5
    ILL_PRVREG = 6
    ILL_COPROC = 7
    ILL_BADSTK = 8
    ILL_BADIADDR = 9
    __ILL_BREAK = 10
    __ILL_BNDMOD = 11
    NSIGILL = 11
    FPE_INTDIV = 1
    FPE_INTOVF = 2
    FPE_FLTDIV = 3
    FPE_FLTOVF = 4
    FPE_FLTUND = 5
    FPE_FLTRES = 6
    FPE_FLTINV = 7
    FPE_FLTSUB = 8
    __FPE_DECOVF = 9
    __FPE_DECDIV = 10
    __FPE_DECERR = 11
    __FPE_INVASC = 12
    __FPE_INVDEC = 13
    FPE_FLTUNK = 14
    FPE_CONDTRAP = 15
    NSIGFPE = 15
    SEGV_MAPERR = 1
    SEGV_ACCERR = 2
    SEGV_BNDERR = 3
    SEGV_PKUERR = 4
    SEGV_ACCADI = 5
    SEGV_ADIDERR = 6
    SEGV_ADIPERR = 7
    NSIGSEGV = 7
    BUS_ADRALN = 1
    BUS_ADRERR = 2
    BUS_OBJERR = 3
    BUS_MCEERR_AR = 4
    BUS_MCEERR_AO = 5
    NSIGBUS = 5
    TRAP_BRKPT = 1
    TRAP_TRACE = 2
    TRAP_BRANCH = 3
    TRAP_HWBKPT = 4
    TRAP_UNK = 5
    NSIGTRAP = 5
    CLD_EXITED = 1
    CLD_KILLED = 2
    CLD_DUMPED = 3
    CLD_TRAPPED = 4
    CLD_STOPPED = 5
    CLD_CONTINUED = 6
    NSIGCHLD = 6
    POLL_IN = 1
    POLL_OUT = 2
    POLL_MSG = 3
    POLL_ERR = 4
    POLL_PRI = 5
    POLL_HUP = 6
    NSIGPOLL = 6
    SYS_SECCOMP = 1
    NSIGSYS = 1
    EMT_TAGOVF = 1
    NSIGEMT = 1
    SIGEV_SIGNAL = 0
    SIGEV_NONE = 1
    SIGEV_THREAD = 2
    SIGEV_THREAD_ID = 4


import torch
from torch import nn

class Simple(nn.Module):

    '''
    A simple image2image CNN.
    '''
    def __init__(self):
        # This line is very important!
        super().__init__()

        self.conv1 = nn.Conv2d(3, 64, 3)
        self.relu1 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(64, 64, 3)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv3 = nn.Conv2d(64, 3, 3)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        '''
        SimpleNet forward function.

        Args:
            x (B x 3 x H x W Tensor): An input image

        Return:
            (B x 3 x H x W Tensor): An output image
        '''
        # Autograd will keep the history.
        x = self.conv1(x)
        x = self.relu1(x)
        x = self.conv2(x)
        x = self.relu2(x)
        x = self.conv3(x)
        return x


from fastapi import FastAPI
from DoItList.Users.router import router as users_router
from DoItList.Items.router import router as items_router

app = FastAPI(title='ToDo App')
from fastapi.middleware.cors import CORSMiddleware

app.include_router(users_router)
app.include_router(items_router)

origins = ['http://localhost:3000',
           'http://localhost',
           'http://localhost:8080']
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*'],
)


import sqlite3
from sqlite3 import Error

def create_connection(file):
    conn = None
    try:
        conn = sqlite3.connect(file)
        print(sqlite3.version)
    except Error as e:
        print(e)
    return conn

def create_table(conn, sql):
    try:
        c = conn.cursor()
        c.execute(sql)
    except Error as e:
        print(e)

def create_database(file):
    sql_offers_table = """ CREATE TABLE IF NOT EXISTS goods (
                            id integer PRIMARY KEY,
                            modified_time text NOT NULL,
                            categoryId text NOT NULL,
                            vendor text NOT NULL,
                            name text NOT NULL,
                            description text NOT NULL,
                            param text NOT NULL,
                            picture text NOT NULL,
                            oldprice text NOT NULL,
                            price text NOT NULL,
                            url text NOT NULL
                        ); """

    conn = create_connection(file)
    if conn is not None:
        create_table(conn, sql_offers_table)
    else:
        print('Ошибка! Не получается создать подключение к Базе данных!')

def add_offer(conn, row):
    sql = """INSERT INTO goods(modified_time, categoryId, vendor, name, description, param, picture, oldprice, price,
            url) VALUES(?,?,?,?,?,?,?,?,?,?) """
    cur = conn.cursor()
    cur.execute(sql, row)
    conn.commit()
    return cur.lastrowid

if __name__ == '__main__':
    # database = r'database.db'
    # create_database(database)
    pass


import setuptools
import subprocess
import os

with open("README.md", "r") as fh:
    long_description = fh.read()

requirements = [
    'osmium',
    'pyparsing',
    'pyyaml',
    'shapely',
    'requests',
    'landez'
]

setuptools.setup(
    name="osm-export-tool",
    version="0.1.0",
    author="Maxar Technologies",
    author_email="foundationgeoint-packaging@maxar.com",
    description="Convert OpenStreetMap data into GIS and mobile mapping file formats.",
    license="BSD-3-Clause",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/radiant-maxar/osm-export-tool-python",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
    ],
    scripts=['bin/osm-export-tool'],
    install_requires = requirements,
    requires_python='>=3.0',
    package_data={'osm_export_tool':['mappings/*.yml']}
)


# Import libraries flask for web server backend processing, random and json python libraries
from flask import Flask, render_template, jsonify, request, json
from multiprocessing import Process, freeze_support, set_start_method
from random import randint
import json
import multiprocessing as mp
import subprocess

app = Flask(__name__) # Initialise flask app instance

# Function to execute flask application
def runFlask():
    app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=True)

# App route to direct to root, renders index.html (main page)
@app.route("/")
def index():
    return render_template('index.html')

# App route "get data" to receive simulation data for parking spot status updates
@app.route('/_get_data/', methods=['GET', 'POST'])
def get_data():
    tempCoorStore = []

    with open('gdf.json', 'r') as f: # Opens pre-process json file which stores coordinates of parking spaces
        coordinateJson = json.load(f)
        coordinateJsonConverted = json.loads(coordinateJson) # Converts into python dictionary for manipulation
        for items in coordinateJsonConverted['features']:
            tempCoorStore.append(items) # Add dictionary items into list for processing
        for i in range(len(tempCoorStore)):
            tempCoorStore[i]['properties'] = {'avail': str(randint(0,1))} # Modifies key "properties" to dictionary
                                                                          # with "avail" and stringified integer as
                                                                          # value
    f.close()
    finalDict = {"type": "FeatureCollection", "features": tempCoorStore} # Constructs final dictionary fit for
                                                                         # jsonification
    return jsonify({'coordinates': json.dumps(finalDict)}) # Returns jsonified dictionary into frontend JavaScript

if __name__ == "__main__":
    p1 = Process(target=runFlask) # Defines process to start Flask for multiprocessing
    p1.start() # Executes Flask


"""Test DBCP helper functions."""
import pytest
import sqlalchemy as sa

import dbcp

def test_missing_get_db_schema_tables():
    """Test when schema does not exist or doesn't have any tables."""
    engine = dbcp.helpers.get_sql_engine()
    with pytest.raises(ValueError):
        _ = dbcp.helpers.get_db_schema_tables(engine, "i_dont_exist")

class TestBQSchemaHelpers:
    """Test the helper functions that convert sqlalchemy schemas to bigquery schemas."""

    def test_get_schema_sql_alchemy_metadata(self):
        """Ensure we get the expected metadata object."""
        metadata = dbcp.helpers.get_schema_sql_alchemy_metadata("data_warehouse")
        assert isinstance(metadata, sa.MetaData)
        assert "data_warehouse.mcoe" in metadata.tables.keys()

    def test_get_schema_sql_alchemy_metadata_exception(self):
        """Make sure an exception is raised when an invalid schema is requested."""
        with pytest.raises(ValueError):
            _ = dbcp.helpers.get_schema_sql_alchemy_metadata("data_lake")

    def test_get_bq_schema_from_metadata(self):
        """Ensure we get the expected Bigquery schema."""
        expected_bq_schema = [
            {"name": "county_id_fips", "mode": "REQUIRED", "type": "STRING"},
            {"name": "state_id_fips", "mode": "REQUIRED", "type": "STRING"},
            {"name": "county_name", "mode": "REQUIRED", "type": "STRING"},
            {"name": "county_name_long", "mode": "REQUIRED", "type": "STRING"},
            {"name": "functional_status", "mode": "REQUIRED", "type": "STRING"},
            {"name": "land_area_km2", "mode": "REQUIRED", "type": "FLOAT"},
            {"name": "water_area_km2", "mode": "REQUIRED", "type": "FLOAT"},
            {"name": "centroid_latitude", "mode": "REQUIRED", "type": "FLOAT"},
            {"name": "centroid_longitude", "mode": "REQUIRED", "type": "FLOAT"},
            {"name": "raw_tribal_land_frac", "mode": "REQUIRED", "type": "FLOAT"},
            {"name": "tribal_land_frac", "mode": "REQUIRED", "type": "FLOAT"},
        ]

        bq_schema = dbcp.helpers.get_bq_schema_from_metadata(
            "county_fips", "data_warehouse"
        )
        assert bq_schema == expected_bq_schema


###############################################################
################# https://www.fardanesh.ir ####################
###############################################################

from openpyxl import Workbook
from openpyxl.drawing.image import Image,PILImage

wb=Workbook()
ws1=wb.active

if PILImage:
    img=Image('/app/lecture09-tables-images/python-logo.png')
    img.width=200
    img.height=200
    ws1.add_image(img,'B5')

wb.save(filename='lecture09-tables-images/imageAdded.xlsx')


from jobs.jobIngestionProviders import DataIngestion
from pyspark.sql import SparkSession

def get_spark_session():
    spark = (
        SparkSession.builder
        .config("spark.yarn.dist.files", "/home/nan/gecko/code/companies-ingestion/prospect_ingestion/out/prospect_ingestion.pex")
        .config("spark.jars.packages",  "org.apache.hudi:hudi-spark3.1-bundle_2.12:0.12.0")
        .getOrCreate()
    )
    return spark

def main():
    spark = get_spark_session()
    DataIngestion(spark).run()

if __name__ == "__main__":
    main()


from api.factories import ExampleFactory

from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = "Seeds the database."

    def add_arguments(self, parser):
        parser.add_argument(
            "--example",
            default=200,
            type=int,
            help="The number of fake examples to create.",
        )

    def handle(self, *args, **options):
        print("Populating database...")
        for _ in range(options["example"]):
            ExampleFactory()
        print("Done.")


#!/usr/bin/env python3

import sys
sys.path.append('./Anafora/')

import os, configparser
import anafora

def main(xml_dir, text_dir, xml_regex, context_size):
  """Main Driver"""

  for sub_dir, text_name, file_names in anafora.walk(xml_dir, xml_regex):

    xml_path = os.path.join(xml_dir, sub_dir, file_names[0])
    ref_data = anafora.AnaforaData.from_file(xml_path)

    text_path = os.path.join(text_dir, text_name)
    text = open(text_path).read()

    for data in ref_data.annotations.select_type('EVENT'):

      start, end = data.spans[0]
      context = text[start-context_size:end+context_size].replace('\n', '')
      event = text[start:end]
      dtr = data.properties['DocTimeRel']

      left = text[start-context_size:start]
      right = text[end:end+context_size]
      tagged = left + ' [ES] ' + event + ' [EE] ' + right
      print('{}|{}|{}'.format(dtr, event, context))
      print(tagged)
      print()

if __name__ == "__main__":

  cfg = configparser.ConfigParser()
  cfg.read(sys.argv[1])
  base = os.environ['DATA_ROOT']

  xml_dir = os.path.join(base, cfg.get('data', 'train_xml'))
  text_dir = os.path.join(base, cfg.get('data', 'train_text'))
  xml_regex = cfg.get('data', 'xml_regex')
  context_size = 10

  main(xml_dir, text_dir, xml_regex, context_size)


#!/usr/bin/python3

import os
import re
import cv2
import time
import pytesseract
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor

TEMP_DIR = 'temp'
EVIDENCE_DIR = 'evidence'
os.makedirs(TEMP_DIR, exist_ok=True)
os.makedirs(EVIDENCE_DIR, exist_ok=True)

def extract_frames(filepath, file, interval):
    # check if the file is a video file
    _, ext = os.path.splitext(filepath)
    if ext not in ['.mp4', '.avi', '.mkv']:
        return

    # use OpenCV to extract frames
    try:
        cap = cv2.VideoCapture(filepath)
    except:
        print("Failed to load video")

    try:
        fps = int(cap.get(cv2.CAP_PROP_FPS))
    except:
        print("Failed to load video (fps)")

    frame_count = 0
    while True:
        ret, frame = cap.read()

        if not ret:
            break

        if frame_count > 1200: # 10 Minutes every 30 seconds snapshot
           cap.release()
           break

        if frame_count % (fps * interval) == 0:
            try:
                temp_frame = os.path.join(TEMP_DIR, f'inspect_{file}_{frame_count}.jpg')
                temp_content = os.path.join(TEMP_DIR, f'content_{file}_{frame_count}.dat')
                cv2.imwrite(temp_frame, frame)
            except:
                print("Failed to analyze video")
            try:
                data = pytesseract.image_to_string(temp_frame, config='-c tessedit_char_whitelist=abcdefghijklmnopqrstuvwxyz01234567890.://@%?=&,;_- ') # Tune OCR

                #
                # ML model goes here basic
                #

                keywords = {
                            'social': ['snapchat','facebook','tiktok','instagram','framatalk','discord','youtube','omegle'],
                            'gambling':['draftkings','gambling'],
                            'pornography':['porn','pornhub','xhamster','xvidoes','sex'],
                            'games':['roblox','steam','minecraf','now.gg','fortnite'],
                            'crypto':['miner','crypto'],
                            'hacking':['masscan'],
                            'proxies':['proxy','ocks5','browserling','homeworkistrash','vpn'],
                            'software':['downloads','setup','.exe','.zip','github','anonfiles'],
                            'accounts':['password','inbox'],
                            'illegal':['torrent','checker']
                        }

                keystore = []
                for key, values in keywords.items():
                    for value in values:
                        keystore.append([key, value])

                for category,keyword in keystore:

                    fraud = re.search(keyword, data)

                    if fraud is not None:
                        evidence_frame = os.path.join(EVIDENCE_DIR, f'{category}_{file}_{frame_count}.jpg')
                        evidence_content = os.path.join(EVIDENCE_DIR, f'{category}_{file}_{frame_count}.dat')
                        cv2.imwrite(evidence_frame, frame)
                        with open(evidence_content, 'a') as f:
                            f.write(data)

            except:
                print("OCR Reading Error")
        frame_count += 1
    cap.release()

def list_files(directory):
    directory = os.path.abspath(directory)
    for root, dirs, files in os.walk(directory):
        for file in files:
            path = os.path.join(root, file)
            print(path)
            executor = ThreadPoolExecutor(max_workers=4)
            executor.submit(extract_frames,path,file,30)
            time.sleep(1.5) # prevents runway threads tune

list_files("videos/") # Change this to location of video files in mp4, avi, mkv formats


import argparse
import os
import logging
import sys
from typing import List, Generator

# list of file extensions, all lowercase, without leading "."
DEFAULT_SIDECAR_EXTENSIONS = [
    "xmp",
    "pp3"
]

def get_args(args):
    '''Configures command line parser and returns parsed parameters'''
    parser = argparse.ArgumentParser(
        description="""Find sidecar files (like XMP) which don't
        have the associated base file anymore""")
    parser.add_argument(
        "base_folder",
        help="Base folder to recursively search for orphaned sidecar files")
    parser.add_argument(
        "-d", "--debug", dest="debuglevel", action="store_const",
        const=logging.DEBUG, default=logging.INFO,
        help="Enables verbose/debug output")
    parser.add_argument(
        "-e", "--extensions", type=lambda x: x.lower().split(","),
        default=DEFAULT_SIDECAR_EXTENSIONS,
        help="Comma seperated list of sidecar file extensions. Defaults to \"{}\"".format(
            ",".join(DEFAULT_SIDECAR_EXTENSIONS)))
    return parser.parse_args(args)

def get_orphaned_files(files_in_folder: List[str], sidecar_extensions: List[str]) -> List[str]:
    '''Returns orphaned files from a specific folder

    In the following example both abc.def.xyz and abc.def are considered as sidecar
    files for abc.def:

    abc.def
    abc.def.xyz
    abc.xyz

    Arguments:

    files_in_folder: List of filenames in folders
    sidecar_extensions: List of sidecar file extensions

    Returns:

    List of orphaned files
    '''

    result = []
    sidecar_files = [] # list of sidecar only files
    orig_files_base = [] # original file names, no sidecar files

    for entry in files_in_folder:
        base, ext = os.path.splitext(entry)
        if ext.lower().lstrip(".") in sidecar_extensions:
            sidecar_files.append(entry)
        else:
            # Match for abc.def
            orig_files_base.append(entry)
            # Match for abc.xyz
            orig_files_base.append(base)

    for entry in sidecar_files:
        base, ext = os.path.splitext(entry)
        if not base in orig_files_base:
            result.append(entry)

    return result

def find_files(base_folder: str, sidecar_extensions) -> Generator[str, None, None]:
    '''Find sidecar files

    Arguments:

    base_folder: The base folder where to start the recursive search
    sidecar_extensions: List of sidecar file extensions

    Returns a list of duplicates (yield)
    '''

    for root, _, files in os.walk(base_folder, followlinks=True):
        if files:
            logging.debug("Checking %s with %d files", root, len(files))
            orphaned_files = get_orphaned_files(files, sidecar_extensions)
            if orphaned_files:
                for file_ in orphaned_files:
                    yield os.path.join(root, file_)

def main():
    '''main function, called when script file is executed directly'''
    args = get_args(sys.argv[1:])
    logging.basicConfig(format="%(message)s", level=args.debuglevel)
    for entry in find_files(args.base_folder, args.extensions):
        print(entry)

if __name__ == "__main__":
    main()


# coding: utf-8

# In[ ]:

import sys
sys.path.append('/Volumes/SANDISK128/Documents/Thesis/Python/')
import numpy as np
import matplotlib.pyplot as plt

import createFolder
import weather_map_plot_fct as PF
import ECMWF_grb as grib
import calc_date

# In[ ]:

year = '2016'
mon = '12'
#day = '10'
#time = '00'

# In[ ]:

pathPV2000 = '../../NORSTORE/SCA/pv2000'
pathPL = '../../NORSTORE/SCA/pl'
pathSFC = '../../NORSTORE/SCA/sfc'

# In[ ]:

directory_DT = '../../synoptic_figs/DynTropo'
directory_AR = '../../synoptic_figs/Atm_Riv'
directory_IWV = '../../synoptic_figs/TCWV_U850'
directory_JTM = '../../synoptic_figs/Geopot_Jet'
createFolder.createFolder(directory_DT)
createFolder.createFolder(directory_AR)
createFolder.createFolder(directory_IWV)
createFolder.createFolder(directory_JTM)

# In[ ]:

form = 'png'
savefigure = 1

# In[ ]:

for day in range(23,29):
    day = str(day)
    for time in range(0,24,6):
        if time ==0:
            time = '00'
        elif time == 6:
            time = '06'
        time = str(time)

### Dates for plotting
        calday, calmon = calc_date.get_dayname(year, mon, day)
        figure_name = '%s%s%s_%s.png' % (year, mon, day,time)

# DT
        grbPT, PT, grbUV_DT, wind_u_DT, wind_v_DT, grbVO, rel_vort = grib.DT_values(pathPV2000, pathPL,
                                                                                year,mon,day,time)
# AR
        grbUV_AR, IVT, IVT_u, IVT_v = grib.AR_values(pathPL, pathSFC, year,mon,day,time)

# IWV - U850
        grbUV_IWV, wind_u_IWV, wind_v_IWV, grbMSL_IWV, MSL_IWV, grbPW_IWV, PW_IWV = grib.IWV_U850_values(pathPL,
                                                                                pathSFC, year,mon,day,time)

# JTHMSLP
        grbUV_JTM, Uabs, grbZ, Z, grbMSL_JTM, MSL_JTM, grbPW_JTM, PW_JTM = grib.JTHMSLP_values(pathPL, pathSFC,
                                                                                year,mon,day,time)

# Plotting data on a map (Example Gallery) https://matplotlib.org/basemap/users/examples.html
        m = PF.create_basemap()

### Latitudes, Longitudes and shiftgrid

# DT
        lonsPT,latsPT, PT = grib.shiftgrb(grbPT,PT,m)
        lonsVO,latsVO, RelVo = grib.shiftgrb(grbVO,rel_vort,m)
        lonsU_DT,latsU_DT, wind_u_DT = grib.shiftgrb(grbUV_DT,wind_u_DT,m)
        lonsV_DT,latsV_DT, wind_v_DT = grib.shiftgrb(grbUV_DT,wind_v_DT,m)

# AR
        lonsIVT,latsIVT, IVT = grib.shiftgrb(grbUV_AR,IVT,m)
        lonsIU,latsIU, IVT_u = grib.shiftgrb(grbUV_AR,IVT_u,m)
        lonsIV,latsIV, IVT_v = grib.shiftgrb(grbUV_AR,IVT_v,m)
# find values larger 250
        lon_flux, lat_flux, u_flux, v_flux = grib.larger250(IVT, IVT_u, IVT_v, lonsIU, latsIV)

# IWV - U850
        lonsU_IWV,latsU_IWV, wind_u_IWV = grib.shiftgrb(grbUV_IWV,wind_u_IWV, m)
        lonsV_IWV,latsV_IWV, wind_v_IWV = grib.shiftgrb(grbUV_IWV,wind_v_IWV, m)
        lonsMSL_IWV,latsMSL_IWV, MSL_IWV = grib.shiftgrb(grbMSL_IWV,MSL_IWV, m)
        lonsPW_IWV,latsPW_IWV, PW_IWV = grib.shiftgrb(grbPW_IWV,PW_IWV, m)

# JTHMSLP
        lonsUV_JTM,latsUV_JTM, Uabs = grib.shiftgrb(grbUV_JTM,Uabs,m)
        lonsZ,latsZ, Z = grib.shiftgrb(grbZ,Z,m)
        lonsMSL_JTM,latsMSL_JTM, MSL_JTM = grib.shiftgrb(grbMSL_JTM,MSL_JTM,m)
        lonsPW_JTM,latsPW_JTM, PW_JTM = grib.shiftgrb(grbPW_JTM,PW_JTM,m)

### Plotting Map
# DT
        PF.plot_DTmap(m, lonsPT, latsPT, PT,
               lonsVO, latsVO, RelVo,
               lonsU_DT, latsU_DT, wind_u_DT, wind_v_DT,
               savefigure, directory_DT, figure_name, form,
               calday, day, calmon, year, time)

        plt.close()

# AR
        m = PF.create_basemap()
        PF.plot_ARmap(m, lonsIVT, latsIVT, IVT,
              lon_flux, lat_flux, u_flux, v_flux,
              savefigure, directory_AR, figure_name, form,
               calday, day, calmon, year, time)

        plt.close()

# IWV - U850

        m = PF.create_basemap()
        PF.plot_IWV_U850map(m, lonsPW_IWV, latsPW_IWV, PW_IWV,
              lonsMSL_IWV, latsMSL_IWV, MSL_IWV,
              lonsU_IWV, latsU_IWV, wind_u_IWV, wind_v_IWV,
              savefigure, directory_IWV, figure_name, form,
               calday, day, calmon, year, time)

        plt.close()

# Jet - Thickness - MSLP

        m = PF.create_basemap()
        PF.plot_JTHMSPL(m, lonsPW_JTM, latsPW_JTM, PW_JTM,
              lonsMSL_JTM, latsMSL_JTM, MSL_JTM,
              lonsUV_JTM, latsUV_JTM, Uabs,
              lonsZ,latsZ, Z,
              savefigure, directory_JTM, figure_name, form,
               calday, day, calmon, year, time)

        plt.close()

        print('figure: %s saved' % (figure_name))

# In[ ]:


import gymnasium as gym
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F

from tqdm import tqdm, trange

device = 'cpu'

def make_env(env_id, seed, idx, run_name):
	def thunk():
		env = gym.make(env_id)
		env.action_space.seed(seed)
		env.observation_space.seed(seed)
		return env
	return thunk()

hidden_dim = 1024

def layer_init(layer, std=np.sqrt(2), bias_const=0.0):
	nn.init.orthogonal_(layer.weight, std)
	nn.init.constant_(layer.bias, bias_const)
	return layer

class Agent(nn.Module):
	def __init__(self, envs):
		super().__init__()

		self.critic = nn.Sequential(
			layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, 1), std=1.0)
		)

		self.actor_mean = nn.Sequential(
			layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, hidden_dim)),
			nn.Tanh(),
			layer_init(nn.Linear(hidden_dim, np.array(envs.single_action_space.shape).prod()), std=0.01)
		)
		self.actor_logstd = nn.Parameter(torch.zeros(1, np.prod(envs.single_action_space.shape)))

	def get_value(self, x):
		return self.critic(x)

	def get_action_and_value(self, x, pre_action=None):
		pass

envs = gym.vector.SyncVectorEnv([
    lambda: make_env('Pusher-v4', 0+i, i, 'the first') for i in range(4)
])

agent = Agent(envs).to(device)
# print(agent.load_state_dict(torch.load('models/pusher_ppo/final.pth')))
agent.load_state_dict(torch.load('models/pusher_ppo/1408.pth'))

env = gym.make('Pusher-v4', render_mode='human')
state, info = env.reset()

loop = tqdm()
while True:
# for _ in range(512):
	with torch.no_grad():
		action = agent.actor_mean(torch.from_numpy(state).float().unsqueeze(0).to(device))
		# action = agent.actor_mean(action)
		# action, _, _, _ = agent.get_action_and_value(torch.from_numpy(state).float().to(device))
	state, reward, terminated, truncated, _ = env.step(action[0].cpu().numpy())

	loop.set_postfix(reward=reward)
	if terminated or truncated:
		state, info = env.reset()


import io
import json
from api.sc2_extract import sc2_extract
from http.server import BaseHTTPRequestHandler
from urllib.request import urlopen

class Encoder(json.JSONEncoder):
  def default(self, obj):
    if isinstance(obj, bytes):
      try:
        return obj.decode('utf-8')
      except:
        return ''
    return json.JSONEncoder.default(self, obj)

class handler(BaseHTTPRequestHandler):
  def do_GET(self):
    if (self.path[:8] == '/api/gg-'): url = f'http://ggtracker.com/matches/{self.path[8:]}/replay'
    if (self.path[:8] == '/api/rs-'): url = f'http://sc2replaystats.com/download/{self.path[8:]}'
    if (self.path[:8] == '/api/st-'): url = f'http://lotv.spawningtool.com/{self.path[8:]}/download/'
    raw = sc2_extract(io.BytesIO(urlopen(url).read()))
    del raw['gameevents'] # HACK: Need to find a wait around zeit 5m content size limit
    self.send_response(200)
    self.send_header('Access-Control-Allow-Origin', '*')
    self.send_header('Access-Control-Request-Method', '*')
    self.send_header('Access-Control-Allow-Methods', 'OPTIONS, GET')
    self.send_header('Access-Control-Allow-Headers', '*')
    self.send_header('Cache-Control', 's-maxage=31536000')
    self.send_header('Content-Type', 'application/json')
    self.end_headers()
    self.wfile.write(json.dumps(raw, cls=Encoder, indent=2, sort_keys=True).encode())
    return


# rewrote the module from @ Fl1yd
from pyrogram.types import Message
from pyrogram import Client, filters
from .utils.utils import modules_help, prefix

import requests
import os

def get_pic(city):
    file_name = f'{city}.png'
    with open(file_name, 'wb') as pic:
        response = requests.get('http://wttr.in/{citys}_2&lang=en.png', stream=True)

        if not response.ok:
            print(response)

        for block in response.iter_content(1024):
            if not block:
                break

            pic.write(block)
        return file_name

@Client.on_message(filters.command('weather', prefix) & filters.me)
async def weather(client: Client, message: Message):
    city = message.command[1]
    await message.edit("```Processing the request...```")
    r = requests.get(f"https://wttr.in/{city}?m?M?0?q?T&lang=en")
    await message.edit(f"```City: {r.text}```")
    await client.send_document(chat_id=message.chat.id, document=get_pic(city), reply_to_message_id=message.message_id)
    os.remove(f'{city}.png')

modules_help.update({'weather': '''weather [city] - Get the weather in the selected city''',
                     'weather module': 'Weather: weather'})


from __future__ import annotations
from dataclasses import dataclass
from dataclasses import field
from urllib.parse import urlsplit
from typing import Any, Optional, TypedDict
from lib.types import Headers
from entities.payload_file import PayloadFile, PayloadFileSerialised
from entities.model import Model

from constants import CALCULATED_TEXT
from lib.types import Headers
from lib.input_parsing.text_wrapper import parse_text, parse_text_with_payload_values
from copy import deepcopy
from mitmproxy.common_types import ProxyRequest

def escape_quotes(value: str) -> str:
    return value.replace('"', r'\"')

class FuzzFormData(TypedDict):
    payload_files: list[PayloadFileSerialised]
    fuzz_type: str
    delay_type: str
    delay_secs: Optional[str]
    delay_secs_min: Optional[str]
    delay_secs_max: Optional[str]

class FormData(TypedDict):
    method: str
    url: str
    headers: Headers
    content: str
    fuzz_data: Optional[FuzzFormData]

@dataclass(kw_only=True)
class HttpRequest(Model):
    # Columns
    id: int = field(init=False, default=0)
    created_at: int = field(init=False, default=0)

    http_version: str
    headers: Headers
    content: Optional[str] = None
    trailers: Optional[str] = None
    timestamp_start: Optional[float] = None
    timestamp_end: Optional[float] = None
    host: str
    port: int
    method: str
    scheme: str
    authority: Optional[str] = None
    path: str
    form_data: FormData

    # Relations

    meta = {
        "relationship_keys": [],
        "json_columns": ["headers", "form_data"],
        "do_not_save_keys": [],
    }

    FUZZ_TYPE_LABELS = ['One To One', 'Cartesian Product']
    FUZZ_TYPE_KEYS = ['one_to_one', 'cartesian']

    DELAY_TYPE_LABELS = ['Disabled', 'Fixed Time Delay', 'Random Time Delay']
    DELAY_TYPE_KEYS = ['disabled', 'fixed', 'random']

    # This is how requests are received from the proxy
    @classmethod
    def from_state(cls, state: ProxyRequest) -> HttpRequest:
        request = HttpRequest(
            http_version = state['http_version'],
            headers = dict(state['headers']),
            content = state['content'],
            trailers = state['trailers'],
            timestamp_start = state['timestamp_start'],
            timestamp_end = state['timestamp_end'],
            host = state['host'],
            port = state['port'],
            method = state['method'],
            scheme = state['scheme'],
            authority = state['authority'],
            path = state['path'],
            form_data = {
                'method': state['method'],
                'url': '',
                'headers': dict(state['headers']),
                'content': state['content'],
                'fuzz_data': None
            },
        )

        request.form_data['url'] = request.get_url()

        return request

    # TODO: Use a TypedDict instead of Any
    # TODO: Make this work
    def get_state(self) -> dict[str, Any]:
        # attributes = self.serialize()
        # attributes['headers'] = json.loads(attributes['headers'])
        return {}

    @classmethod
    def build_blank_for_editor(cls):
        return HttpRequest(
            http_version = 'HTTP/1.1',
            headers = {},
            host = '',
            port = 80,
            method = 'GET',
            scheme = 'http',
            path = '',
            content = '',
            form_data = {'method': 'GET', 'url': 'http://', 'headers': {}, 'content': '', 'fuzz_data': None},
        )

    @classmethod
    def build_blank_placeholder(cls):
        return HttpRequest(
            http_version = 'HTTP/1.1',
            headers = {},
            host = '',
            port = 80,
            method = 'GET',
            scheme = 'http',
            path = '',
            content = '',
            form_data = {'method': 'GET', 'url': 'http://', 'headers': {}, 'content': '', 'fuzz_data': None},
        )

    def set_headers(self, headers: Headers) -> None:
        self.headers = headers

    def get_headers(self) -> Headers:
        return self.headers

    def get_header_line_no_http_version(self) -> str:
        return f'{self.method} {self.path}'

    def get_header_line(self) -> str:
        return f'{self.method} {self.path} {self.http_version}'

    def get_url(self) -> str:
        if self.port not in [80, 443, None]:
            port = ':' + str(self.port)
        else:
            port = ''

        return f'{self.scheme}://{self.host}{port}{self.path}'

    def duplicate(self) -> HttpRequest:
        return HttpRequest(
            http_version = self.http_version,
            headers = self.headers,
            content = self.content,
            trailers = self.trailers,
            host = self.host,
            port = self.port,
            method = self.method,
            scheme = self.scheme,
            authority = self.authority,
            path = self.path,
            form_data = self.form_data or self.generate_form_data(),
        )

    def modify(self, modified_method: str, modified_path: str, modified_headers: Headers, modified_content: str):
        self.method = modified_method
        self.path = modified_path
        self.headers = modified_headers
        self.content = modified_content

        if modified_headers.get('Host'):
            host, port = self.__get_host_and_port(modified_headers['Host'])
            self.host = host
            if port:
                self.port = port
        elif modified_headers.get('host'):
            # TODO: DRY up this duplication
            host, port = self.__get_host_and_port(modified_headers['Host'])
            self.host = host
            if port:
                self.port = port

    def overwrite_calculated_headers(self) -> None:
        calc_text = CALCULATED_TEXT
        headers = self.get_headers()

        if headers is None:
            return

        if headers.get('Host'):
            headers['Host'] = calc_text

        if headers.get('Content-Length'):
            headers['Content-Length'] = calc_text

        self.set_headers(headers)

    def apply_payload_values(self, payload_values: dict[str, str]) -> None:
        method = parse_text_with_payload_values(str(self.form_data['method']), payload_values)
        url = parse_text_with_payload_values(str(self.form_data['url']), payload_values)
        content = parse_text_with_payload_values(str(self.form_data['content']), payload_values)

        # TODO:
        headers = self.form_data['headers']

        self.set_form_data({
            'method': method,
            'url': url,
            'headers': headers,
            'content': content,
            'fuzz_data': None
        })
        return

    def reset_form_data(self) -> None:
        self.set_form_data(self.form_data)

    def set_form_data(self, form_data: FormData) -> None:
        self.form_data = form_data

        # 1. Set method
        self.method = str(self.form_data['method'])

        # 2. Set URL related attributes
        parsed_url = parse_text(str(self.form_data['url']))
        url_data = urlsplit(parsed_url)
        if url_data.hostname:
            self.host = url_data.hostname

        if url_data.port:
            self.port = url_data.port
        else:
            if url_data.scheme == "https":
                self.port = 443
            else:
                self.port = 80

        self.scheme = url_data.scheme

        if url_data.query == '':
            self.path = url_data.path
        else:
            self.path = url_data.path + '?' + url_data.query

        # 3. Set content
        self.content = parse_text(str(form_data['content']))

        # 4. Set headers
        parsed_headers = deepcopy(form_data['headers'])
        for key, value in form_data['headers'].items():
            parsed_headers[key] = parse_text(value)

        self.set_headers(parsed_headers)

    def generate_form_data(self) -> FormData:
        headers = self.get_headers() or {}
        content = self.content or ''

        return {
            'method': self.method,
            'url': self.get_url(),
            'headers': headers,
            'content': content,
            'fuzz_data': None
        }

    def fuzz_data(self) -> Optional[FuzzFormData]:
        return self.form_data.get('fuzz_data')

    def payload_files(self) -> list[PayloadFile]:
        fuzz_data = self.fuzz_data()

        if fuzz_data is None:
            return []

        return [PayloadFile.from_serialised(p) for p in fuzz_data['payload_files']]

    def payload_keys(self) -> list[str]:
        fuzz_data = self.fuzz_data()

        if fuzz_data is None:
            return []

        return [p['key'] for p in fuzz_data['payload_files']]

    def get_curl_command(self) -> str:
        headers_dict = self.get_headers()
        headers = ""

        if headers_dict is not None:
            headers = ['"{0}: {1}"'.format(k, escape_quotes(v)) for k, v in headers_dict.items() if v != CALCULATED_TEXT]
            headers = " -H ".join(headers)

        content = getattr(self, 'content', None)
        if content is None or content == "":
            return f"curl -X {self.method} -H {headers} \"{self.get_url()}\""
        else:
            return f"curl -X {self.method} -H {headers} -d {content}' {self.get_url()}"

    def __eq__(self, other) -> bool:
        return (
            self.method == other.method and
            self.host == other.host and
            self.port == other.port and
            self.path == other.path and
            self.headers == other.headers and
            self.content == other.content
        )

    def __get_host_and_port(self, host: str) -> tuple[str, Optional[int]]:
        host_arr = host.split(':')
        if len(host_arr) == 2:
            return (host_arr[0], int(host_arr[1]))
        else:
            return [host_arr[0], None] # type: ignore


from __future__ import print_function
import datetime
import os
import pymongo
import pandas as pd
import math
import shapely
import geopandas as gpd
import rasterio
import rasterio.features
import rasterio.enums
import PIL.Image
import numpy as np
from geopy.distance import great_circle
import matplotlib
import rasterstats

matplotlib.use('PS')

def save_image(raster, name):
    img = PIL.Image.fromarray(
        np.array(255 * raster / raster.max(),
        dtype=np.uint8))
    img.save(name + ".png", "PNG")

world_df = gpd.read_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), "world.geo.json"))
# Use cylindrical equal-area projection
# https://gis.stackexchange.com/questions/218450/getting-polygon-areas-using-geopandas
world_df = world_df.to_crs({'proj': 'cea'})

def get_airport_to_country_code(db):
    nameToISOs = {
      'North Korea': 'KP',
      'South Korea': 'KR',
      'United States Minor Outlying Islands': 'US',
      'Macau': 'MO',
      'Reunion': 'RE',
      'Christmas Island': 'CX',
      'Guadeloupe': 'GP',
      'Ivory Coast (Cote d\'Ivoire)': 'CI',
      'French Guiana': 'GF',
      'Western Samoa': 'WS',
      'Saint Vincent and Grenadines': 'VC',
      'Guinea Bissau': 'GW',
      'Cocos (Keeling) Islands': 'CC',
      'Grenada and South Grenadines': 'GD',
      'Mayotte': 'YT',
      'Martinique': 'MQ',
      'Tuvalu': 'TV',
      'Gibraltar': 'GI',
      'Bonaire, Saint Eustatius & Saba': None,
      'Palestinian Territory': None,
      'Curacao': None,
      'Unknown Country': None
    }
    name_props = [
      'name',
      'name_long',
      'formal_en',
      'name_alt',
      'name_sort',
      'formal_en',
      'brk_name'
    ]
    for idx, row in world_df.iterrows():
        for prop in name_props:
            value = row[prop]
            if value:
                nameToISOs[value] = row.iso_a2
    airport_to_country_code = {
        airport['_id']: nameToISOs[airport['countryName']]
        for airport in db.airports.find({})
    }
    return airport_to_country_code

def plot_airport(long_lat, rasterio_handle, out_raster, magnitude):
    """
    Plot the airport catchment weight function at the given
    longitude and latitude on the out_raster array
    using the transform specified by the rasterio_handle.
    The weight function has a zero to one value which is multiplied by
    the magnitude parameter.
    """
    # Pixel distance around airport where catchment portions will be computed.
    WINDOW_SIZE = (20, 20)
    # Distance decay functions and parameters from:
    # http://www.tsi.lv/sites/default/files/editor/science/Publikacii/RelStat_09/sess_8_pavlyuk.pdf
    a = -11.13
    b = 2.72
    row, col = np.array(rasterio_handle.index(*long_lat)).astype(int)
    for i in range(row - int(WINDOW_SIZE[0] / 2), row + int(WINDOW_SIZE[0] / 2)):
        for j in range(col - int(WINDOW_SIZE[1] / 2), col + int(WINDOW_SIZE[1] / 2)):
            distance_km = great_circle(
                long_lat[::-1],
                rasterio_handle.xy(i, j)[::-1]).kilometers
            value = magnitude / (1 + math.exp(a + b * math.log(distance_km, math.e)))
            out_raster[i % out_raster.shape[0], j % out_raster.shape[1]] += value

def create_projected_point(lon_lat):
    """
    Create a shaplely geometric point at the given longitude and latitude that
    is projected to cylindrical equal area coordinates.
    """
    return gpd.GeoDataFrame(
        crs={'init':'epsg:4326'},
        geometry=[shapely.geometry.Point(*lon_lat)]
    ).to_crs({'proj':'cea'}).geometry.iloc[0]

def create_location_shapes(resolved_location_tree, parent_shape=None):
    """
    Tranform a resolved value location tree into shapely geometric shapes
    that have values corresponding to their overall resolved value minus
    that of their contained shapes.
    """
    shapes = []
    for child in resolved_location_tree.get('children', []):
        new_shape = None
        shape_value = None
        country_shape = None
        location = child['location']
        matching_countries = world_df[world_df.iso_a2 == location.get('countryCode', 'NoCountry')]
        excess_value = child['value'] - sum(child2['value'] for child2 in child.get('children', []))
        # Excess value could be slightly negative due to floating point error.
        if excess_value < -0.01:
            print(child)
            print([child2['value'] for child2 in child['children']])
            raise Exception('Bad tree')
        excess_value = max(0, excess_value)
        if len(matching_countries):
            country_shape = matching_countries.geometry.iloc[0]
        if location.get('featureCode', '').startswith('PCL') and country_shape:
            new_shape = country_shape
        else:
            # Plot the location as a circle clipped by its containing location.
            lon_lat = location.get('longitude'), location.get('latitude')
            if lon_lat[0] and lon_lat[1]:
                # default area 10 square km
                shape_area = location.get('area', 10e3)
                if len(matching_countries) and 'population' in location:
                    # Set the shape area as a proportion of the country area
                    # equal to the proportion of the country's population it contains.
                    country_portion = float(location['population']) / matching_countries.pop_est.iloc[0]
                    shape_area = max(country_shape.area * country_portion, shape_area)
                new_shape = create_projected_point(lon_lat).buffer(math.sqrt(shape_area / math.pi))
                if parent_shape:
                    new_shape = new_shape.intersection(parent_shape)
                if new_shape.area == 0:
                    new_shape = None
        child_shapes = create_location_shapes(child, new_shape)
        child_shapes_out = []
        for child_shape_val in child_shapes:
            if child_shape_val[0] is None:
                # If the child location has no shape absorb it into the
                # parent location.
                excess_value += child_shape_val[1]
            else:
                child_shapes_out.append(child_shape_val)
        # # Remove overlapping child shape regions from parent shape.
        # if len(child_shapes_out) > 0:
        #     all_child_shape = child_shapes_out[0][0]
        #     for child_shape, value in child_shapes_out[1:]:
        #         all_child_shape = all_child_shape.union(child_shape)
        #     new_shape = new_shape.difference(all_child_shape)
        shapes.append((new_shape, excess_value,))
        shapes += child_shapes_out
    return shapes

def compute_case_raster(resolved_location_tree, population_raster, population_raster_data):
    """
    Convert a resolved value location tree into a raster map where the
    value of each pixel corresponds to the number of cases that occurred
    within it.
    """
    shape_values = create_location_shapes(resolved_location_tree)
    print("Removed cases:", sum([value
        for shape, value in shape_values
        if not(shape and shape.area > 0)]))
    shape_values = [shape_value
        for shape_value in shape_values
        if shape_value[0] and shape_value[0].area > 0]
    projected_shapes = list(gpd.GeoDataFrame(
        crs={'proj':'cea'},
        geometry=[shape for shape, _ in shape_values]).to_crs({
            'init':'epsg:4326'
        }).geometry)
    case_raster = np.zeros(population_raster.shape)
    if len(shape_values) > 0:
        for projected_shape, (_, value) in zip(projected_shapes, shape_values):
            # There a couple options for combining sub-locations with their
            # parents. One is to limit sub-location desitys to the cases within
            # them, replacing the parent location pixels even when
            # they have a greater density.
            # The other is to add the sub-location densitys to the parent location
            # densitys. I think adding makes more sense because data for
            # sub-locations is likely to be more incomplete.
            mask = rasterio.features.rasterize(
                [(projected_shape, 1.0,)],
                out_shape=population_raster.shape,
                transform=population_raster.transform,
                all_touched=True)
            masked_pop = population_raster_data * mask
            case_raster += masked_pop * (value / (0.1 + masked_pop.sum()))
    return case_raster

def compute_outflows(db, match_query):
    """
    Compute the total outbound seats of each airports for flights matching the
    given query.
    """
    result = {}
    for pair in db.flights.aggregate([
        {
            "$match": match_query
        }, {
            '$group': {
                '_id': "$departureAirport",
                'totalSeats': {
                    '$sum': '$totalSeats'
                }
            }
        }
    ]):
        if pair['totalSeats'] > 0:
            result[pair['_id']] = pair['totalSeats']
    return result

if __name__ == "__main__":
    import os
    import requests
    import yaml

    population_raster = rasterio.open('gpw/gpw_v4_population_count_rev10_2015_15_min.tif')
    population_raster_data = population_raster.read(1)
    population_raster_data[population_raster_data < 0] = 0

    end_date = datetime.datetime(2018, 4, 17)
    start_date = datetime.datetime(2018, 3, 29)

    results = requests.get('https://eidr-connect.eha.io/api/events-with-resolved-data', params={
        'ids': ['RJ8oAew6rntCQMuZY'],
        'startDate': start_date.isoformat(),
        'endDate': end_date.isoformat(),
        'eventType': 'auto',
        'fullLocations': True,
        'activeCases': True
    }).json()['events']

    import yaml
    print(yaml.safe_dump(results))
    db = pymongo.MongoClient(os.environ['MONGO_HOST'])['flirt']

    resolved_event_data = results[0]
    case_raster = compute_case_raster(resolved_event_data['fullLocations'], population_raster, population_raster_data)
    print('total cases:', case_raster.sum())
    actual_case_total = sum(child2['value'] for child2 in resolved_event_data['fullLocations']['children'])
    print('error:', case_raster.sum() / actual_case_total)
    save_image(case_raster ** 0.2, "case_raster")

    outflows = compute_outflows(db, {
        "departureDateTime": {
            "$lte": end_date,
            "$gte": start_date
        }
    })
    max_outflow = max(outflows.values())

    all_airport_raster_data = np.zeros(population_raster.shape)
    for airport in list(db.airports.find()):
        magnitude = float(outflows.get(airport['_id'], 0)) / max_outflow
        if magnitude > 0:
            plot_airport(
                airport['loc']['coordinates'],
                population_raster, all_airport_raster_data,
                magnitude)
    save_image(all_airport_raster_data, "all_airport_raster")

    rows = []
    for airport in db.airports.find():
        result = np.zeros(population_raster_data.shape)
        magnitude = float(outflows.get(airport['_id'], 0)) / max_outflow
        if magnitude > 0:
            plot_airport(
                airport['loc']['coordinates'],
                population_raster, result,
                magnitude)
            result[all_airport_raster_data > 0] = result[all_airport_raster_data > 0] / all_airport_raster_data[all_airport_raster_data > 0]
            if airport['_id'] == 'SEA':
                save_image(result, 'SEA_airport_raster')
            airport['catchment_cases'] = (result * case_raster).sum()
            airport['catchment_pop'] = (result * population_raster_data).sum()
            rows.append(airport)

    df = pd.DataFrame(rows)
    df['prob_infected'] = df.catchment_cases / df.catchment_pop
    df = df.query('catchment_pop > 0')
    print(df.sort_values('prob_infected')[['prob_infected', 'name']])


import os
from typing import Any, Dict

import pytest
import yaml

from swh.core.api import RPCServerApp
from swh.core.config import load_from_envvar
from swh.search.api import server
from swh.search.api.server import load_and_check_config, make_app_from_configfile

def teardown_function():
    # Ensure there is no configuration loaded from a previous test
    server.api_cfg = None

def _write_config_file(tmp_path, monkeypatch, content):
    conf_path = os.path.join(str(tmp_path), "search.yml")
    with open(conf_path, "w") as f:
        f.write(yaml.dump(content))
    monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path)
    return conf_path

@pytest.fixture
def swh_search_server_config_without_indexes() -> Dict[str, Any]:
    return {
        "search": {
            "cls": "elasticsearch",
            "hosts": ["es1"],
        }
    }

@pytest.fixture
def swh_search_server_config_with_indexes(
    swh_search_server_config_without_indexes,
) -> Dict[str, Any]:
    return {
        "search": {
            **{"indexes": {"origin": {"index": "test"}}},
            **swh_search_server_config_without_indexes["search"],
        }
    }

@pytest.fixture
def swh_search_config_without_indexes(
    monkeypatch, swh_search_server_config_without_indexes, tmp_path
):
    return _write_config_file(
        tmp_path, monkeypatch, swh_search_server_config_without_indexes
    )

@pytest.fixture
def swh_search_config_with_indexes(
    monkeypatch, swh_search_server_config_with_indexes, tmp_path
):
    return _write_config_file(
        tmp_path, monkeypatch, swh_search_server_config_with_indexes
    )

def prepare_config_file(tmpdir, config_dict: Dict, name: str = "config.yml") -> str:
    """Prepare configuration file in `$tmpdir/name` with content `content`.

    Args:
        tmpdir (LocalPath): root directory
        content: Content of the file either as string or as a dict.
                            If a dict, converts the dict into a yaml string.
        name: configuration filename

    Returns
        path of the configuration file prepared.

    """
    config_path = tmpdir / name
    config_path.write_text(yaml.dump(config_dict), encoding="utf-8")
    # pytest on python3.5 does not support LocalPath manipulation, so
    # convert path to string
    return str(config_path)

@pytest.mark.parametrize("config_file", [None, ""])
def test_load_and_check_config_no_configuration(config_file):
    """Inexistent configuration files raises"""
    with pytest.raises(EnvironmentError, match="Configuration file must be defined"):
        load_and_check_config(config_file)

def test_load_and_check_config_inexistent_file():
    config_path = "/some/inexistent/config.yml"
    expected_error = f"Configuration file {config_path} does not exist"
    with pytest.raises(EnvironmentError, match=expected_error):
        load_and_check_config(config_path)

def test_load_and_check_config_wrong_configuration(tmpdir):
    """Wrong configuration raises"""
    config_path = prepare_config_file(tmpdir, {"something": "useless"})
    with pytest.raises(KeyError, match="Missing 'search' configuration"):
        load_and_check_config(config_path)

def test_load_and_check_config_local_config_fine(
    swh_search_server_config_with_indexes, tmpdir
):
    """'local' complete configuration is fine"""
    config_path = prepare_config_file(tmpdir, swh_search_server_config_with_indexes)
    cfg = load_and_check_config(config_path)
    assert cfg == swh_search_server_config_with_indexes

def test_server_make_app_from_config_file_without_indexes(
    swh_search_config_without_indexes,
):
    app = make_app_from_configfile()
    expected_cfg = load_from_envvar()

    assert app is not None
    assert isinstance(app, RPCServerApp)
    assert app.config["search"] == expected_cfg["search"]

    app2 = make_app_from_configfile()
    assert app is app2

def test_server_make_app_from_config_file_with_indexes(
    swh_search_config_with_indexes,
):
    app = make_app_from_configfile()
    expected_cfg = load_from_envvar()
    assert app is not None
    assert isinstance(app, RPCServerApp)
    assert app.config["search"] == expected_cfg["search"]

    app2 = make_app_from_configfile()
    assert app is app2

def test_server_first_call_initialize_elasticsearch(
    swh_search_config_with_indexes, mocker
):
    """Test the initialize method is called during the first and first only
    request to the server"""
    mock = mocker.patch("swh.search.elasticsearch.ElasticSearch.initialize")

    app = make_app_from_configfile()
    app.config["TESTING"] = True
    tc = app.test_client()

    tc.get("/")
    assert mock.call_count == 1

    tc.get("/")
    assert mock.call_count == 1