from ewah.dwhooks.base_dwhook import EWAHBaseDWHook
from ewah.constants import EWAHConstants as EC

import os
import csv
import snowflake.connector
from tempfile import NamedTemporaryFile
from airflow.utils.file import TemporaryDirectory
from airflow.models import BaseOperator

class SnowflakeOperator(BaseOperator):
    "Operate to execute SQL on Snowflake"

    def __init__(
        self,
        sql,
        snowflake_conn_id,
        database,
        params=None,
    *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.sql = sql
        self.conn_id = snowflake_conn_id
        self.database = database
        self.params = params

    def execute(self, context):
        hook = EWAHDWHookSnowflake(
            conn_id=self.conn_id,
            database=self.database,
        )
        self.log.info('execute: '+self.sql)
        hook.execute(self.sql, commit=True, params=self.params)
        hook.close()

class EWAHDWHookSnowflake(EWAHBaseDWHook):

    _QUERY_SCHEMA_CHANGES_COLUMNS = """
        SELECT
            column_name
        FROM "{database_name}".information_schema.columns
        WHERE table_catalog = %(database_name)s
            AND table_schema = %(schema_name)s
            AND table_name = %(table_name)s
        ORDER BY ordinal_position ASC;
    """
    _QUERY_SCHEMA_CHANGES_DROP_COLUMN = """
        ALTER TABLE "{database_name}"."{schema_name}"."{table_name}"
        DROP COLUMN IF EXISTS "{column_name}" CASCADE;
    """
    _QUERY_SCHEMA_CHANGES_ADD_COLUMN = """
        ALTER TABLE "{database_name}"."{schema_name}"."{table_name}"
        ADD COLUMN "{column_name}" {column_type};
    """
    _QUERY_TABLE = """
        SELECT * FROM "{database_name}"."{schema_name}"."{table_name}"
    """

    def __init__(self, *args, database=None, **kwargs):
        self.database = database
        super().__init__(EC.DWH_ENGINE_SNOWFLAKE, *args, **kwargs)

    def _create_conn(self, database=None):
        creds = self.credentials
        extra = creds.extra_dejson
        extra['password'] = creds.password
        if creds.host:
            extra['account'] = creds.host
        if creds.login:
            extra['user'] = creds.login
        if database or self.database:
            extra['database'] = database or self.database

        return snowflake.connector.connect(**extra)

    def _create_or_update_table(
        self,
        data,
        table_name,
        schema_name,
        database_name,
        columns_definition,
        columns_partial_query,
        update_on_columns,
        drop_and_replace,
        logging_function,
    ):
        logging_function('Preparing DWH Tables...')
        new_table_name = table_name + '_new'
        self.execute(
            sql="""CREATE OR REPLACE TABLE
                    "{database_name}"."{schema_name}"."{table_name}"
                    ({columns});
            """.format(**{
                'database_name': database_name,
                'schema_name': schema_name,
                'table_name': new_table_name,
                'columns': columns_partial_query,
            }),
            commit=False,
        )

        list_of_columns = self.execute_and_return_result(
            sql="""SELECT
                column_name
                FROM "{0}".information_schema.columns
                WHERE table_catalog = '{0}'
                AND table_schema = '{1}'
                AND table_name = '{2}'
                ORDER BY ordinal_position ASC
            """.format(
                database_name,
                schema_name,
                new_table_name,
            ),
        )
        list_of_columns = [col[0].strip() for col in list_of_columns]

        self.log.info('Writing data to a temporary .csv file')
        with TemporaryDirectory(prefix='uploadtosnowflake') as tmp_dir:
            with NamedTemporaryFile(
                dir=tmp_dir,
                prefix=new_table_name,
                suffix='.csv',
            ) as datafile:
                file_name = os.path.abspath(datafile.name)
                with open(file_name, mode='w') as csv_file:
                    csvwriter = csv.writer(
                        csv_file,
                        delimiter=',',
                        quotechar='"',
                        quoting=csv.QUOTE_MINIMAL,
                    )

                    for _ in range(len(data)):
                        datum = data.pop(0)
                        # Make sure order of csv is the same as order of columns
                        csvwriter.writerow(
                            [datum.get(col) for col in list_of_columns],
                        )

                # now stage and copy into snowflake!
                sql_upload = '''
                    USE DATABASE "{2}";
                    USE SCHEMA "{3}";
                    DROP FILE FORMAT IF EXISTS {1}_format;
                    CREATE OR REPLACE FILE FORMAT {1}_format
                        type = 'CSV'
                        field_delimiter = ','
                        field_optionally_enclosed_by = '"'
                    ;
                    DROP STAGE IF EXISTS {1}_stage;
                    CREATE OR REPLACE STAGE {1}_stage
                        file_format = {1}_format;
                    PUT file://{0} @{1}_stage;
                    COPY INTO "{2}"."{3}"."{1}" FROM '@{1}_stage/{4}.gz';
                '''.format(
                    file_name,
                    new_table_name,
                    database_name,
                    schema_name,
                    file_name[file_name.rfind('/')+1:],
                )
                self.log.info('Uploading data to Snowflake...')
                self.execute(sql_upload)

        if drop_and_replace or (not self.test_if_table_exists(
            table_name=table_name,
            schema_name=schema_name,
            database_name=database_name,
        )):
            sql_final = '''
                USE SCHEMA "{0}"."{1}";
                DROP TABLE IF EXISTS "{0}"."{1}"."{2}" CASCADE;
                ALTER TABLE "{0}"."{1}"."{3}" RENAME TO "{2}";
            '''.format(
                database_name,
                schema_name,
                table_name,
                new_table_name,
            )
        else:
            update_set_cols = []
            for col in columns_definition.keys():
                if not (col in update_on_columns):
                    update_set_cols += [col]

            sql_final = '''
                USE SCHEMA "{0}"."{1}";
                MERGE INTO "{0}"."{1}"."{2}" AS a
                    USING "{0}"."{1}"."{3}" AS b
                    ON {4}
                    WHEN MATCHED THEN UPDATE
                    SET {5}
                    WHEN NOT MATCHED THEN INSERT
                    ({6}) VALUES ({7})
                    ;
                DROP TABLE "{0}"."{1}"."{3}" CASCADE;
            '''.format(
                database_name,
                schema_name,
                table_name,
                new_table_name,
                ' AND '.join([
                    'a."{0}" = b."{0}"'.format(col) for col in update_on_columns
                ]),
                ', '.join([
                    'a."{0}" = b."{0}"'.format(col) for col in update_set_cols
                ]),
                '"'+'", "'.join(list(columns_definition.keys()))+'"',
                ', '.join([
                    'b."{0}"'.format(col) \
                    for col in list(columns_definition.keys())
                ]),
            )

        self.log.info('Final Step: Merging data')
        self.execute(sql_final)

    def commit(self):
        self.cur.execute('COMMIT;')
        self.cur.execute('BEGIN;')

    def rollback(self):
        self.cur.execute('ROLLBACK;')
        self.cur.execute('BEGIN;')

    def test_if_table_exists(
        self,
        table_name,
        schema_name,
        database_name=None,
    ):
        self.execute('USE DATABASE {0}'.format(
            database_name or self.database_name,
        ))
        return 0 < len(self.execute_and_return_result(
            """
            SELECT * FROM "{0}".information_schema.tables
            WHERE table_schema LIKE '{1}'
            AND table_name LIKE '{2}'
            """.format(
                database_name or self.database_name,
                schema_name,
                table_name,
            )
        ))


"""
odc-little-cube

use elatic instead of postgresql -

1. less code
2. more cloud-native

"""

#import logging
import sys
import os
#import json
#import pprint

from collections import OrderedDict

#from dcindexLib.xml_meta_lib import get_metadata_docs_bucket   # uses xml based metadata
from dcindexLib.parse_json import get_metadata_docs_json

from dcindexLib.projection_stuff import get_projection_info
# from dcindexLib.elastic_index import connect_elasticsearch
# from dcindexLib.elastic_index import l_create_index
# from dcindexLib.elastic_index import store_record

def create_footprint(coord):

    # still need to figure out if its lon,lat or lat,lon
    # in most cases the routines are now longitude the latitude or (x then y)
    print("TONY foot coord:", coord)

    foot = {
                "type": "Polygon",
                "coordinates": [
                    [
                        [
                            float(coord['ul']['lon']),
                            float(coord['ul']['lat'])
                        ],
                        [
                            float(coord['ur']['lon']),
                            float(coord['ur']['lat'])
                        ],
                        [
                            float(coord['lr']['lon']),
                            float(coord['lr']['lat'])
                        ],
                        [
                            float(coord['ll']['lon']),
                            float(coord['ll']['lat'])
                        ],
                        [
                            float(coord['ul']['lon']),
                            float(coord['ul']['lat'])
                        ]
                    ]
                ]

    }

    print (foot)

    return foot

def elastic_flatten_doc(mdoc):
    foot = create_footprint(mdoc['extent']['coord'])
    elastic_doc = {
                    'creation_dt': mdoc['creation_dt'],
                    'processing_level': mdoc['processing_level'],
                    'red': mdoc['image']['bands']['red']['path'],
                    'green': mdoc['image']['bands']['green']['path'],
                    'blue': mdoc['image']['bands']['blue']['path'],
                    'nir': mdoc['image']['bands']['nir']['path'],
                    'pixel_qa': mdoc['image']['bands']['pixel_qa']['path'],
                    'ul': {
                            'lat': mdoc['extent']['coord']['ul']['lat'],
                            'lon': mdoc['extent']['coord']['ul']['lon'],
                        },
                    'footprint': foot
                }
    return elastic_doc


from django.contrib import admin
from .models import Vendor, VendorMenu, UserRating

# Register your models here.

class VendorAdmin(admin.ModelAdmin):

    model = Vendor
    list_display = ('id', 'name')

class VendorMenuAdmin(admin.ModelAdmin):

    model = VendorMenu
    list_display = ('item_id', 'vendor_id', 'item_name', 'price', 'description')

class UserRatingAdmin(admin.ModelAdmin):

    model = UserRating
    list_display = ('md5', 'vendor_id', 'rating')

admin.site.register(Vendor, VendorAdmin)
admin.site.register(VendorMenu, VendorMenuAdmin)
admin.site.register(UserRating, UserRatingAdmin)


# there's a smarter way to do this... will figure out tomorrow

throw_to_int = {
    'A': 1,
    'B': 2,
    'C': 3,
    'X': 1,
    'Y': 2,
    'Z': 3,
}

def tally_rps1(filename):
    total = 0
    with open(filename) as f:
        lines = f.readlines()
        for l in lines:
            opp, play = l.strip().split()

            # score for what you threw
            total += throw_to_int[play]

            # score for if you tied:
            if throw_to_int[opp] == throw_to_int[play]:
                total += 3

            # if you win
            # i.e., your play is one "point" higher
            # the modulo is for when your opponent plays scissors (3) and you play rock (1)
            elif (throw_to_int[opp] % 3) + 1 == throw_to_int[play]:
                total += 6

    return total

# print(tally_rps('input/test.txt'))
print(tally_rps1('input/02.txt'))

def tally_rps2(filename):
    total = 0

    opp_choices = 'ABC'
    choice_to_points = {'A': 0, 'B': 1, 'C': 2, 'X': 0, 'Y': 1, 'Z': 2}
    with open(filename) as f:
        lines = f.readlines()
        for l in lines:
            opp, choice = l.strip().split()

            total += choice_to_points[choice] * 3

            if choice == 'Y':
                play = opp
            elif choice == 'X':
                play_idx = choice_to_points[opp] - 1
                play = opp_choices[play_idx]
            elif choice == 'Z':
                play_idx = (choice_to_points[opp] + 1) % 3
                play = opp_choices[play_idx]

            total += throw_to_int[play]

    return total

# print(tally_rps2('input/test.txt'))
print(tally_rps2('input/02.txt'))


import turtle
import colorsys
screen=turtle.Screen()
t = turtle.Turtle()
screen.setup(1020,720)
turtle.Screen().bgcolor("black")
t.speed(0)
n = 36
h = 0
for i in range(150):
    c= colorsys.hsv_to_rgb(h,1,0.9)
    h+=1/n
    t.color(c)
    t.left(145)
    for i in range(5):
        t.forward(300)
        t.left(150)
t.hideturtle()
turtle.done()

Python/Turtle graphic desing python/Tutorial-19.py


import torch
import PIL.Image as Image
import requests

# from transformers import CLIPProcessor, CLIPModel

# processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

feature_dim = 512
# def define_model(device='cuda'):
#     model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
#     return model

import clip

def define_model(device='cuda'):
    model, preprocess = clip.load("ViT-B/16", device=device)
    model = model.eval()
    return model, preprocess

# def get_image_features(model, img_tensor):
#     inputs = processor(images=img_tensor, return_tensors="pt")
#     image_features = model.get_image_features(**inputs)
#     return image_features

def get_image_features(model, img_tensor):
    with torch.no_grad():
        image_features = model.encode_image(img_tensor)
    return image_features.float()

if __name__ == '__main__':
    url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    image = Image.open(requests.get(url, stream=True).raw)
    print(image.size)
    features = get_image_features(image)

    print(features.shape)