Skip to content

Get optimizations from target-postgres dependency #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nsmith22 merged 8 commits into from
Jun 9, 2022
+18 −4
Merged

Get optimizations from target-postgres dependency #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d5696e3
Get latest target-postgres version which contains multiple performanc…
nsmith22 Jun 9, 2022
683e470
Try https git link
nsmith22 Jun 9, 2022
773ff8d
Fix typo
nsmith22 Jun 9, 2022
c64cb64
Another try at the git link
nsmith22 Jun 9, 2022
12cec71
Fix package name
nsmith22 Jun 9, 2022
d909745
Avoid doing work to insert zero-length record batches
nsmith22 Jun 9, 2022
478630a
Add date time serialization cache optimization
nsmith22 Jun 9, 2022
fee3dcb
Fix comment
nsmith22 Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
py_modules=['target_snowflake'],
install_requires=[
'singer-python==5.9.0',
'singer-target-postgres==0.2.4',
'singer-target-postgres@git+https://github.com/datamill-co/target-postgres.git#279cb62d2a80b1bd8e8ab1191e7a1d17c19383a8',
'target-redshift==0.2.4',
'botocore<1.13.0,>=1.12.253',
'snowflake-connector-python==2.2.5'
Expand Down
20 changes: 17 additions & 3 deletions target_snowflake/snowflake.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import os
import re
import uuid
from functools import lru_cache

import arrow
from psycopg2 import sql
Expand All @@ -21,6 +22,16 @@
from target_snowflake.connection import connect
from target_snowflake.exceptions import SnowflakeError

# copied in from optimization in PostgresTarget: https://github.com/datamill-co/target-postgres/commit/6a3da026d2bb4681fdf46bd7ca69fbb164489d8a
@lru_cache(maxsize=128)
def _format_datetime(value):
"""
Format a datetime value. This is only called from the
SnowflakeTarget.serialize_table_record_datetime_value
but this non-method version allows caching
"""
return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ')

class SnowflakeTarget(SQLInterface):
"""
Specific Snowflake implementation of a Singer Target.
Expand Down Expand Up @@ -334,7 +345,7 @@ def serialize_table_record_null_value(self, remote_schema, streamed_schema, fiel
return value

def serialize_table_record_datetime_value(self, remote_schema, streamed_schema, field, value):
return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ')
return _format_datetime(value)

def perform_update(self, cur, target_table_name, temp_table_name, key_properties, columns, subkeys):
full_table_name = '{}.{}.{}'.format(
Expand Down Expand Up @@ -545,8 +556,11 @@ def persist_csv_rows(self,
subkeys)

def write_table_batch(self, cur, table_batch, metadata):
remote_schema = table_batch['remote_schema']
record_count = len(table_batch['records'])
if record_count == 0:
return 0

remote_schema = table_batch['remote_schema']

## Create temp table to upload new data to
target_table_name = self.canonicalize_identifier('tmp_' + str(uuid.uuid4()))
Expand Down Expand Up @@ -586,7 +600,7 @@ def transform():
csv_headers,
csv_rows)

return len(table_batch['records'])
return record_count

def add_column(self, cur, table_name, column_name, column_schema):
cur.execute('''
Expand Down