From f536f21d3c4ea9e666dd047ad96c08fc14c9a8e4 Mon Sep 17 00:00:00 2001
From: Eli Ribble <eli@authentise.com>
Date: Tue, 28 Jun 2016 15:46:18 -0600
Subject: [PATCH] Standardize columns in our table
Like when we last did an update and whether or not something is deleted.
Nice for cleanup and code reuse
---
.../ab038e16ec9a_standardize_columns.py | 28 +++++++++++++
templates/accounts.html | 11 ++++-
tests/files/query_transactions.ofx | 42 +++++++++++++++++++
vanth/celery.py | 19 +++++++++
vanth/download.py | 17 ++++++++
vanth/pages/accounts.py | 41 +++++++++---------
vanth/platform/ofxaccount.py | 37 +++++++++++-----
vanth/platform/ofxrecord.py | 34 +++++++++++++++
vanth/platform/ofxsource.py | 11 ++++-
vanth/tables.py | 37 ++++++++--------
10 files changed, 226 insertions(+), 51 deletions(-)
create mode 100644 alembic/versions/ab038e16ec9a_standardize_columns.py
create mode 100644 tests/files/query_transactions.ofx
create mode 100644 vanth/celery.py
create mode 100644 vanth/download.py
create mode 100644 vanth/platform/ofxrecord.py
diff --git a/alembic/versions/ab038e16ec9a_standardize_columns.py b/alembic/versions/ab038e16ec9a_standardize_columns.py
new file mode 100644
index 0000000..fa1b888
--- /dev/null
+++ b/alembic/versions/ab038e16ec9a_standardize_columns.py
@@ -0,0 +1,28 @@
+"""standardize_columns
+
+Revision ID: ab038e16ec9a
+Revises: 4b8ce290f890
+Create Date: 2016-06-28 15:44:52.141256
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ab038e16ec9a'
+down_revision = '4b8ce290f890'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.add_column('ofxaccount', sa.Column('deleted', sa.DateTime(), nullable=True))
+ op.add_column('ofxrecord', sa.Column('deleted', sa.DateTime(), nullable=True))
+ op.add_column('ofxsource', sa.Column('deleted', sa.DateTime(), nullable=True))
+
+
+def downgrade():
+ op.drop_column('ofxsource', 'deleted')
+ op.drop_column('ofxrecord', 'deleted')
+ op.drop_column('ofxaccount', 'deleted')
diff --git a/templates/accounts.html b/templates/accounts.html
index c719ca9..1f7fd15 100644
--- a/templates/accounts.html
+++ b/templates/accounts.html
@@ -3,12 +3,19 @@
<h1>Accounts</h1>
{% if accounts %}
<table class="table">
- <tr><th>Name</th><th>Type</th><th>Institution</th></tr>
+ <tr><th>Name</th><th>Type</th><th>Institution</th><th>Last Update</th><th></th></tr>
{% for account in accounts %}
<tr>
<td>{{ account.name }}</td>
<td>{{ account.type }}</td>
- <td>{{ account.institution }}</td>
+ <td>{{ account.source.name }}</td>
+ <td>Never</td>
+ <td>
+ <form method="POST" action="/update/">
+ <input type="hidden" name="account_uuid" value="{{ account.uuid }}"></input>
+ <input type="submit" value="Update" class="btn btn-primary"></input>
+ </form>
+ </td>
</tr>
{% endfor %}
</table>
diff --git a/tests/files/query_transactions.ofx b/tests/files/query_transactions.ofx
new file mode 100644
index 0000000..83a9ed9
--- /dev/null
+++ b/tests/files/query_transactions.ofx
@@ -0,0 +1,42 @@
+OFXHEADER:100
+DATA:OFXSGML
+VERSION:102
+SECURITY:NONE
+ENCODING:USASCII
+CHARSET:1252
+COMPRESSION:NONE
+OLDFILEUID:NONE
+NEWFILEUID:NONE
+
+<OFX>
+<SIGNONMSGSRQV1>
+<SONRQ>
+<DTCLIENT>20160102030405.000[-7:MST]
+<USERID>123456789
+<USERPASS>1234
+<LANGUAGE>ENG
+<FI>
+<ORG>AFCU
+<FID>12345
+</FI>
+<APPID>QWIN
+<APPVER>1200
+</SONRQ>
+</SIGNONMSGSRQV1>
+<BANKMSGSRQV1>
+<STMTTRNRQ>
+<TRNUID>00000000
+<STMTRQ>
+<BANKACCTFROM>
+<BANKID>1234567
+<ACCTID>123456-0.9:CHK
+<ACCTTYPE>CHECKING
+</BANKACCTFROM>
+<INCTRAN>
+<DTSTART>20160102
+<INCLUDE>Y
+</INCTRAN>
+</STMTRQ>
+</STMTTRNRQ>
+</BANKMSGSRQV1>
+</OFX>
diff --git a/vanth/celery.py b/vanth/celery.py
new file mode 100644
index 0000000..468a140
--- /dev/null
+++ b/vanth/celery.py
@@ -0,0 +1,19 @@
+import celery
+
+import vanth.download
+import vanth.main
+import vanth.platform.ofxaccount
+import vanth.platform.ofxrecord
+import vanth.platform.ofxsource
+
+app = celery.Celery('vanth')
+app.conf.CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']
+app.conf.CELERY_TASK_SERIALIZER = 'json'
+app.conf.CELERY_ALWAYS_EAGER = True
+
+@app.task()
+def update_account(account_uuid):
+ account = vanth.platform.ofxaccount.by_uuid(account_uuid)[0]
+ source = vanth.platform.ofxsource.by_uuid(account['source']['uuid'])[0]
+ document = vanth.download.transactions(source, account)
+ vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items)
diff --git a/vanth/download.py b/vanth/download.py
new file mode 100644
index 0000000..a9f0daf
--- /dev/null
+++ b/vanth/download.py
@@ -0,0 +1,17 @@
+import requests
+
+import vanth.ofx
+import vanth.platform.ofxaccount
+
+
+def do_all():
+ sources = {source['name']: source for source in vanth.platform.ofxsource.get()}
+ accounts = vanth.platform.ofxaccount.by_user(user_id=None)
+ for account in accounts:
+ transactions(sources[account['institution']], account)
+
+def transactions(source, account):
+ body = vanth.ofx.query_transactions(source, account)
+ response = requests.post('https://ofx.americafirst.com/', data=body, headers={'Content-Type': 'application/x-ofx'})
+ assert response.ok, response.text
+ return vanth.ofx.parse(response.text)
diff --git a/vanth/pages/accounts.py b/vanth/pages/accounts.py
index 96a2935..9b7f61d 100644
--- a/vanth/pages/accounts.py
+++ b/vanth/pages/accounts.py
@@ -1,5 +1,7 @@
import flask
+import vanth.celery
+import vanth.pages.tools
import vanth.platform.ofxaccount
import vanth.platform.ofxsource
@@ -7,27 +9,28 @@ blueprint = flask.Blueprint('accounts', __name__)
@blueprint.route('/accounts/', methods=['GET'])
def get_accounts():
- my_accounts = vanth.platform.ofxaccount.get(flask.session['user_id'])
+ my_accounts = vanth.platform.ofxaccount.by_user(flask.session['user_id'])
sources = vanth.platform.ofxsource.get()
return flask.render_template('accounts.html', accounts=my_accounts, sources=sources)
@blueprint.route('/account/', methods=['POST'])
-def post_account():
- account_id = flask.request.form.get('account_id')
- account_type = flask.request.form.get('account_type')
- institution = flask.request.form.get('institution')
- name = flask.request.form.get('name')
- password = flask.request.form.get('password')
- user_id = flask.request.form.get('user_id')
-
-
- vanth.platform.ofxaccount.create({
- 'owner' : flask.session['user_id'],
- 'account_id' : account_id,
- 'institution' : institution,
- 'name' : name,
- 'password' : password,
- 'type' : account_type,
- 'user_id' : user_id,
- })
+@vanth.pages.tools.parse({
+ 'account_id' : str,
+ 'account_type' : str,
+ 'institution' : str,
+ 'name' : str,
+ 'password' : str,
+ 'user_id' : str,
+})
+def post_account(arguments):
+ arguments['owner'] = flask.session['user_id']
+ vanth.platform.ofxaccount.create(arguments)
+ return flask.redirect('/accounts/')
+
+@blueprint.route('/update/', methods=['POST'])
+@vanth.pages.tools.parse({
+ 'account_uuid' : str,
+})
+def post_update(arguments):
+ vanth.celery.update_account.delay(**arguments)
return flask.redirect('/accounts/')
diff --git a/vanth/platform/ofxaccount.py b/vanth/platform/ofxaccount.py
index 1c7b69f..f931939 100644
--- a/vanth/platform/ofxaccount.py
+++ b/vanth/platform/ofxaccount.py
@@ -7,32 +7,49 @@ import vanth.platform.ofxsource
import vanth.tables
-def get(user_id):
- engine = chryso.connection.get()
- query = sqlalchemy.select([
- vanth.tables.OFXSource.c.name.label('institution'),
+def _select():
+ return sqlalchemy.select([
+ vanth.tables.OFXAccount.c.account_id,
vanth.tables.OFXAccount.c.name,
+ vanth.tables.OFXAccount.c.password,
vanth.tables.OFXAccount.c.source,
vanth.tables.OFXAccount.c.type,
vanth.tables.OFXAccount.c.user_id,
vanth.tables.OFXAccount.c.uuid,
+ vanth.tables.OFXSource.c.name.label('source.name'),
+ vanth.tables.OFXSource.c.uuid.label('source.uuid'),
]).where(
vanth.tables.OFXAccount.c.source == vanth.tables.OFXSource.c.uuid
)
- if user_id:
- query = query.where(
- vanth.tables.OFXAccount.c.owner == user_id
- )
+
+def _execute_and_convert(query):
+ engine = chryso.connection.get()
results = engine.execute(query)
return [{
- 'institution' : result[vanth.tables.OFXSource.c.name.label('institution')],
+ 'account_id' : result[vanth.tables.OFXAccount.c.account_id],
'name' : result[vanth.tables.OFXAccount.c.name],
- 'source' : result[vanth.tables.OFXAccount.c.source],
+ 'password' : result[vanth.tables.OFXAccount.c.password],
+ 'source' : {
+ 'name' : result[vanth.tables.OFXSource.c.name.label('source.name')],
+ 'uuid' : result[vanth.tables.OFXSource.c.name.label('source.uuid')],
+ },
'type' : result[vanth.tables.OFXAccount.c.type],
'user_id' : result[vanth.tables.OFXAccount.c.user_id],
'uuid' : result[vanth.tables.OFXAccount.c.uuid],
} for result in results]
+def by_uuid(account_uuid):
+ query = _select().where(vanth.tables.OFXAccount.c.uuid == account_uuid)
+ return _execute_and_convert(query)
+
+def by_user(user_id):
+ query = _select()
+ if user_id:
+ query = query.where(
+ vanth.tables.OFXAccount.c.owner == user_id
+ )
+ return _execute_and_convert(query)
+
def create(values):
engine = chryso.connection.get()
diff --git a/vanth/platform/ofxrecord.py b/vanth/platform/ofxrecord.py
new file mode 100644
index 0000000..4369684
--- /dev/null
+++ b/vanth/platform/ofxrecord.py
@@ -0,0 +1,34 @@
+import logging
+import uuid
+
+import chryso.connection
+import sqlalchemy
+
+import vanth.tables
+
+LOGGER = logging.getLogger(__name__)
+
+def ensure_exists(account, transactions):
+ engine = chryso.connection.get()
+ query = sqlalchemy.select([
+ vanth.tables.OFXRecord.c.fid,
+ ]).where(vanth.tables.OFXRecord.c.ofxaccount == account['uuid'])
+ results = engine.execute(query).fetchall()
+ LOGGER.debug("Found %d OFX records for %s", len(results), account)
+ known_records = {result[vanth.tables.OFXRecord.c.fid] for result in results}
+ new_records = [transaction for transaction in transactions if transaction.id not in known_records]
+ LOGGER.debug("Have %d new transactions to save", len(new_records))
+ to_insert = [{
+ 'amount' : transaction.amount,
+ 'available' : transaction.available,
+ 'fid' : transaction.id,
+ 'memo' : transaction.memo,
+ 'name' : transaction.name,
+ 'ofxaccount' : account['uuid'],
+ 'posted' : transaction.posted,
+ 'type' : transaction.type,
+ 'uuid' : uuid.uuid4(),
+ } for transaction in new_records]
+ if to_insert:
+ engine.execute(vanth.tables.OFXRecord.insert(), to_insert) # pylint: disable=no-value-for-parameter
+ LOGGER.debug("Done inserting %d records", len(new_records))
diff --git a/vanth/platform/ofxsource.py b/vanth/platform/ofxsource.py
index 764745f..3d577ca 100644
--- a/vanth/platform/ofxsource.py
+++ b/vanth/platform/ofxsource.py
@@ -6,8 +6,15 @@ import vanth.tables
LOGGER = logging.getLogger(__name__)
-def get():
+def _query_and_convert(query):
engine = chryso.connection.get()
- query = vanth.tables.OFXSource.select()
results = engine.execute(query).fetchall()
return [dict(result) for result in results]
+
+def by_uuid(uuid):
+ query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid)
+ return _query_and_convert(query)
+
+def get():
+ query = vanth.tables.OFXSource.select()
+ return _query_and_convert(query)
diff --git a/vanth/tables.py b/vanth/tables.py
index 34db0ca..e126946 100644
--- a/vanth/tables.py
+++ b/vanth/tables.py
@@ -1,10 +1,22 @@
import chryso.constants
from sqlalchemy import (Column, Date, DateTime, Float, ForeignKey, Integer,
- MetaData, String, Table, UniqueConstraint, func)
+ MetaData, String, Table, UniqueConstraint, func, text)
from sqlalchemy.dialects.postgresql import UUID
metadata = MetaData(naming_convention=chryso.constants.CONVENTION)
+def table(name, *args, **kwargs):
+ return Table(
+ name,
+ metadata,
+ Column('uuid', UUID(as_uuid=True), primary_key=True, server_default=text('gen_random_uuid()')),
+ Column('created', DateTime, server_default=func.now(), nullable=False),
+ Column('updated', DateTime, server_default=func.now(), onupdate=func.now(), nullable=False),
+ Column('deleted', DateTime, nullable=True),
+ *args,
+ **kwargs
+ )
+
User = Table('users', metadata,
Column('uuid', UUID(), primary_key=True),
Column('username', String(255), nullable=False),
@@ -17,8 +29,7 @@ User = Table('users', metadata,
UniqueConstraint('username', name='uq_user_username'),
)
-CreditCard = Table('credit_card', metadata,
- Column('uuid', UUID(as_uuid=True), primary_key=True),
+CreditCard = table('credit_card',
Column('brand', String(20), nullable=False), # The brand of the card, like 'visa'
Column('card_id', String(100), nullable=False), # The ID of the card from Stripe
Column('country', String(1024), nullable=False), # The Country of the card, like 'US'
@@ -28,23 +39,18 @@ CreditCard = Table('credit_card', metadata,
Column('last_four', Integer(), nullable=False), # The last four digits of the card
Column('token', String(), nullable=False), # The token we can use with Stripe to do stuff
Column('user_uri', String(2048), nullable=False), # The URI of the user that created the record
- Column('created', DateTime(), nullable=False, server_default=func.now()),
- Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
- Column('deleted', DateTime(), nullable=True),
UniqueConstraint('card_id', name='uq_credit_card_id'),
)
-OFXSource = Table('ofxsource', metadata,
- Column('uuid', UUID(as_uuid=True), primary_key=True),
+OFXSource = table('ofxsource',
Column('name', String(255), nullable=False), # The name of the institution such as 'America First Credit Union'
Column('fid', String(255), nullable=False), # The FID of the institution, such as 54324
- Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516. This may be a routing number
- Column('created', DateTime(), nullable=False, server_default=func.now()),
- Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
+ Column('bankid', String(255), nullable=False), # The bank ID of the institution such as 324377516.
+ # This may be a routing number
UniqueConstraint('fid', name='uq_ofxsource_fid'),
)
-OFXAccount = Table('ofxaccount', metadata,
+OFXAccount = table('ofxaccount',
Column('account_id', String(255), nullable=False), # 123456-0.9:CHK
Column('name', String(255), nullable=False), # My checking account
Column('owner', None, ForeignKey(User.c.uuid, name='fk_user'), nullable=False),
@@ -54,12 +60,9 @@ OFXAccount = Table('ofxaccount', metadata,
Column('user_id', String(255), nullable=False), # The user ID for the bank
Column('uuid', UUID(as_uuid=True), primary_key=True),
- Column('created', DateTime(), nullable=False, server_default=func.now()),
- Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
)
-OFXRecord = Table('ofxrecord', metadata,
- Column('uuid', UUID(as_uuid=True), primary_key=True),
+OFXRecord = table('ofxrecord',
Column('fid', String(255), nullable=False), # The Financial institution's ID
Column('amount', Float(), nullable=False), # The amount of the record, like -177.91
Column('available', Date(), nullable=True), # The date the record was available
@@ -68,6 +71,4 @@ OFXRecord = Table('ofxrecord', metadata,
Column('memo', String(2048), nullable=True), # The memo of the transaction, like 'POINT OF SALE PURCHASE #0005727'
Column('type', String(255), nullable=True), # The type of the record, like 'POS'
Column('ofxaccount', None, ForeignKey(OFXAccount.c.uuid, name='fk_ofxaccount'), nullable=False),
- Column('created', DateTime(), nullable=False, server_default=func.now()),
- Column('updated', DateTime(), nullable=False, server_default=func.now(), onupdate=func.now()),
)