Standardize columns in our table

Like when we last did an update and whether or not something is deleted.
Nice for cleanup and code reuse

alembic/versions/ab038e16ec9a_standardize_columns.py

@@ -0,0 +1,28 @@
+"""standardize_columns
+
+Revision ID: ab038e16ec9a
+Revises: 4b8ce290f890
+Create Date: 2016-06-28 15:44:52.141256
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ab038e16ec9a'
+down_revision = '4b8ce290f890'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('ofxaccount', sa.Column('deleted', sa.DateTime(), nullable=True))
+    op.add_column('ofxrecord', sa.Column('deleted', sa.DateTime(), nullable=True))
+    op.add_column('ofxsource', sa.Column('deleted', sa.DateTime(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('ofxsource', 'deleted')
+    op.drop_column('ofxrecord', 'deleted')
+    op.drop_column('ofxaccount', 'deleted')

templates/accounts.html

@@ -3,12 +3,19 @@
 <h1>Accounts</h1>
 {% if accounts %}
   <table class="table">
-  <tr><th>Name</th><th>Type</th><th>Institution</th></tr>
+  <tr><th>Name</th><th>Type</th><th>Institution</th><th>Last Update</th><th></th></tr>
   {% for account in accounts %}
   <tr>
     <td>{{ account.name }}</td>
     <td>{{ account.type }}</td>
-    <td>{{ account.institution }}</td>
+    <td>{{ account.source.name }}</td>
+    <td>Never</td>
+    <td>
+      <form method="POST" action="/update/">
+        <input type="hidden" name="account_uuid" value="{{ account.uuid }}"></input>
+        <input type="submit" value="Update" class="btn btn-primary"></input>
+      </form>
+    </td>
   </tr>
   {% endfor %}
   </table>

tests/files/query_transactions.ofx

@@ -0,0 +1,42 @@
+OFXHEADER:100
+DATA:OFXSGML
+VERSION:102
+SECURITY:NONE
+ENCODING:USASCII
+CHARSET:1252
+COMPRESSION:NONE
+OLDFILEUID:NONE
+NEWFILEUID:NONE
+
+<OFX>
+<SIGNONMSGSRQV1>
+<SONRQ>
+<DTCLIENT>20160102030405.000[-7:MST]
+<USERID>123456789
+<USERPASS>1234
+<LANGUAGE>ENG
+<FI>
+<ORG>AFCU
+<FID>12345
+</FI>
+<APPID>QWIN
+<APPVER>1200
+</SONRQ>
+</SIGNONMSGSRQV1>
+<BANKMSGSRQV1>
+<STMTTRNRQ>
+<TRNUID>00000000
+<STMTRQ>
+<BANKACCTFROM>
+<BANKID>1234567
+<ACCTID>123456-0.9:CHK
+<ACCTTYPE>CHECKING
+</BANKACCTFROM>
+<INCTRAN>
+<DTSTART>20160102
+<INCLUDE>Y
+</INCTRAN>
+</STMTRQ>
+</STMTTRNRQ>
+</BANKMSGSRQV1>
+</OFX>

vanth/celery.py

@@ -0,0 +1,19 @@
+import celery
+
+import vanth.download
+import vanth.main
+import vanth.platform.ofxaccount
+import vanth.platform.ofxrecord
+import vanth.platform.ofxsource
+
+app = celery.Celery('vanth')
+app.conf.CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']
+app.conf.CELERY_TASK_SERIALIZER = 'json'
+app.conf.CELERY_ALWAYS_EAGER = True
+
+@app.task()
+def update_account(account_uuid):
+    account = vanth.platform.ofxaccount.by_uuid(account_uuid)[0]
+    source = vanth.platform.ofxsource.by_uuid(account['source']['uuid'])[0]
+    document = vanth.download.transactions(source, account)
+    vanth.platform.ofxrecord.ensure_exists(account, document.body.statement.transactions.items)

vanth/download.py

@@ -0,0 +1,17 @@
+import requests
+
+import vanth.ofx
+import vanth.platform.ofxaccount
+
+
+def do_all():
+    sources = {source['name']: source for source in vanth.platform.ofxsource.get()}
+    accounts = vanth.platform.ofxaccount.by_user(user_id=None)
+    for account in accounts:
+        transactions(sources[account['institution']], account)
+
+def transactions(source, account):
+    body = vanth.ofx.query_transactions(source, account)
+    response = requests.post('https://ofx.americafirst.com/', data=body, headers={'Content-Type': 'application/x-ofx'})
+    assert response.ok, response.text
+    return vanth.ofx.parse(response.text)

vanth/pages/accounts.py

@@ -1,5 +1,7 @@
 import flask
 
+import vanth.celery
+import vanth.pages.tools
 import vanth.platform.ofxaccount
 import vanth.platform.ofxsource
 
@@ -7,27 +9,28 @@ blueprint = flask.Blueprint('accounts', __name__)
 
 @blueprint.route('/accounts/', methods=['GET'])
 def get_accounts():
-    my_accounts = vanth.platform.ofxaccount.get(flask.session['user_id'])
+    my_accounts = vanth.platform.ofxaccount.by_user(flask.session['user_id'])
     sources = vanth.platform.ofxsource.get()
     return flask.render_template('accounts.html', accounts=my_accounts, sources=sources)
 
 @blueprint.route('/account/', methods=['POST'])
-def post_account():
+@vanth.pages.tools.parse({
-    account_id   = flask.request.form.get('account_id')
+    'account_id'    : str,
-    account_type = flask.request.form.get('account_type')
+    'account_type'  : str,
-    institution  = flask.request.form.get('institution')
+    'institution'   : str,
-    name         = flask.request.form.get('name')
+    'name'          : str,
-    password     = flask.request.form.get('password')
+    'password'      : str,
-    user_id      = flask.request.form.get('user_id')
+    'user_id'       : str,
-
+})
-
+def post_account(arguments):
-    vanth.platform.ofxaccount.create({
+    arguments['owner'] = flask.session['user_id']
-        'owner'         : flask.session['user_id'],
+    vanth.platform.ofxaccount.create(arguments)
-        'account_id'    : account_id,
+    return flask.redirect('/accounts/')
-        'institution'   : institution,
+
-        'name'          : name,
+@blueprint.route('/update/', methods=['POST'])
-        'password'      : password,
+@vanth.pages.tools.parse({
-        'type'          : account_type,
+    'account_uuid'  : str,
-        'user_id'       : user_id,
+})
-    })
+def post_update(arguments):
+    vanth.celery.update_account.delay(**arguments)
     return flask.redirect('/accounts/')

vanth/platform/ofxaccount.py

@@ -7,32 +7,49 @@ import vanth.platform.ofxsource
 import vanth.tables
 
 
-def get(user_id):
+def _select():
-    engine = chryso.connection.get()
+    return sqlalchemy.select([
-    query = sqlalchemy.select([
+        vanth.tables.OFXAccount.c.account_id,
-        vanth.tables.OFXSource.c.name.label('institution'),
         vanth.tables.OFXAccount.c.name,
+        vanth.tables.OFXAccount.c.password,
         vanth.tables.OFXAccount.c.source,
         vanth.tables.OFXAccount.c.type,
         vanth.tables.OFXAccount.c.user_id,
         vanth.tables.OFXAccount.c.uuid,
+        vanth.tables.OFXSource.c.name.label('source.name'),
+        vanth.tables.OFXSource.c.uuid.label('source.uuid'),
     ]).where(
         vanth.tables.OFXAccount.c.source == vanth.tables.OFXSource.c.uuid
     )
-    if user_id:
+
-        query = query.where(
+def _execute_and_convert(query):
-            vanth.tables.OFXAccount.c.owner == user_id
+    engine = chryso.connection.get()
-        )
     results = engine.execute(query)
     return [{
-        'institution'   : result[vanth.tables.OFXSource.c.name.label('institution')],
+        'account_id'    : result[vanth.tables.OFXAccount.c.account_id],
         'name'          : result[vanth.tables.OFXAccount.c.name],
-        'source'        : result[vanth.tables.OFXAccount.c.source],
+        'password'      : result[vanth.tables.OFXAccount.c.password],
+        'source'        : {
+            'name'      : result[vanth.tables.OFXSource.c.name.label('source.name')],
+            'uuid'      : result[vanth.tables.OFXSource.c.name.label('source.uuid')],
+        },
         'type'          : result[vanth.tables.OFXAccount.c.type],
         'user_id'       : result[vanth.tables.OFXAccount.c.user_id],
         'uuid'          : result[vanth.tables.OFXAccount.c.uuid],
     } for result in results]
 
+def by_uuid(account_uuid):
+    query = _select().where(vanth.tables.OFXAccount.c.uuid == account_uuid)
+    return _execute_and_convert(query)
+
+def by_user(user_id):
+    query = _select()
+    if user_id:
+        query = query.where(
+            vanth.tables.OFXAccount.c.owner == user_id
+        )
+    return _execute_and_convert(query)
+
 def create(values):
     engine = chryso.connection.get()
 

vanth/platform/ofxrecord.py

@@ -0,0 +1,34 @@
+import logging
+import uuid
+
+import chryso.connection
+import sqlalchemy
+
+import vanth.tables
+
+LOGGER = logging.getLogger(__name__)
+
+def ensure_exists(account, transactions):
+    engine = chryso.connection.get()
+    query = sqlalchemy.select([
+        vanth.tables.OFXRecord.c.fid,
+    ]).where(vanth.tables.OFXRecord.c.ofxaccount == account['uuid'])
+    results = engine.execute(query).fetchall()
+    LOGGER.debug("Found %d OFX records for %s", len(results), account)
+    known_records = {result[vanth.tables.OFXRecord.c.fid] for result in results}
+    new_records = [transaction for transaction in transactions if transaction.id not in known_records]
+    LOGGER.debug("Have %d new transactions to save", len(new_records))
+    to_insert = [{
+        'amount'        : transaction.amount,
+        'available'     : transaction.available,
+        'fid'           : transaction.id,
+        'memo'          : transaction.memo,
+        'name'          : transaction.name,
+        'ofxaccount'    : account['uuid'],
+        'posted'        : transaction.posted,
+        'type'          : transaction.type,
+        'uuid'          : uuid.uuid4(),
+    } for transaction in new_records]
+    if to_insert:
+        engine.execute(vanth.tables.OFXRecord.insert(), to_insert) # pylint: disable=no-value-for-parameter
+        LOGGER.debug("Done inserting %d records", len(new_records))

vanth/platform/ofxsource.py

@@ -6,8 +6,15 @@ import vanth.tables
 
 LOGGER = logging.getLogger(__name__)
 
-def get():
+def _query_and_convert(query):
     engine = chryso.connection.get()
-    query = vanth.tables.OFXSource.select()
     results = engine.execute(query).fetchall()
     return [dict(result) for result in results]
+
+def by_uuid(uuid):
+    query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid)
+    return _query_and_convert(query)
+
+def get():
+    query = vanth.tables.OFXSource.select()
+    return _query_and_convert(query)

vanth/tables.py

@@ -1,10 +1,22 @@
 import chryso.constants
 from sqlalchemy import (Column, Date, DateTime, Float, ForeignKey, Integer,
-                        MetaData, String, Table, UniqueConstraint, func)
+                        MetaData, String, Table, UniqueConstraint, func, text)
 from sqlalchemy.dialects.postgresql import UUID
 
 metadata = MetaData(naming_convention=chryso.constants.CONVENTION)
 
+def table(name, *args, **kwargs):
+    return Table(
+        name,
+        metadata,
+        Column('uuid', UUID(as_uuid=True), primary_key=True, server_default=text('gen_random_uuid()')),
+        Column('created', DateTime, server_default=func.now(), nullable=False),
+        Column('updated', DateTime, server_default=func.now(), onupdate=func.now(), nullable=False),
+        Column('deleted', DateTime, nullable=True),
+        *args,
+        **kwargs
+    )
+
 User = Table('users', metadata,
     Column('uuid',          UUID(), primary_key=True),
     Column('username',      String(255), nullable=False),
@@ -17,8 +29,7 @@ User = Table('users', metadata,
     UniqueConstraint('username', name='uq_user_username'),
 )
 
-CreditCard = Table('credit_card', metadata,
+CreditCard = table('credit_card',
-    Column('uuid',              UUID(as_uuid=True),         primary_key=True),
     Column('brand',             String(20),     nullable=False), # The brand of the card, like 'visa'
     Column('card_id',           String(100),    nullable=False), # The ID of the card from Stripe
     Column('country',           String(1024),   nullable=False), # The Country of the card, like 'US'
@@ -28,23 +39,18 @@ CreditCard = Table('credit_card', metadata,
     Column('last_four',         Integer(),      nullable=False), # The last four digits of the card
     Column('token',             String(),       nullable=False), # The token we can use with Stripe to do stuff
     Column('user_uri',          String(2048),   nullable=False), # The URI of the user that created the record
-    Column('created',           DateTime(),     nullable=False, server_default=func.now()),
-    Column('updated',           DateTime(),     nullable=False, server_default=func.now(), onupdate=func.now()),
-    Column('deleted',           DateTime(),     nullable=True),
     UniqueConstraint('card_id', name='uq_credit_card_id'),
 )
 
-OFXSource = Table('ofxsource',  metadata,
+OFXSource = table('ofxsource',
-    Column('uuid',              UUID(as_uuid=True),         primary_key=True),
     Column('name',              String(255),     nullable=False), # The name of the institution such as 'America First Credit Union'
     Column('fid',               String(255),     nullable=False), # The FID of the institution, such as 54324
-    Column('bankid',            String(255),     nullable=False), # The bank ID of the institution such as 324377516. This may be a routing number
+    Column('bankid',            String(255),     nullable=False), # The bank ID of the institution such as 324377516.
-    Column('created',           DateTime(),      nullable=False, server_default=func.now()),
+                                                                  # This may be a routing number
-    Column('updated',           DateTime(),     nullable=False, server_default=func.now(), onupdate=func.now()),
     UniqueConstraint('fid', name='uq_ofxsource_fid'),
 )
 
-OFXAccount = Table('ofxaccount', metadata,
+OFXAccount = table('ofxaccount',
     Column('account_id',        String(255),         nullable=False), # 123456-0.9:CHK
     Column('name',              String(255),         nullable=False), # My checking account
     Column('owner',             None,                ForeignKey(User.c.uuid, name='fk_user'), nullable=False),
@@ -54,12 +60,9 @@ OFXAccount = Table('ofxaccount', metadata,
     Column('user_id',           String(255),         nullable=False), # The user ID for the bank
     Column('uuid',              UUID(as_uuid=True),  primary_key=True),
 
-    Column('created',           DateTime(),          nullable=False, server_default=func.now()),
-    Column('updated',           DateTime(),          nullable=False, server_default=func.now(), onupdate=func.now()),
 )
 
-OFXRecord = Table('ofxrecord',  metadata,
+OFXRecord = table('ofxrecord',
-    Column('uuid',              UUID(as_uuid=True),         primary_key=True),
     Column('fid',               String(255),    nullable=False), # The Financial institution's ID
     Column('amount',            Float(),        nullable=False), # The amount of the record, like -177.91
     Column('available',         Date(),         nullable=True), # The date the record was available
@@ -68,6 +71,4 @@ OFXRecord = Table('ofxrecord',  metadata,
     Column('memo',              String(2048),   nullable=True), # The memo of the transaction, like 'POINT OF SALE PURCHASE #0005727'
     Column('type',              String(255),    nullable=True), # The type of the record, like 'POS'
     Column('ofxaccount',        None,           ForeignKey(OFXAccount.c.uuid, name='fk_ofxaccount'), nullable=False),
-    Column('created',           DateTime(),     nullable=False, server_default=func.now()),
-    Column('updated',           DateTime(),     nullable=False, server_default=func.now(), onupdate=func.now()),
 )