diff --git a/bin/import-sources b/bin/import-sources new file mode 100755 index 0000000..2b81779 --- /dev/null +++ b/bin/import-sources @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import argparse + +import vanth.main +import vanth.ofxhome +import vanth.platform.ofxsource + + +def main(): + vanth.main.setup_logging() + config = vanth.main.get_config() + vanth.main.create_db_connection(config) + + parser = argparse.ArgumentParser() + parser.add_argument('dbfile', help='The database file of XML dumped from the open OFX Home DB') + args = parser.parse_args() + + with open(args.dbfile, 'r') as f: + data = vanth.ofxhome.parse(f.read()) + vanth.platform.ofxsource.ensure_exist(data) + +if __name__ == '__main__': + main() diff --git a/vanth/ofxhome.py b/vanth/ofxhome.py new file mode 100644 index 0000000..99d154b --- /dev/null +++ b/vanth/ofxhome.py @@ -0,0 +1,12 @@ +import xml.etree.ElementTree + + +def parse_child(element): + values = {child.tag: child.text for child in element} + values['id'] = element.attrib['id'] + return values + +def parse(data): + root = xml.etree.ElementTree.fromstring(data) + assert root.tag == 'institutions' + return [parse_child(element) for element in root] diff --git a/vanth/platform/ofxsource.py b/vanth/platform/ofxsource.py index 3d577ca..65fbada 100644 --- a/vanth/platform/ofxsource.py +++ b/vanth/platform/ofxsource.py @@ -1,6 +1,8 @@ import logging +import uuid import chryso.connection +import sqlalchemy import vanth.tables @@ -11,10 +13,32 @@ def _query_and_convert(query): results = engine.execute(query).fetchall() return [dict(result) for result in results] -def by_uuid(uuid): - query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid) +def by_uuid(uuid_): + query = vanth.tables.OFXSource.select().where(vanth.tables.OFXSource.c.uuid == uuid_) return _query_and_convert(query) def get(): query = vanth.tables.OFXSource.select() return _query_and_convert(query) + +def ensure_exist(institutions): + engine = chryso.connection.get() + query = sqlalchemy.select([ + vanth.tables.OFXSource.c.fid, + ]) + results = engine.execute(query).fetchall() + LOGGER.debug("Found %d OFX sources", len(results)) + known_records = {result[vanth.tables.OFXSource.c.fid] for result in results} + new_records = [institution for institution in institutions if institution['fid'] not in known_records] + LOGGER.debug("Have %d new transactions to save", len(new_records)) + to_insert = [{ + 'name' : institution['name'], + 'fid' : institution['fid'], + 'url' : institution['url'], + 'uuid' : str(uuid.uuid4()), + } for institution in new_records] + if to_insert: + engine.execute(vanth.tables.OFXSource.insert(), to_insert) # pylint: disable=no-value-for-parameter + LOGGER.debug("Done inserting %d records", len(new_records)) + else: + LOGGER.debug("Not performing insert, nothing to do")