hswaw/kasownik
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

backing-pekaobiznes: rework CLI tool

Browse source
This commit is contained in:
informatic 2022-01-01 14:58:58 +01:00
parent d81adaeadb
commit 7e440678d0
2 changed files with 181 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

205
fetch/banking-pekaobiznes.py
View file

@ -1,5 +1,11 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p python3 python3Packages.requests python3Packages.requests-cache python3Packages.beautifulsoup4
import argparse
import os
import configparser
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import requests
import sys
import hashlib
@ -16,7 +22,8 @@ from urllib.parse import urljoin, urlparse, parse_qs
from bs4 import BeautifulSoup
from binascii import unhexlify
from models import RawTransfer
from models import RawTransfer, get_schema
# loginPass = string zbudowany z wyswietlonych inputów, * w zablokowanycha
# loginMaskArray = hex2bytes ze zmiennej "loginMask"
@ -117,6 +124,7 @@ class CAMT052Parser:
transfer = RawTransfer()
transfer.index = 1
transfer.uid = txdtls.find("ns:Refs", ns).find("ns:TxId", ns).text
transfer.on_account = on_account
transfer.raw = ET.tostring(entry).decode()
transfer.amount = int(Decimal(amt.text) * 100)
@ -127,6 +135,7 @@ class CAMT052Parser:
).date()
transfer.title = txdtls.find("ns:RmtInf", ns).find("ns:Ustrd", ns).text
# TODO: OUT_TO_OWN, IN_FROM_OWN
if tx_type == "DBIT":
transfer.type = "BANK_FEE" if remote_party_acct is None else "OUT"
transfer.to_account = (
@ -146,18 +155,6 @@ class CAMT052Parser:
)
transfer.from_name = remote_party_info
print(
tx_type,
amt.attrib["Ccy"],
int(Decimal(amt.text) * 100),
entry.find("ns:BookgDt", ns).find("ns:DtTm", ns).text,
txdtls.find("ns:Refs", ns).find("ns:TxId", ns).text,
remote_party_acct,
txdtls.find("ns:RmtInf", ns).find("ns:Ustrd", ns).text,
"|||",
remote_party_info,
)
yield transfer
@ -194,12 +191,12 @@ class PekaoClient:
"Mozilla/5.0 (X11; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0",
)
def login(self):
def login(self, alias, password):
self._go("https://www.pekaobiznes24.pl/do/login")
self._submit_form(
"LoginAliasForm",
{
"p_alias": self.config["alias"],
"p_alias": alias,
"deviceFingerprint": self.config["tdid"],
},
)
@ -211,7 +208,7 @@ class PekaoClient:
"MaskLoginForm",
{
"p_passmasked_bis": mask_password(
self.config["password"], login_mask, self.config["alias"]
password, login_mask, alias,
)
},
)
@ -310,7 +307,7 @@ class PekaoClient:
self.resp = self.session.request(method, url, **args)
self.resp.raise_for_status()
self.logger.debug("=> %s %s", method, self.resp.url)
self.bs = BeautifulSoup(self.resp.text)
self.bs = BeautifulSoup(self.resp.text, features='html.parser')
def _submit_form(self, name, values):
form = self.bs.find("form", {"name": name})
@ -321,28 +318,164 @@ class PekaoClient:
data = {**form_data, **values}
target = urljoin(self.resp.url, form.get("action"))
self._go(target, form.get("method").upper(), data=data)
print(target, data)
def lock(fn):
if os.path.isfile(fn):
logging.error("Lock file %s exists, aborting", fn)
sys.exit(3)
logging.debug("Setting up lock file %s", fn)
open(fn,'w').close()
if not os.path.isfile(fn):
logging.error("Lock file %s somehow does not exist, aborting", fn)
sys.exit(3)
def release(fn):
logging.debug("Removing lock file %s", fn)
if not os.path.isfile(fn):
logging.error("Lock file %s somehow does not exist, WTF?", fn)
sys.exit(3)
os.remove(fn)
if os.path.isfile(fn):
logging.error("Lock file %s somehow still exists, WTF?", fn)
sys.exit(3)
parser = argparse.ArgumentParser()
parser.add_argument('--config', help="Load configuration file", default="config.ini")
parser.add_argument('-n', '--no-action', action="store_true", help='do not commit any database changes')
parser.add_argument('-c', '--cached', action="store_true", help='use cached data (test)')
parser.add_argument('-l', '--load', action='append', help='process specified files (test)')
parser.add_argument('-t', '--token', help='use authentication token')
parser.add_argument('--no-lock', action='store_true', help='don\'t use lockfile (test)')
parser.add_argument('--print-schema', action="store_true", help='print table schema and quit')
if __name__ == "__main__":
with open(sys.argv[1]) as fd:
c = PekaoClient(json.load(fd))
c.login()
accounts = c.list_accounts()
args = parser.parse_args()
for a, info in accounts.items():
print(
r"[{p_acc_id}] {p_acc_no} {p_acc_avail_balance: >10} {p_acc_currency}".format(
**info
)
)
config = configparser.ConfigParser()
config.read(args.config)
for a, info in accounts.items():
print("*** Fetching transfers for", info["p_acc_no"], info["p_acc_alias"])
transfers = c.fetch_transfers_camt052(a)
if transfers is None:
print("No transfers found")
logging.basicConfig(level=config['logging']['level'], format=config['logging'].get('format', '%(asctime)s [%(levelname)s] %(name)s: %(message)s'))
logging.getLogger('chardet').setLevel(logging.WARN)
CACHE_DIR = config['general']['cache_dir']
engine = create_engine(config['database']['uri'])
session = sessionmaker(bind=engine)()
if args.print_schema:
logging.debug("Called with --print-schema, will print the create " +
"statement and quit.")
print(get_schema(engine))
sys.exit()
if not args.no_lock:
lock(config['general']['lockfile'])
balances = {}
history_logs = {}
if args.load:
logging.debug("Using manually supplied files")
for fn in args.load:
an, f = fn.split(':')
account_number = IBParser.parse_account_number(an)
if account_number is None:
logging.error("File name number \"{}\" unparseable".format(f))
continue
logging.debug('Loading "%s" as "%s"', f, account_number)
with open(f, 'r') as fd:
history_logs[account_number] = json.loads(fd.read())
elif args.cached:
logging.debug("Loading cached files from {}".format(CACHE_DIR))
for f in os.listdir(CACHE_DIR):
if f.startswith('balance-'):
continue
account_number = CAMT052Parser.parse_account_number(f)
if account_number is None:
logging.error("File name number \"{}\" unparseable".format(f))
continue
with open(CACHE_DIR + "/" + f, 'r') as fd:
history_logs[account_number] = fd.read()
logging.debug("Loading \"{}\" as \"{}\"".format(f, account_number))
else:
logging.debug("Normal run - will connect to the bank")
fetcher = PekaoClient(config['scraper'])
if args.token:
fetcher.token = args.token
logging.debug("Using provided token")
elif "alias" not in config['scraper'] or "password" not in config['scraper']:
fetcher.login(input("[?] ID: "), input("[?] Password: "))
else:
logging.debug("Using saved credentials")
fetcher.login(config["scraper"]['alias'], config["scraper"]['password'])
accounts = fetcher.list_accounts()
for account_id, account in accounts.items():
account_number = CAMT052Parser.parse_account_number(account['p_acc_no'])
logging.debug("Fetching history for account {} ({}) {}".format(
account_number, account_id, account["p_acc_alias"],
))
history = fetcher.fetch_transfers_camt052(account_id)
history_logs[account_number] = history
with open(CACHE_DIR + "/" + account_number, 'w') as fd:
fd.write('' if history is None else history)
balances[account_number] = (
account["p_acc_avail_balance"], account["p_acc_currency"])
with open(CACHE_DIR + "/balance-"+account_number, 'w') as fd:
fd.write("{} {}\n".format(*balances[account_number]))
if not history_logs:
logging.error('Nothing to process')
sys.exit()
parsed = {}
stats = {}
for account_number, history in history_logs.items():
logging.debug("Parsing history for account {}".format(account_number))
if not history:
logging.debug('No transfers for that account, continuing...')
continue
parser = CAMT052Parser(history)
rows = parser.parse()
stats[account_number] = {}
stats[account_number]["added"] = 0
stats[account_number]["skipped"] = 0
for row in rows:
if not session.query(RawTransfer).filter_by(uid=row.uid).first():
session.add(row)
stats[account_number]["added"] += 1
else:
parser = CAMT052Parser(transfers)
for transfer in parser.parse():
print(transfer)
stats[account_number]["skipped"] += 1
if args.no_action:
logging.info('Running with --no-action, not commiting.')
else:
session.commit()
# That is pretty ugly, but the only alternative would be to change handler
# level in runtime, and that'd still need some rollback anyway.
if any(v['added'] for v in stats.values()):
log_summary = logging.info
else:
log_summary = logging.debug
if balances:
log_summary("Account balances:")
for account_number, v in balances.items():
balance, currency = v
log_summary("\t{}: {} {}".format(
account_number, balance, currency))
log_summary("Done: %r", stats)
if not args.no_lock:
release(config['general']['lockfile'])

13
fetch/models.py
View file

@ -43,7 +43,7 @@ class RawTransfer(Base):
def __str__(self):
return u'{} *{} #{} @{} -"{}" -#{} => +"{}" +#{} [{}.{:02d} {}] ~"{}"'.format(
self.type,
self.index,
self.uid,
self.on_account,
self.date,
self.from_name,
@ -58,3 +58,14 @@ class RawTransfer(Base):
def __repr__(self):
return "<Transfer %s>" % (str(self),)
def get_schema(engine):
schema = ""
m = MetaData()
schema += "%s;\n" % (CreateTable(RawTransfer.__table__).compile(engine),)
for index in RawTransfer.__table__.indexes:
schema += "%s;\n" % (CreateIndex(index).compile(engine),)
return schema