Update fetcher to use the new IB history form and new CSV format, add the --no-action flag
Remigiusz Marcinkiewicz
parent
e1552eac06
commit
968803ca77
|
|
@ -88,56 +88,94 @@ class IBRow(RawTransfer):
|
|||
|
||||
def __init__(self, row, on_account, raw):
|
||||
self.raw = raw
|
||||
self.uid = row[IBField.uid]
|
||||
self.index = 1
|
||||
self.on_account = on_account
|
||||
self.date = datetime.strptime(row[IBField.date_completed], "%d.%m.%Y").date()
|
||||
self.to_account = IBParser.parse_account_number(row[IBField.to_account])
|
||||
self.to_name = row[IBField.to_name]
|
||||
self.from_account = IBParser.parse_account_number(row[IBField.from_account])
|
||||
self.from_name = row[IBField.from_name]
|
||||
self.date = datetime.strptime(row[IBField.date_completed], "%Y%m%d").date()
|
||||
self.title = row[IBField.title]
|
||||
|
||||
af = re.compile(r"([0-9]+)\.([0-9]{2})([A-Z]+)")
|
||||
m = af.match(''.join(row[IBField.amount].split()))
|
||||
af = re.compile(r"([0-9]+)\.([0-9]{2})")
|
||||
m = af.match(row[IBField.amount])
|
||||
if m is None:
|
||||
raise IBParseError("Can't parse amount value \"{}\"".format(row[IBField.amount]), row)
|
||||
a,b,c = m.groups()
|
||||
a,b = m.groups()
|
||||
self.amount = int(a)*100+int(b)
|
||||
self.currency = c
|
||||
m = af.match(''.join(row[IBField.balance].split()))
|
||||
if m is None:
|
||||
raise IBParseError("Can't parse balance value \"{}\"".format(row[IBField.balance]), row)
|
||||
a,b,c = m.groups()
|
||||
self.balance = int(a)*100+int(b)
|
||||
self.balance_currency = c
|
||||
self.currency = row[IBField.currency]
|
||||
|
||||
if self.from_account == self.to_account:
|
||||
self.type = "BANK_FEE"
|
||||
elif self.from_account in self.OWN_ACCOUNTS and self.to_account in self.OWN_ACCOUNTS:
|
||||
if self.to_account == self.on_account:
|
||||
self.type = "OUT_FROM_OWN"
|
||||
else:
|
||||
self.type = "OUT_TO_OWN"
|
||||
elif self.from_account == self.on_account:
|
||||
self.type = "OUT"
|
||||
elif self.to_account == self.on_account:
|
||||
own_account = IBParser.parse_account_number(row[IBField.own_account])
|
||||
own_name = "Stowarzyszenie \"Warszawski Hackerspace\""
|
||||
|
||||
if own_account not in self.OWN_ACCOUNTS:
|
||||
raise IBParseError("own_account {} not in OWN_ACCOUNTS - format change?".format(own_account))
|
||||
self.on_account = own_account
|
||||
|
||||
other_account = IBParser.parse_account_number(row[IBField.other_account])
|
||||
other_name = row[IBField.other_name]
|
||||
|
||||
|
||||
direction = row[IBField.direction]
|
||||
if direction == "uznanie":
|
||||
direction = "IN"
|
||||
self.type = "IN"
|
||||
elif direction == u"Obiciążenie": # sic!
|
||||
direction = "OUT"
|
||||
self.type = "OUT"
|
||||
else:
|
||||
raise IBParseError("Can't parse direction specifier \"{}\"", direction)
|
||||
|
||||
if own_account == other_account:
|
||||
self.type = "BANK_FEE"
|
||||
self.from_account = self.to_account = own_account
|
||||
self.from_name = self.to_name = own_name
|
||||
elif own_account in self.OWN_ACCOUNTS and other_account in self.OWN_ACCOUNTS:
|
||||
self.from_name = self.to_name = own_name
|
||||
if direction == "IN":
|
||||
self.type = "IN_FROM_OWN"
|
||||
self.from_account = other_account
|
||||
self.to_account = own_account
|
||||
elif direction == "OUT":
|
||||
self.type = "OUT_TO_OWN"
|
||||
self.from_account = own_account
|
||||
self.to_account = other_account
|
||||
else:
|
||||
raise IBParseError("Can't figure out details of an own-to-own transfer")
|
||||
elif direction == "IN":
|
||||
self.type = "IN"
|
||||
self.from_account = other_account
|
||||
self.to_account = own_account
|
||||
self.from_name = other_name
|
||||
self.to_name = own_name
|
||||
elif direction == "OUT":
|
||||
self.type = "OUT"
|
||||
self.from_account = own_account
|
||||
self.to_account = other_account
|
||||
self.from_name = own_name
|
||||
self.to_name = other_name
|
||||
else:
|
||||
raise IBParseError("Can't figure out transfer type for current row", row)
|
||||
|
||||
self.uid = hashlib.sha256(self.SECRET + str(self)).hexdigest()
|
||||
if None in (self.type, self.to_account, self.from_account, self.to_name, self.from_name):
|
||||
raise IBParseError("Something went wrong - one of the mandatory values empty")
|
||||
|
||||
class IBField(enum.Enum):
|
||||
from_name = u"Nadawca"
|
||||
from_account = u"Rachunek nadawcy"
|
||||
title = u"Tytułem"
|
||||
to_name = u"Odbiorca"
|
||||
to_account = u"Rachunek odbiorcy"
|
||||
date_issued = u"Data złożenia dyspozycji"
|
||||
#Data waluty;Data zlecenia;Numer rachunku nadawcy;Numer banku nadawcy;Kwota w walucie rachunku;Waluta;Kurs;Kwota w walucie zlecenia;Numer rachunku odbiorcy;Odbiorca;Numer banku odbiorcy;Tytuł;Obciążenie/uznanie;Numer transakcji w systemie centralnym;
|
||||
date_completed = u"Data waluty"
|
||||
amount = u"Kwota operacji"
|
||||
balance = u"Saldo po operacji"
|
||||
|
||||
date_issued = u"Data zlecenia"
|
||||
own_account = u"Numer rachunku nadawcy"
|
||||
own_bank = u"Numer banku nadawcy"
|
||||
|
||||
amount = u"Kwota w walucie rachunku"
|
||||
currency = u"Waluta"
|
||||
rate = u"Kurs"
|
||||
transfer_amount = "Kwota w walucie zlecenia"
|
||||
|
||||
other_account = u"Numer rachunku odbiorcy"
|
||||
other_name = u"Odbiorca"
|
||||
other_bank = u"Numer banku odbiorcy"
|
||||
|
||||
title = u"Tytuł"
|
||||
direction = u"Obciążenie/uznanie"
|
||||
uid = u"Numer transakcji w systemie centralnym"
|
||||
|
||||
class IBParser(object):
|
||||
def __init__(self, account_number):
|
||||
self.account_number = account_number
|
||||
|
|
@ -145,8 +183,15 @@ class IBParser(object):
|
|||
self.fields = []
|
||||
|
||||
def parse(self, snapshot):
|
||||
kek = u"IMPLR - STARVING - SKŁADKA ;".encode("utf-8")
|
||||
if snapshot.find(kek) == -1:
|
||||
raise IBParseError("double ; not found, format changed?")
|
||||
snapshot = snapshot.replace(kek, kek[:-1])
|
||||
lines = snapshot.splitlines()
|
||||
header = lines.pop(0).decode("utf-8").split(";")
|
||||
if not header[-1] == "":
|
||||
raise IBParseError("Last column no longer empty?")
|
||||
header = header[:-1]
|
||||
|
||||
for hf in header:
|
||||
try:
|
||||
|
|
@ -154,8 +199,11 @@ class IBParser(object):
|
|||
except ValueError as e:
|
||||
raise IBParseError("Unexpected field name \"{}\"".format(hf),e)
|
||||
|
||||
|
||||
|
||||
c = csv.reader(reversed(lines), delimiter=";")
|
||||
for row in c:
|
||||
row = row[:-1]
|
||||
if not len(row) == len(self.fields):
|
||||
raise IBParseError("Row has {} fields, {} expected after parsing the header: \"{}\"".format(len(row), len(self.fields), ';'.join(row)))
|
||||
d = dict(zip(self.fields, [r.decode("utf-8") for r in row]))
|
||||
|
|
@ -364,32 +412,43 @@ class IBFetcher(object):
|
|||
return self.process_wallet_page(wallet_page)
|
||||
|
||||
def fetch_account_history(self, account_id):
|
||||
account_page = self._get("accounts/index/{}/2".format(account_id))
|
||||
self._wait(4)
|
||||
#account_page = self._get("accounts/index/{}/2".format(account_id))
|
||||
#self._wait(4)
|
||||
|
||||
#data = {
|
||||
#"code": account_id,
|
||||
#"basic": 1,
|
||||
#"date_from": self.START_DATE,
|
||||
#"date_to": '{:02d}.{:02d}.{:04d}'.format(date.today().day, date.today().month, date.today().year),
|
||||
#"interval_time": "",
|
||||
#"interval_type": "",
|
||||
#"last": "",
|
||||
#"advanced[0]": "0",
|
||||
#"advanced[1]": "1",
|
||||
#"operation_type": "3",
|
||||
#"amount_from": "",
|
||||
#"amount_to": "",
|
||||
#"transaction_type": "",
|
||||
#"from": "",
|
||||
#"title": "",
|
||||
#"send": "send",
|
||||
#"ajaxSend": "true"
|
||||
#}
|
||||
#history_page = self._post("accounts/history/{}".format(account_id), data)
|
||||
#self._wait(2)
|
||||
|
||||
#r = self._getraw("accounts/printHistoryFile")
|
||||
|
||||
data = {
|
||||
"code": account_id,
|
||||
"basic": 1,
|
||||
"date_from": self.START_DATE,
|
||||
"date_to": '{:02d}.{:02d}.{:04d}'.format(date.today().day, date.today().month, date.today().year),
|
||||
"interval_time": "",
|
||||
"interval_type": "",
|
||||
"last": "",
|
||||
"advanced[0]": "0",
|
||||
"advanced[1]": "1",
|
||||
"operation_type": "3",
|
||||
"amount_from": "",
|
||||
"amount_to": "",
|
||||
"transaction_type": "",
|
||||
"from": "",
|
||||
"title": "",
|
||||
"send": "send",
|
||||
"ajaxSend": "true"
|
||||
"report_type": "csv_dr",
|
||||
"start_date": self.START_DATE,
|
||||
"end_date": '{:02d}.{:02d}.{:04d}'.format(date.today().day, date.today().month, date.today().year),
|
||||
"banking": self.token
|
||||
}
|
||||
history_page = self._post("accounts/history/{}".format(account_id), data)
|
||||
self._wait(2)
|
||||
|
||||
r = self._getraw("accounts/printHistoryFile")
|
||||
r = self._postraw("accounts/getHistoryDailyReportsFile", data)
|
||||
|
||||
return r.content.decode("utf-8-sig").encode("utf-8")
|
||||
|
||||
def usage():
|
||||
|
|
@ -419,7 +478,7 @@ def release():
|
|||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
opts, args = getopt(sys.argv[1:], "hcl:", ["help", "cached", "load=", "print-schema"])
|
||||
opts, args = getopt(sys.argv[1:], "hcl:n", ["help", "cached", "load=", "no-action", "print-schema"])
|
||||
except GetoptError as err:
|
||||
# print help information and exit:
|
||||
print str(err) # will print something like "option -a not recognized"
|
||||
|
|
@ -431,6 +490,7 @@ if __name__ == "__main__":
|
|||
session = sessionmaker(bind=engine)()
|
||||
|
||||
cached = False
|
||||
noaction = False
|
||||
load_files = {}
|
||||
for o, a in opts:
|
||||
if o in ("-h", "--help"):
|
||||
|
|
@ -460,6 +520,9 @@ if __name__ == "__main__":
|
|||
load_files[account_number] = history
|
||||
cached = True
|
||||
print "[i] Loading \"{}\" as \"{}\"".format(f, account_number)
|
||||
elif o in ("-n", "--no-action"):
|
||||
print "[i] Called with --no-action, will not do any database operations."
|
||||
noaction = True
|
||||
else:
|
||||
assert False, "unhandled option"
|
||||
|
||||
|
|
@ -512,11 +575,13 @@ if __name__ == "__main__":
|
|||
stats[account_number]["added"] = 0
|
||||
stats[account_number]["skipped"] = 0
|
||||
for row in parser.get():
|
||||
if not session.query(IBRow).filter_by(uid=row.uid).first():
|
||||
if not session.query(IBRow).filter_by(uid=row.uid).first() and not noaction:
|
||||
session.add(row)
|
||||
stats[account_number]["added"] += 1
|
||||
else:
|
||||
stats[account_number]["skipped"] += 1
|
||||
if noaction:
|
||||
print "[i] --no-action set, skipping row {}".format(row)
|
||||
session.commit()
|
||||
|
||||
print "[i] Done: ", stats
|
||||
|
|
|
|||
Loading…
Reference in New Issue