1
0
Fork 0

personal/q3k: cleanup script for b/32 (owncloud postgres)

Change-Id: I7a330d460763d99bfbd736cecae33c0df7a41aae
master
q3k 2021-07-10 16:53:41 +00:00
parent f97c9688d5
commit 2f712bf531
3 changed files with 218 additions and 0 deletions

View File

@ -0,0 +1,10 @@
load("@rules_python//python:defs.bzl", "py_binary")
load("@pydeps//:requirements.bzl", "requirement")
py_binary(
name = "cleanup",
srcs = ["cleanup.py"],
deps = [
requirement("psycopg2"),
],
)

View File

@ -0,0 +1,150 @@
# Script to attempt to clean up our owncloud database (b/32) after The Postgres
# Fuckup (b/30).
#
# Think of it as a one-shot fsck, documented in the form of the code that q3k@
# used to recover from this kerfuffle.
#
# SECURITY: It's full of manual SQL query crafting without parametrization.
# Don't attempt to use it for anything else other than this one-shot usecase.
#
# You will need to tunnel to the postgreses running on Boston:
# $ ssh \
# -L15432:127.0.0.1:5432 \
# -L15433:127.0.0.1:5433 \
# hackerspace.pl
from datetime import datetime
import os
import psycopg2
incident_start = 1611529200 # when pg12 started to run
incident_end = 1611788400 # when we rolled back to pg9
OWNCLOUD_PASSWORD = os.environ.get("OWNCLOUD_PASSWORD").strip()
if not OWNCLOUD_PASSWORD:
# Get it from boston, /var/www/owncloud/config/config.php.
raise Exception("OWNCLOUD_PASSWORD must be set to owncloud postgres password")
conn9 = psycopg2.connect(host="localhost", port=15432, user="owncloud", password=OWNCLOUD_PASSWORD, dbname="owncloud")
conn12 = psycopg2.connect(host="localhost", port=15433, user="owncloud", password=OWNCLOUD_PASSWORD, dbname="owncloud")
def idset(conn, table, keyname="id"):
"""Return a set of IDs from a given table, one per row."""
cur = conn.cursor()
cur.execute(f"SELECT {keyname} FROM oc_{table}")
res = cur.fetchall()
cur.close()
return set([r[0] for r in res])
def valset(conn, table, keys):
"""Return a set of concatenated values for the given keys in a table, one per row."""
keynames = ", ".join(keys)
cur = conn.cursor()
cur.execute(f"SELECT {keynames} FROM oc_{table}")
res = cur.fetchall()
cur.close()
res = [';;;'.join([str(elem) for elem in r]) for r in res]
return set(res)
# Check accounts difference.
#
# RESULT: Thankfully, no accounts have been accidentally roled back.
accounts12 = idset(conn12, "accounts", keyname="uid")
accounts9 = idset(conn9, "accounts", keyname="uid")
print("Accounts missing in 9:", accounts12 - accounts9)
assert (accounts12 - accounts9) == set()
def account_by_uid(conn, uid):
"""Return SSO UID for a given Owncloud UID."""
cur = conn.cursor()
cur.execute(f"SELECT ldap_dn FROM oc_ldap_user_mapping WHERE owncloud_name = '{uid}'")
dn, = cur.fetchone()
cur.close()
part = dn.split(',')[0]
assert part.startswith('uid=')
return part[4:]
def storage_owner_by_id(conn, id_):
"""Return SSO UID for a given storage numerical ID."""
cur = conn.cursor()
cur.execute(f"SELECT id FROM oc_storages WHERE numeric_id = '{id_}'")
oid, = cur.fetchone()
cur.close()
if oid == 'object::store:amazon::nextcloud':
return "S3"
assert oid.startswith('object::user:')
userid = oid[13:]
assert len(userid) > 0
if userid == "gallery":
return "GALLERY"
return account_by_uid(conn, userid)
# Check shares table. This table contains the intent of sharing some file with someone else.
#
# RESULT: we only have things that have been removed after rollback to PG9,
# nothing was created in PG12 and lost.
shareids12 = idset(conn12, "share")
shareids9 = idset(conn9, "share")
print("Shares missing in 9:", len(shareids12 - shareids9))
cur12 = conn12.cursor()
for id_ in list(shareids12-shareids9):
cur12.execute(f"SELECT uid_owner, file_target, stime, share_with FROM oc_share WHERE id = {id_}")
uid_owner, file_target, stime, share_with = cur12.fetchone()
account = account_by_uid(conn12, uid_owner)
stime_human = datetime.utcfromtimestamp(stime).strftime('%Y-%m-%d %H:%M:%S')
print(f"Missing share {id_} {file_target} owned by {account}..")
if stime < incident_start or stime > incident_end:
print(f" Skipping, created at {stime_human}")
continue
raise Exception("Unhandled.")
cur12.close()
# Check mounts table. This contains root file storages for each user, but also
# incoming shares 'mounted' into a user's account.
# From what I cen tell, storage_id/root_id are the source path that's being
# mounted (root_id being the fileid inside an oc_filecache, and storage_id
# being the storage in which that file is kept), while user_id/mount_point are
# the mount destination (ie. path into which this is mounted for a user's
# view).
#
# RESULT: we only have share-mounts missing for a handful of users. We choose
# to ignore it, as we assume next time these users log in they will get the
# mounts again.
# TODO(q3k): verify this
mounts12 = valset(conn12, "mounts", ["storage_id", "root_id", "user_id", "mount_point"])
mounts9 = valset(conn9, "mounts", ["storage_id", "root_id", "user_id", "mount_point"])
print("Mounts missing in 9:", len(mounts12 - mounts9))
# Mounts that appearify normally whenever you log into owncloud, as they are the result of shares':
mount_names_ok = set(["2020-03-26_covid_templar", "camera", "Public Shaming", "przylbice.md", "Test.txt", "covid"])
# Mounts that used to be from a share that existed, but has been since deleted in PG9.
mount_names_ok |= set(["Covid-instrukcje", "Chaos_modele_covid", "Covid_proces_presspack"])
mounts_sorted = []
for m in list(mounts12 - mounts9):
storage_id, root_id, user_id, mount_point = m.split(';;;')
mounts_sorted.append((storage_id, root_id, user_id, mount_point))
mounts_sorted = sorted(mounts_sorted, key=lambda el: el[2])
for storage_id, root_id, user_id, mount_point in mounts_sorted:
assert mount_point.startswith("/" + user_id + "/")
mount_point = mount_point[len(user_id)+1:]
account = account_by_uid(conn12, user_id)
print(f"Missing mount {mount_point}, storage ID {storage_id}, owned by {account}..")
storage_owner = storage_owner_by_id(conn12, storage_id)
print(f" Storage owner: {storage_owner}")
parts = mount_point.split('/')
if len(parts) == 4 and parts[0] == '' and parts[1] == 'files' and parts[2] in mount_names_ok and parts[3] == '':
print(" Skipping, known okay")
continue
raise Exception("Unhandled")

View File

@ -0,0 +1,58 @@
-----BEGIN PGP MESSAGE-----
hQEMAzhuiT4RC8VbAQf9Ecn9tDsi84AKCyPySLpGFBj5Fp8Rc/9b3RA5f4614tqE
2LKn5UQP7Ejg6LzCZEBlplu4CFBM5fVe+sx0pZNTVdi+qOPFYB4ruV0TLLjacaAY
6hyD7mzmMEWVhHYHJpqCwUV8Vx7vN/6SG91nObCuEjbfYrAXjwdiXvDiBumH6d9O
N5CTh5EYVqazZD4SvSXYPyG/iv6/1nrxlfYA/LxD+ULVPhjKBboNTjfVtjAhQVvD
ZRMl4/rP1/WXRXz7svGaENTEG6TQ95ZrnSYPZQD+amWKumRiCdneN6I3N4s2F9dJ
+FP5WJ214rUAx/rusf+Gt5v2FPyfNw6QJR41WFrjhIUBDANcG2tp6fXqvgEH/0pJ
Hlwe6F1ltWm5gdhUUJ5+/tJQH3e4cq0EI+26dENE9633pVA8ERFY3zmcTcxOZ04k
K9r8+ifiagdIvWldNLKXrHaNxZWM+r4fJ1RTo0x/gskid8e9otdxo9kR8t3yh9SZ
DHRSlCKr2+/H167h89dPWtJh5meUPWAlw2Zmcl2GDue5zjDVrHEkZ8fbxOwf9E/o
J93oaux9Ijfrp8tP7lO90qAoaXvTAMeI7hnkWHnX2akcVh03U4FgDqLpZSlVCRP/
mIFXRnaLuZEFdmrO+raZUi58t8xaadTZy9hsmCtlsgSKIwgA8zsF2CmJCRvZD95w
Q70vLg91E7Bfyr1duhKFAgwDodoT8VqRl4UBEACoBXEXQaxVvZ3vClxUkxrq468/
YC6NyTXWOt9KvpuGUWCtbFIQQ4CSfRu8UpvPasppY5shLE4L+0f79OV8aQfV4mF+
EdHZ0zCWuVIBcYh+iiAsz5zYpbB7vSW+J+DG7ZH0pCVJZ8CSTJhK42BmyDh0a6ut
glHqLQaeC6dPur5To+C+Ozl8v6GduJDzZQ5ERCaML0nhauH0yoEe3My52BlEZ9MK
mjJlm3Q3VCYTfT8M08ZLTART6zMotYTemE/u+U8rVVPPDY+7kgy8yt44QAoUzE49
shr5llK9GBjDeo4URf//0KvLs95H4TdGVYMfELcFyqyCs0YJ2vee2KvoyuXPGmkO
6kuASSgGn5zFqdX13P0y4QIC5OxWnX22CxMdXpvG18RqCW2qyMT3wscqTCYuIHyc
yPZWfi+MM1tDB+CdQbFLus5gWPUSZ8kK+sur/5lb0ToamWvYHhNTKMOBh8MCODAy
F6lxrUIc81uojF8VKbgK+lOyQUtL4R3+1Wbk+9TyBYUJNjMi/S2mpL1bOUkKZkpb
hmqloSPP7W8xa9JuxFDgyPLWpmGSJkDz+6MPiAPfPBgmczUORNuGReck4SDYlZib
SRWrhs54zkmlZPmFCEoqbLqyFfQiNqLhQ8J/ya//HZuXAUNd9612P2a+idAAE6gx
pz54H+jeH9o8Db01BYUCDAPiA8lOXOuz7wEP/1Uw8k0t4kPrkD1hZ0p2KFklyM6O
Ri3j1Y2IcYcQZ23OKMx5qo90/aLBzYSRtA/NHWuqcaDjudyFJ12lSTNyQX0sFnUP
nbig6smYzOu3XnkTnRBrOe4YN5YJiyUFTsK5wPcUcArCuLASvRCxzkwHyTQTnW6Y
r92oArKEzXzE8sYFMPpRYpV29sQgqXUBEq0bA9codN1Z1m5N3aGvMiYyimq4jXoq
Va+Tsry5KON2S0/h8UZsLnY/USSXjWdhb266tU9MLgY2EIK9DaTfU6mxsMTLysVQ
RhBmtHQhzczkfueMYa7KigbXJNxvEjUlR22RVRiH9F3lhhismsW1xtgfI/IlGmQx
6uhCFMDIsgZh39kWRP4vUxzWTvnPSD76omBcdjVTKGDEd8vqEwIBeORg9E6NoN+Q
8HR6Fb6y0pAk30VO1mP3xC0Li9q13ips1p1w+Xu9WOPFVEwJaSFn1oaEWTuQttn3
gPdng35LjYLnch588exe1bhoj6WiUaCclZF5yjewMCGowlvCO05QEyivRj2YGeHS
D+oH+dv/Ex0PIJR/8P+clAcB/u3qQl2W40pPjHOmMGb8Gi+GSVU5HX0lENdmMrfw
QfZuqKh08j8gVK77yX0UjtNhN0XVu5toCncgSFLxZSkQd4opZTonmdji0vwSapx2
FbPI2PVqRRWjknuj0ukBu3zMhA+90PvWjmWxaSVBs0FDOvOApYUf7QwYuVpk/hkN
GFqrarltId1Xy2UQSLHdKgb1fj4OhnVSApK9cKxD0mjJ87QibQFLeR2KCIOX4/EJ
qAuC7xV8VTtzFQqOYF1RrXN8NbU/htMAecKYX3mj3S9FXgOUHEj/fYdFHuFhVhdw
onJZ1CReuQX6nGU6c7bCKj7mvt/QISq7eiPT2zqt+f0X+Bz6ODkk/5QHyu2mQUTt
PQOayDWpi5vcNsiMj8DLR7nLjErHau/joIyRMopssYzdDb5d2tsofIjoL9VzYbUu
PTs/VJCIJ7XvRt1SA0pyQ9JUrvNhskz095CVuPF1LoA8HlcHWCUhiT19bWegsYIu
rFqglNCrNuN2t0mQpeA9108EB8m5bmaEK6tlepwhqno3S35KnYGT+FZTo8A3xt38
sOATC71bTPyLSNklXK+7t4YPD5nulINNLDqoxUe7ruCwZRkiWZHbp0+AON/jk5CZ
O9pNUPA524+sLZzY1XGumifv+f7H4vExOhAMWsPAVfwBdwW4//wIeafV65RMsEUD
SzMUfdNQVvf6w1YP3MpDvCHOLlCTrMf8fKKQAkT6Dj/nGHt5bVA10IYY/jaIdXj0
VsONFWWaqKxmh9kVGmHdjvZ/tnDFXRPXS+3ddA/jqfVt2JzebHBDeropYYmDXMEe
wA4Nbl5qjSoxFsmVMNROIXMrMv66LVG3kVxuHUix147qix8JTOWCFlgaBBvp16Dv
8k52poapN6QnMnR12QWxZyroZEeTV4RvNCB/QdoVHEcx8/XBmE3psSceUh9GNU6S
BZsb+68KosvRV9bFDcg0DJ+Qyp7k25F4+ItdSxIceW0phAQft9GJafehvnXaQj9A
vRLuLTtM37QL8YX7vs5DKJAzG7RCgmrUfbVS6BnWhLAaUWTFgQXHd9gtw2XCkb3y
GUCztuO3t5zRwxTlvXWt1KBdLvIm4xrmU/yfuUOK2eLmwdH+rqouVUW1fRrTXhZD
OGyvSLIB/kujiukjIJ4idBImzmJcqMewdEdiYw/zsns3RZrfFop1IBZ8fpdAXY4m
S0j9Yhy9qqyZ+h9ZYFG5vK6xdevaeqIGGpc7Zk7xTZjXrA+kbrjZyJa6LMUF9dFH
MJx4tytpu546euLWP2t9OEu7T+0b9sOrQCrNNIS+qDPN37FNzvxOwCf+i5xr00nI
7ifX1c5tA/K6b/IhDVcACqfnUAcgPg+S2qkRtleATzKE0g2ISozbw2/LNb/Th3L1
/fH2VmHZyTHJg06PpLsaqnNFpiwIS3Kb2RKtgo3ZNLg+S6QVUd90wEMEX+cdgg==
=aO5Z
-----END PGP MESSAGE-----