forked from hswaw/hscloud
personal/q3k: cleanup script for b/32 (owncloud postgres)
Change-Id: I7a330d460763d99bfbd736cecae33c0df7a41aaemaster
parent
f97c9688d5
commit
2f712bf531
|
|
@ -0,0 +1,10 @@
|
|||
load("@rules_python//python:defs.bzl", "py_binary")
|
||||
load("@pydeps//:requirements.bzl", "requirement")
|
||||
|
||||
py_binary(
|
||||
name = "cleanup",
|
||||
srcs = ["cleanup.py"],
|
||||
deps = [
|
||||
requirement("psycopg2"),
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
# Script to attempt to clean up our owncloud database (b/32) after The Postgres
|
||||
# Fuckup (b/30).
|
||||
#
|
||||
# Think of it as a one-shot fsck, documented in the form of the code that q3k@
|
||||
# used to recover from this kerfuffle.
|
||||
#
|
||||
# SECURITY: It's full of manual SQL query crafting without parametrization.
|
||||
# Don't attempt to use it for anything else other than this one-shot usecase.
|
||||
#
|
||||
# You will need to tunnel to the postgreses running on Boston:
|
||||
# $ ssh \
|
||||
# -L15432:127.0.0.1:5432 \
|
||||
# -L15433:127.0.0.1:5433 \
|
||||
# hackerspace.pl
|
||||
|
||||
from datetime import datetime
|
||||
import os
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
incident_start = 1611529200 # when pg12 started to run
|
||||
incident_end = 1611788400 # when we rolled back to pg9
|
||||
|
||||
|
||||
OWNCLOUD_PASSWORD = os.environ.get("OWNCLOUD_PASSWORD").strip()
|
||||
if not OWNCLOUD_PASSWORD:
|
||||
# Get it from boston, /var/www/owncloud/config/config.php.
|
||||
raise Exception("OWNCLOUD_PASSWORD must be set to owncloud postgres password")
|
||||
|
||||
|
||||
conn9 = psycopg2.connect(host="localhost", port=15432, user="owncloud", password=OWNCLOUD_PASSWORD, dbname="owncloud")
|
||||
conn12 = psycopg2.connect(host="localhost", port=15433, user="owncloud", password=OWNCLOUD_PASSWORD, dbname="owncloud")
|
||||
|
||||
|
||||
def idset(conn, table, keyname="id"):
|
||||
"""Return a set of IDs from a given table, one per row."""
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {keyname} FROM oc_{table}")
|
||||
res = cur.fetchall()
|
||||
cur.close()
|
||||
return set([r[0] for r in res])
|
||||
|
||||
|
||||
def valset(conn, table, keys):
|
||||
"""Return a set of concatenated values for the given keys in a table, one per row."""
|
||||
keynames = ", ".join(keys)
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {keynames} FROM oc_{table}")
|
||||
res = cur.fetchall()
|
||||
cur.close()
|
||||
res = [';;;'.join([str(elem) for elem in r]) for r in res]
|
||||
return set(res)
|
||||
|
||||
|
||||
# Check accounts difference.
|
||||
#
|
||||
# RESULT: Thankfully, no accounts have been accidentally roled back.
|
||||
accounts12 = idset(conn12, "accounts", keyname="uid")
|
||||
accounts9 = idset(conn9, "accounts", keyname="uid")
|
||||
print("Accounts missing in 9:", accounts12 - accounts9)
|
||||
assert (accounts12 - accounts9) == set()
|
||||
|
||||
|
||||
def account_by_uid(conn, uid):
|
||||
"""Return SSO UID for a given Owncloud UID."""
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT ldap_dn FROM oc_ldap_user_mapping WHERE owncloud_name = '{uid}'")
|
||||
dn, = cur.fetchone()
|
||||
cur.close()
|
||||
part = dn.split(',')[0]
|
||||
assert part.startswith('uid=')
|
||||
return part[4:]
|
||||
|
||||
|
||||
def storage_owner_by_id(conn, id_):
|
||||
"""Return SSO UID for a given storage numerical ID."""
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT id FROM oc_storages WHERE numeric_id = '{id_}'")
|
||||
oid, = cur.fetchone()
|
||||
cur.close()
|
||||
if oid == 'object::store:amazon::nextcloud':
|
||||
return "S3"
|
||||
assert oid.startswith('object::user:')
|
||||
userid = oid[13:]
|
||||
assert len(userid) > 0
|
||||
if userid == "gallery":
|
||||
return "GALLERY"
|
||||
return account_by_uid(conn, userid)
|
||||
|
||||
|
||||
# Check shares table. This table contains the intent of sharing some file with someone else.
|
||||
#
|
||||
# RESULT: we only have things that have been removed after rollback to PG9,
|
||||
# nothing was created in PG12 and lost.
|
||||
shareids12 = idset(conn12, "share")
|
||||
shareids9 = idset(conn9, "share")
|
||||
print("Shares missing in 9:", len(shareids12 - shareids9))
|
||||
cur12 = conn12.cursor()
|
||||
for id_ in list(shareids12-shareids9):
|
||||
cur12.execute(f"SELECT uid_owner, file_target, stime, share_with FROM oc_share WHERE id = {id_}")
|
||||
uid_owner, file_target, stime, share_with = cur12.fetchone()
|
||||
account = account_by_uid(conn12, uid_owner)
|
||||
stime_human = datetime.utcfromtimestamp(stime).strftime('%Y-%m-%d %H:%M:%S')
|
||||
print(f"Missing share {id_} {file_target} owned by {account}..")
|
||||
if stime < incident_start or stime > incident_end:
|
||||
print(f" Skipping, created at {stime_human}")
|
||||
continue
|
||||
raise Exception("Unhandled.")
|
||||
cur12.close()
|
||||
|
||||
|
||||
# Check mounts table. This contains root file storages for each user, but also
|
||||
# incoming shares 'mounted' into a user's account.
|
||||
# From what I cen tell, storage_id/root_id are the source path that's being
|
||||
# mounted (root_id being the fileid inside an oc_filecache, and storage_id
|
||||
# being the storage in which that file is kept), while user_id/mount_point are
|
||||
# the mount destination (ie. path into which this is mounted for a user's
|
||||
# view).
|
||||
#
|
||||
# RESULT: we only have share-mounts missing for a handful of users. We choose
|
||||
# to ignore it, as we assume next time these users log in they will get the
|
||||
# mounts again.
|
||||
# TODO(q3k): verify this
|
||||
mounts12 = valset(conn12, "mounts", ["storage_id", "root_id", "user_id", "mount_point"])
|
||||
mounts9 = valset(conn9, "mounts", ["storage_id", "root_id", "user_id", "mount_point"])
|
||||
print("Mounts missing in 9:", len(mounts12 - mounts9))
|
||||
# Mounts that appearify normally whenever you log into owncloud, as they are the result of shares':
|
||||
mount_names_ok = set(["2020-03-26_covid_templar", "camera", "Public Shaming", "przylbice.md", "Test.txt", "covid"])
|
||||
# Mounts that used to be from a share that existed, but has been since deleted in PG9.
|
||||
mount_names_ok |= set(["Covid-instrukcje", "Chaos_modele_covid", "Covid_proces_presspack"])
|
||||
mounts_sorted = []
|
||||
for m in list(mounts12 - mounts9):
|
||||
storage_id, root_id, user_id, mount_point = m.split(';;;')
|
||||
mounts_sorted.append((storage_id, root_id, user_id, mount_point))
|
||||
mounts_sorted = sorted(mounts_sorted, key=lambda el: el[2])
|
||||
for storage_id, root_id, user_id, mount_point in mounts_sorted:
|
||||
assert mount_point.startswith("/" + user_id + "/")
|
||||
mount_point = mount_point[len(user_id)+1:]
|
||||
account = account_by_uid(conn12, user_id)
|
||||
print(f"Missing mount {mount_point}, storage ID {storage_id}, owned by {account}..")
|
||||
storage_owner = storage_owner_by_id(conn12, storage_id)
|
||||
print(f" Storage owner: {storage_owner}")
|
||||
|
||||
parts = mount_point.split('/')
|
||||
if len(parts) == 4 and parts[0] == '' and parts[1] == 'files' and parts[2] in mount_names_ok and parts[3] == '':
|
||||
print(" Skipping, known okay")
|
||||
continue
|
||||
raise Exception("Unhandled")
|
||||
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
-----BEGIN PGP MESSAGE-----
|
||||
|
||||
hQEMAzhuiT4RC8VbAQf9Ecn9tDsi84AKCyPySLpGFBj5Fp8Rc/9b3RA5f4614tqE
|
||||
2LKn5UQP7Ejg6LzCZEBlplu4CFBM5fVe+sx0pZNTVdi+qOPFYB4ruV0TLLjacaAY
|
||||
6hyD7mzmMEWVhHYHJpqCwUV8Vx7vN/6SG91nObCuEjbfYrAXjwdiXvDiBumH6d9O
|
||||
N5CTh5EYVqazZD4SvSXYPyG/iv6/1nrxlfYA/LxD+ULVPhjKBboNTjfVtjAhQVvD
|
||||
ZRMl4/rP1/WXRXz7svGaENTEG6TQ95ZrnSYPZQD+amWKumRiCdneN6I3N4s2F9dJ
|
||||
+FP5WJ214rUAx/rusf+Gt5v2FPyfNw6QJR41WFrjhIUBDANcG2tp6fXqvgEH/0pJ
|
||||
Hlwe6F1ltWm5gdhUUJ5+/tJQH3e4cq0EI+26dENE9633pVA8ERFY3zmcTcxOZ04k
|
||||
K9r8+ifiagdIvWldNLKXrHaNxZWM+r4fJ1RTo0x/gskid8e9otdxo9kR8t3yh9SZ
|
||||
DHRSlCKr2+/H167h89dPWtJh5meUPWAlw2Zmcl2GDue5zjDVrHEkZ8fbxOwf9E/o
|
||||
J93oaux9Ijfrp8tP7lO90qAoaXvTAMeI7hnkWHnX2akcVh03U4FgDqLpZSlVCRP/
|
||||
mIFXRnaLuZEFdmrO+raZUi58t8xaadTZy9hsmCtlsgSKIwgA8zsF2CmJCRvZD95w
|
||||
Q70vLg91E7Bfyr1duhKFAgwDodoT8VqRl4UBEACoBXEXQaxVvZ3vClxUkxrq468/
|
||||
YC6NyTXWOt9KvpuGUWCtbFIQQ4CSfRu8UpvPasppY5shLE4L+0f79OV8aQfV4mF+
|
||||
EdHZ0zCWuVIBcYh+iiAsz5zYpbB7vSW+J+DG7ZH0pCVJZ8CSTJhK42BmyDh0a6ut
|
||||
glHqLQaeC6dPur5To+C+Ozl8v6GduJDzZQ5ERCaML0nhauH0yoEe3My52BlEZ9MK
|
||||
mjJlm3Q3VCYTfT8M08ZLTART6zMotYTemE/u+U8rVVPPDY+7kgy8yt44QAoUzE49
|
||||
shr5llK9GBjDeo4URf//0KvLs95H4TdGVYMfELcFyqyCs0YJ2vee2KvoyuXPGmkO
|
||||
6kuASSgGn5zFqdX13P0y4QIC5OxWnX22CxMdXpvG18RqCW2qyMT3wscqTCYuIHyc
|
||||
yPZWfi+MM1tDB+CdQbFLus5gWPUSZ8kK+sur/5lb0ToamWvYHhNTKMOBh8MCODAy
|
||||
F6lxrUIc81uojF8VKbgK+lOyQUtL4R3+1Wbk+9TyBYUJNjMi/S2mpL1bOUkKZkpb
|
||||
hmqloSPP7W8xa9JuxFDgyPLWpmGSJkDz+6MPiAPfPBgmczUORNuGReck4SDYlZib
|
||||
SRWrhs54zkmlZPmFCEoqbLqyFfQiNqLhQ8J/ya//HZuXAUNd9612P2a+idAAE6gx
|
||||
pz54H+jeH9o8Db01BYUCDAPiA8lOXOuz7wEP/1Uw8k0t4kPrkD1hZ0p2KFklyM6O
|
||||
Ri3j1Y2IcYcQZ23OKMx5qo90/aLBzYSRtA/NHWuqcaDjudyFJ12lSTNyQX0sFnUP
|
||||
nbig6smYzOu3XnkTnRBrOe4YN5YJiyUFTsK5wPcUcArCuLASvRCxzkwHyTQTnW6Y
|
||||
r92oArKEzXzE8sYFMPpRYpV29sQgqXUBEq0bA9codN1Z1m5N3aGvMiYyimq4jXoq
|
||||
Va+Tsry5KON2S0/h8UZsLnY/USSXjWdhb266tU9MLgY2EIK9DaTfU6mxsMTLysVQ
|
||||
RhBmtHQhzczkfueMYa7KigbXJNxvEjUlR22RVRiH9F3lhhismsW1xtgfI/IlGmQx
|
||||
6uhCFMDIsgZh39kWRP4vUxzWTvnPSD76omBcdjVTKGDEd8vqEwIBeORg9E6NoN+Q
|
||||
8HR6Fb6y0pAk30VO1mP3xC0Li9q13ips1p1w+Xu9WOPFVEwJaSFn1oaEWTuQttn3
|
||||
gPdng35LjYLnch588exe1bhoj6WiUaCclZF5yjewMCGowlvCO05QEyivRj2YGeHS
|
||||
D+oH+dv/Ex0PIJR/8P+clAcB/u3qQl2W40pPjHOmMGb8Gi+GSVU5HX0lENdmMrfw
|
||||
QfZuqKh08j8gVK77yX0UjtNhN0XVu5toCncgSFLxZSkQd4opZTonmdji0vwSapx2
|
||||
FbPI2PVqRRWjknuj0ukBu3zMhA+90PvWjmWxaSVBs0FDOvOApYUf7QwYuVpk/hkN
|
||||
GFqrarltId1Xy2UQSLHdKgb1fj4OhnVSApK9cKxD0mjJ87QibQFLeR2KCIOX4/EJ
|
||||
qAuC7xV8VTtzFQqOYF1RrXN8NbU/htMAecKYX3mj3S9FXgOUHEj/fYdFHuFhVhdw
|
||||
onJZ1CReuQX6nGU6c7bCKj7mvt/QISq7eiPT2zqt+f0X+Bz6ODkk/5QHyu2mQUTt
|
||||
PQOayDWpi5vcNsiMj8DLR7nLjErHau/joIyRMopssYzdDb5d2tsofIjoL9VzYbUu
|
||||
PTs/VJCIJ7XvRt1SA0pyQ9JUrvNhskz095CVuPF1LoA8HlcHWCUhiT19bWegsYIu
|
||||
rFqglNCrNuN2t0mQpeA9108EB8m5bmaEK6tlepwhqno3S35KnYGT+FZTo8A3xt38
|
||||
sOATC71bTPyLSNklXK+7t4YPD5nulINNLDqoxUe7ruCwZRkiWZHbp0+AON/jk5CZ
|
||||
O9pNUPA524+sLZzY1XGumifv+f7H4vExOhAMWsPAVfwBdwW4//wIeafV65RMsEUD
|
||||
SzMUfdNQVvf6w1YP3MpDvCHOLlCTrMf8fKKQAkT6Dj/nGHt5bVA10IYY/jaIdXj0
|
||||
VsONFWWaqKxmh9kVGmHdjvZ/tnDFXRPXS+3ddA/jqfVt2JzebHBDeropYYmDXMEe
|
||||
wA4Nbl5qjSoxFsmVMNROIXMrMv66LVG3kVxuHUix147qix8JTOWCFlgaBBvp16Dv
|
||||
8k52poapN6QnMnR12QWxZyroZEeTV4RvNCB/QdoVHEcx8/XBmE3psSceUh9GNU6S
|
||||
BZsb+68KosvRV9bFDcg0DJ+Qyp7k25F4+ItdSxIceW0phAQft9GJafehvnXaQj9A
|
||||
vRLuLTtM37QL8YX7vs5DKJAzG7RCgmrUfbVS6BnWhLAaUWTFgQXHd9gtw2XCkb3y
|
||||
GUCztuO3t5zRwxTlvXWt1KBdLvIm4xrmU/yfuUOK2eLmwdH+rqouVUW1fRrTXhZD
|
||||
OGyvSLIB/kujiukjIJ4idBImzmJcqMewdEdiYw/zsns3RZrfFop1IBZ8fpdAXY4m
|
||||
S0j9Yhy9qqyZ+h9ZYFG5vK6xdevaeqIGGpc7Zk7xTZjXrA+kbrjZyJa6LMUF9dFH
|
||||
MJx4tytpu546euLWP2t9OEu7T+0b9sOrQCrNNIS+qDPN37FNzvxOwCf+i5xr00nI
|
||||
7ifX1c5tA/K6b/IhDVcACqfnUAcgPg+S2qkRtleATzKE0g2ISozbw2/LNb/Th3L1
|
||||
/fH2VmHZyTHJg06PpLsaqnNFpiwIS3Kb2RKtgo3ZNLg+S6QVUd90wEMEX+cdgg==
|
||||
=aO5Z
|
||||
-----END PGP MESSAGE-----
|
||||
Loading…
Reference in New Issue