From 2ce367681aea03a8c2122b48f92e8be9e33deecb Mon Sep 17 00:00:00 2001 From: Serge Bazanski Date: Sun, 14 Jul 2019 16:29:52 +0200 Subject: [PATCH] *: move away from python_rules python_rules is completely broken when it comes to py2/py3 support. Here, we replace it with native python rules from new Bazel versions [1] and rules_pip for PyPI dependencies [2]. rules_pip is somewhat little known and experimental, but it seems to work much better than what we had previously. We also unpin rules_docker and fix .bazelrc to force Bazel into Python 2 mode - hopefully, this repo will now work fine under operating systems where `python` is python2 (as the standard dictates). [1] - https://docs.bazel.build/versions/master/be/python.html [2] - https://github.com/apt-itude/rules_pip Change-Id: Ibd969a4266db564bf86e9c96275deffb9610dd44 --- .bazelrc | 4 + .gitignore | 6 +- BUILD | 6 ++ WORKSPACE | 66 +++++++------ cluster/clustercfg/BUILD | 6 +- cluster/clustercfg/clustercfg.py | 2 +- pip/BUILD | 12 +++ pip/requirements-linux.txt | 119 ++++++++++++++++++++++++ requirements.txt => pip/requirements.in | 0 tools/BUILD | 9 +- 10 files changed, 185 insertions(+), 45 deletions(-) create mode 100644 pip/BUILD create mode 100644 pip/requirements-linux.txt rename requirements.txt => pip/requirements.in (100%) diff --git a/.bazelrc b/.bazelrc index 17fa9d43..3a589a2e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,3 +1,7 @@ +# https://github.com/bazelbuild/rules_docker/issues/842 +build --host_force_python=PY2 +test --host_force_python=PY2 +run --host_force_python=PY2 # Required for app/gerrit/gerrit-oauth-provider build --workspace_status_command=./tools/workspace-status.sh test --build_tests_only diff --git a/.gitignore b/.gitignore index 02838a6a..34b85ba9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ *swp -bazel-bin -bazel-genfiles -bazel-out -bazel-hscloud -bazel-testlogs +bazel-* .kubectl diff --git a/BUILD b/BUILD index e411b251..b29b1c5f 100644 --- a/BUILD +++ b/BUILD @@ -6,3 +6,9 @@ load("@bazel_gazelle//:def.bzl", "gazelle") gazelle( name = "gazelle", ) + +config_setting( + name = "linux", + constraint_values = ["@bazel_tools//platforms:linux"], + visibility = ["//visibility:public"], +) diff --git a/WORKSPACE b/WORKSPACE index aee5f12f..58dc8e53 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,6 +1,32 @@ + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +# Skylib + +skylib_version = "0.8.0" +http_archive( + name = "bazel_skylib", + type = "tar.gz", + url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version), + sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", +) + +# Docker rules + +http_archive( + name = "io_bazel_rules_docker", + sha256 = "87fc6a2b128147a0a3039a2fd0b53cc1f2ed5adb8716f50756544a572999ae9a", + strip_prefix = "rules_docker-0.8.1", + urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.8.1.tar.gz"], +) + +load( + "@io_bazel_rules_docker//repositories:repositories.bzl", + container_repositories = "repositories", +) + +container_repositories() # Nix rules http_archive( name = "io_tweag_rules_nixpkgs", @@ -24,42 +50,26 @@ nixpkgs_package( ) # Python rules + git_repository( - name = "io_bazel_rules_python", - remote = "https://github.com/bazelbuild/rules_python.git", - commit = "ebd7adcbcafcc8abe3fd8e5b0e42e10ced1bfe27", + name = "com_apt_itude_rules_pip", + commit = "e5ed5e72bf5a7521244e1d2119821628bbf17263", + remote = "https://github.com/apt-itude/rules_pip.git", ) # Python dependencies +load("@com_apt_itude_rules_pip//rules:dependencies.bzl", "pip_rules_dependencies") -load("@io_bazel_rules_python//python:pip.bzl", "pip_import") +pip_rules_dependencies() -pip_import( - name = "py_deps", - requirements = "//:requirements.txt", +load("@com_apt_itude_rules_pip//rules:repository.bzl", "pip_repository") + +pip_repository( + name = "pip36", + python_interpreter = "python3.6", + requirements = "//pip:requirements-linux.txt", ) -load("@py_deps//:requirements.bzl", "pip_install") - -pip_install() - -# Docker rules - -git_repository( - name = "io_bazel_rules_docker", - # Invoke puller.par with python2 - # TODO(q3k): use upstream when py2/py3 bazel madness if fully fixed - remote = "https://github.com/q3k/rules_docker", - commit = "4642a2a7775e5713596bd6e37758735a0f38aa6e", -) - -load( - "@io_bazel_rules_docker//repositories:repositories.bzl", - container_repositories = "repositories", -) - -container_repositories() - # Docker base images load("@io_bazel_rules_docker//container:container.bzl", "container_pull") diff --git a/cluster/clustercfg/BUILD b/cluster/clustercfg/BUILD index 5ada23c1..28a776dd 100644 --- a/cluster/clustercfg/BUILD +++ b/cluster/clustercfg/BUILD @@ -1,15 +1,13 @@ -load("@py_deps//:requirements.bzl", "requirement") - py_binary( name = "clustercfg", + python_version = "PY3", srcs = [ "clustercfg.py", "ca.py", ], visibility = ["//visibility:public"], deps = [ - requirement("fabric"), - requirement("future"), + "@pip36//fabric", "//tools:secretstore_lib", ], ) diff --git a/cluster/clustercfg/clustercfg.py b/cluster/clustercfg/clustercfg.py index dac2a133..41614c74 100644 --- a/cluster/clustercfg/clustercfg.py +++ b/cluster/clustercfg/clustercfg.py @@ -210,7 +210,7 @@ def nodestrap(args, nocerts=False): def usage(): - sys.stderr.write("Usage: {} \n".format(sys.argv[0])) + sys.stderr.write("Usage: clustercfg \n") def main(): diff --git a/pip/BUILD b/pip/BUILD new file mode 100644 index 00000000..c97dfc45 --- /dev/null +++ b/pip/BUILD @@ -0,0 +1,12 @@ +load("@com_apt_itude_rules_pip//rules:compile.bzl", "compile_pip_requirements") + +compile_pip_requirements( + name = "compile", + python_interpreter = "python3.6", + requirements_in = ":requirements.in", + requirements_txt = select({ + "//:linux": "requirements-linux.txt", + }), +) + +exports_files(glob(["requirements*"])) diff --git a/pip/requirements-linux.txt b/pip/requirements-linux.txt new file mode 100644 index 00000000..ce36ac37 --- /dev/null +++ b/pip/requirements-linux.txt @@ -0,0 +1,119 @@ +asn1crypto==0.24.0 \ + --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ + --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 +bcrypt==3.1.5 \ + --hash=sha256:05d8b762cb8a9bd0ad92ee95ed34b6119200a8760b625dadacfe88537ae691a3 \ + --hash=sha256:136243dc44e5bab9b61206bd46fff3018bd80980b1a1dfbab64a22ff5745957f \ + --hash=sha256:1de0df7a9ca76d68ec8122573ae584aab78dcfb728fc2c78ecafb15750b79465 \ + --hash=sha256:214c720cfcd394ab9fd1cac59303847b0d45cc8feeb8126ec55619c77d85ec19 \ + --hash=sha256:290e07820d408e8c81f79f848279b95cef693d6e6ce148fa6b1e573e89a4305b \ + --hash=sha256:2d60412b11994ab91d25572f780f8461748cecdb6014c23e33b2ea0aabc99782 \ + --hash=sha256:62ff976497590c7ef714f426aff8b908f2a11686364bb01cfc7d338e86a2ee27 \ + --hash=sha256:77c99c50bd7ac4e9e9f948015c4638176ebe0a495b22b6ae4857f3ba077b12d8 \ + --hash=sha256:9af0a7e135e7f5feca9c16ba33064af545b33b7297c1bb65daedb11c0fa653c4 \ + --hash=sha256:9b08088fd103eedfd750d832819555d1f96bc8bec749c6d35a3f3de3b9e8c98d \ + --hash=sha256:a185efb05ef8bac9a531474abfefb8323f3ec8d524d308d6720657eaeda068b5 \ + --hash=sha256:c7a733c4c309c9ab572644cf7f8779845addcd5ecf474bb5c376f05731842c41 \ + --hash=sha256:cc3f53fa3287c0fc2bc1636e9514b896d4777444b03d9e0e4f16762a856bfe8a \ + --hash=sha256:d216ee4e8e64d43d819acaf8aa0db6cb518859072152cf35ada4987bf5c92bff \ + --hash=sha256:db3c7d712c4049eff365f00c9236279602af17c0ba44ca759008641c7fd892b7 \ + --hash=sha256:e1bb330c56ddec65ad9ce989e9e8664901ce96badfe47853a5ed03bfeb76f91a \ + --hash=sha256:efcaace6e2915434d84e865c44f0cfe34e802269378afbb39a4aa6381aaec78b \ + --hash=sha256:f4431e01f1a5fdea95c78758e24c9565651499d92024ff34663b1ab12c8a10e5 \ + --hash=sha256:fd21155abee7cd4c0ba8fad5138636f2531174ea79ad1751b25dc30d833e1723 +cffi==1.11.5 \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb +cryptography==2.4.2 \ + --hash=sha256:05a6052c6a9f17ff78ba78f8e6eb1d777d25db3b763343a1ae89a7a8670386dd \ + --hash=sha256:0eb83a24c650a36f68e31a6d0a70f7ad9c358fa2506dc7b683398b92e354a038 \ + --hash=sha256:0ff4a3d6ea86aa0c9e06e92a9f986de7ee8231f36c4da1b31c61a7e692ef3378 \ + --hash=sha256:1699f3e916981df32afdd014fb3164db28cdb61c757029f502cb0a8c29b2fdb3 \ + --hash=sha256:1b1f136d74f411f587b07c076149c4436a169dc19532e587460d9ced24adcc13 \ + --hash=sha256:21e63dd20f5e5455e8b34179ac43d95b3fb1ffa54d071fd2ed5d67da82cfe6dc \ + --hash=sha256:2454ada8209bbde97065453a6ca488884bbb263e623d35ba183821317a58b46f \ + --hash=sha256:3cdc5f7ca057b2214ce4569e01b0f368b3de9d8ee01887557755ccd1c15d9427 \ + --hash=sha256:418e7a5ec02a7056d3a4f0c0e7ea81df374205f25f4720bb0e84189aa5fd2515 \ + --hash=sha256:471a097076a7c4ab85561d7fa9a1239bd2ae1f9fd0047520f13d8b340bf3210b \ + --hash=sha256:5ecaf9e7db3ca582c6de6229525d35db8a4e59dc3e8a40a331674ed90e658cbf \ + --hash=sha256:63b064a074f8dc61be81449796e2c3f4e308b6eba04a241a5c9f2d05e882c681 \ + --hash=sha256:6afe324dfe6074822ccd56d80420df750e19ac30a4e56c925746c735cf22ae8b \ + --hash=sha256:70596e90398574b77929cd87e1ac6e43edd0e29ba01e1365fed9c26bde295aa5 \ + --hash=sha256:70c2b04e905d3f72e2ba12c58a590817128dfca08949173faa19a42c824efa0b \ + --hash=sha256:8908f1db90be48b060888e9c96a0dee9d842765ce9594ff6a23da61086116bb6 \ + --hash=sha256:af12dfc9874ac27ebe57fc28c8df0e8afa11f2a1025566476b0d50cdb8884f70 \ + --hash=sha256:b4fc04326b2d259ddd59ed8ea20405d2e695486ab4c5e1e49b025c484845206e \ + --hash=sha256:da5b5dda4aa0d5e2b758cc8dfc67f8d4212e88ea9caad5f61ba132f948bab859 +fabric==2.4.0 \ + --hash=sha256:93684ceaac92e0b78faae551297e29c48370cede12ff0f853cdebf67d4b87068 \ + --hash=sha256:98538f2f3f63cf52497a8d0b24d18424ae83fe67ac7611225c72afb9e67f2cf6 +future==0.17.1 \ + --hash=sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8 +idna==2.8 \ + --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \ + --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c +invoke==1.2.0 \ + --hash=sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6 \ + --hash=sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7 \ + --hash=sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e +paramiko==2.4.2 \ + --hash=sha256:3c16b2bfb4c0d810b24c40155dbfd113c0521e7e6ee593d704e84b4c658a1f3b \ + --hash=sha256:a8975a7df3560c9f1e2b43dc54ebd40fd00a7017392ca5445ce7df409f900fcb +pyasn1==0.4.5 \ + --hash=sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7 \ + --hash=sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e +pycparser==2.19 \ + --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 +pynacl==1.3.0 \ + --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \ + --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \ + --hash=sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e \ + --hash=sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae \ + --hash=sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621 \ + --hash=sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56 \ + --hash=sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39 \ + --hash=sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310 \ + --hash=sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1 \ + --hash=sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a \ + --hash=sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786 \ + --hash=sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b \ + --hash=sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b \ + --hash=sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f \ + --hash=sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20 \ + --hash=sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415 \ + --hash=sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715 \ + --hash=sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1 \ + --hash=sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0 +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 diff --git a/requirements.txt b/pip/requirements.in similarity index 100% rename from requirements.txt rename to pip/requirements.in diff --git a/tools/BUILD b/tools/BUILD index 4b3eb1bf..e9a13221 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -1,27 +1,22 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar", "pkg_deb") -load("@py_deps//:requirements.bzl", "requirement") load("//bzl:rules.bzl", "copy_go_binary") py_library( name = "secretstore_lib", srcs = ["secretstore.py"], visibility = ["//visibility:public"], - deps = [ - requirement("future"), - ], ) py_binary( name = "secretstore", + python_version = "PY3", srcs = ["secretstore.py"], visibility = ["//visibility:public"], - deps = [ - requirement("future"), - ], ) py_binary( name = "pass", + python_version = "PY3", srcs = ["pass.py"], visibility = ["//visibility:public"], )