cluster/tools/rook-s3cmd-config: build using bazel

This turns the existing script into a proper sh_binary, and injects dependencies (kubectl and jq) as deps into it. This change also pulls in BUILDfiles for jq, and a dep (oniguruma) into //third_party, and adds buildable external repositories for them. The jq/oniguruma BUILDfiles are lifted from https://github.com/attilaolah/bazel-tools/. Change-Id: If2e548bd60a8fd34e4f3be767ae59c6b2f2286d9
2020-06-13 22:43:06 +02:00 · 2020-06-13 22:43:06 +02:00 · 0037edaa5b
parent d9df5879e3
commit 0037edaa5b
10 changed files with 637 additions and 37 deletions
--- a/22
+++ b/22
@ -306,6 +306,28 @@ load(
 )

 go_image_repositories()
+
+# oniguruma, with build from //third_party/oniguruma
+
+http_archive(
+    name = "com_github_kkos_oniguruma",
+    urls = ["https://github.com/kkos/oniguruma/releases/download/v6.9.5_rev1/onig-6.9.5_rev1.tar.gz"],
+    strip_prefix = "onig-6.9.5",
+    sha256 = "d33c849d1672af227944878cefe0a8fcf26fc62bedba32aa517f2f63c314a99e",
+    build_file = "@hscloud//third_party/oniguruma:BUILD.external",
+)
+
+# jq, with build from //third_party/jq
+
+http_archive(
+    name = "com_github_stedolan_jq",
+    urls = ["https://github.com/stedolan/jq/releases/download/jq-1.6/jq-1.6.tar.gz"],
+    strip_prefix = "jq-1.6",
+    sha256 = "5de8c8e29aaa3fb9cc6b47bb27299f271354ebb72514e3accadc7d38b5bbaa72",
+    build_file = "@hscloud//third_party/jq:BUILD.external",
+)
+
+
 # Go repositories

 go_repository(
--- a/cluster/tools/BUILD
+++ b/cluster/tools/BUILD
@ -37,3 +37,12 @@ sh_binary(
    data = ["@nixops//:bin", "//tools:secretstore"],
 )

+sh_binary(
+    name = "rook-s3cmd-config",
+    srcs = ["rook-s3cmd-config.sh"],
+    data = [
+        "@bazel_tools//tools/bash/runfiles",
+        "@com_github_stedolan_jq//:jq",
+        ":kubectl",
+    ],
+)
--- a/cluster/tools/rook-s3cmd-config
+++ b/cluster/tools/rook-s3cmd-config
@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Generates s3cmd config from rook.io CephObjectStoreUser secrets fetched from
-# Kubernetes apiserver. Accepts extra K8S_INTERNAL=1 environment variable flag
-# that generates config that connects to internal rgw service.
-#
-# Usage:
-#   ./rook-s3cmd-config USERNAME STORENAME CLUSTERNAME > config
-#   s3cmd -c config --region "STORENAME:default-placement" mb s3://test/
-
-set -e
-
-username="${1:-registry}"
-storename="${2:-waw-hdd-redundant-2-object}"
-clustername="${3:-ceph-waw2}"
-
-secret="$(kubectl get secrets rook-ceph-object-user-$storename-$username -n $clustername -o json)"
-accesskey="$(echo "$secret" | jq -r '.data.AccessKey' | base64 -d)"
-secretkey="$(echo "$secret" | jq -r '.data.SecretKey' | base64 -d)"
-
-if [[ ! -z "$K8S_INTERNAL" ]]; then
-    domain="rook-ceph-rgw-$storename.$clustername.svc.cluster.local"
-else
-    domain="object.$clustername.hswaw.net"
-fi
-
-cat <<EOF
-[default]
-access_key = $accesskey
-secret_key = $secretkey
-host_base = $domain
-host_bucket = $domain
-EOF
-
-if [[ ! -z "$K8S_INTERNAL" ]]; then
-    echo "use_https = False"
-fi
--- a/cluster/tools/rook-s3cmd-config.sh
+++ b/cluster/tools/rook-s3cmd-config.sh
@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Generates s3cmd config from rook.io CephObjectStoreUser secrets fetched from
+# Kubernetes apiserver. Accepts extra K8S_INTERNAL=1 environment variable flag
+# that generates config that connects to internal rgw service.
+#
+# Usage:
+#   bazel run //cluster/tools:rook-s3cmd-config > config
+#   s3cmd -c config --region "STORENAME:default-placement" mb s3://test/
+
+set -euo pipefail
+
+# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# endpaste
+
+kubectl=$(rlocation "hscloud/cluster/tools/kubectl")
+if [ -z "$kubectl" ]; then
+    echo "Could not find kubectl in runfiles" >&2
+    exit 1
+fi
+
+jq=$(rlocation "com_github_stedolan_jq/jq")
+if [ -z "$jq" ]; then
+    echo "Could not find jq in runfiles" >&2
+    exit 1
+fi
+
+username="${1}"
+storename="${2:-waw-hdd-redundant-3-object}"
+clustername="${3:-ceph-waw3}"
+
+if [ -z "$username" ]; then
+    echo "Usage: $0 <username>" >&2
+    exit 1
+fi
+
+
+secret="$($kubectl get secrets rook-ceph-object-user-$storename-$username -n $clustername -o json)"
+accesskey="$(echo "$secret" | $jq -r '.data.AccessKey' | base64 -d)"
+secretkey="$(echo "$secret" | $jq -r '.data.SecretKey' | base64 -d)"
+
+if [[ ! -z "${K8S_INTERNAL:-}" ]]; then
+    domain="rook-ceph-rgw-$storename.$clustername.svc.cluster.local"
+else
+    domain="object.$clustername.hswaw.net"
+fi
+
+cat <<EOF
+[default]
+access_key = $accesskey
+secret_key = $secretkey
+host_base = $domain
+host_bucket = $domain
+EOF
+
+if [[ ! -z "${K8S_INTERNAL:-}" ]]; then
+    echo "use_https = False"
+fi
--- a/third_party/jq/BUILD
+++ b/third_party/jq/BUILD
--- a/third_party/jq/BUILD.external
+++ b/third_party/jq/BUILD.external
@ -0,0 +1,78 @@
+# Copyright 2019 Google LLC
+# Copyright 2020 Sergiusz 'q3k' Bazanski
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The jq binary.
+cc_binary(
+    name = "jq",
+    srcs = [
+        ":source_files",
+    ],
+    copts = [
+        "-Iexternal/com_github_kkos_oniguruma/src",
+        "-Wno-cpp",
+        "-Wno-unused-function",
+        "-Wno-unused-variable",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_kkos_oniguruma//:oniguruma",
+    ],
+)
+
+# Generate the source files.
+# This will run ./configure and extract a host-dependent confdefs.h file, then
+# prefix each .h acd .c file with '#include "confdefs.h"'.
+genrule(
+    name = "source_files",
+    srcs = glob(["**"]),
+    outs = ["build/" + f for f in glob(
+        [
+            "src/*.h",
+            "src/*.c",
+        ],
+        exclude = [
+            "src/inject_errors.c",
+        ],
+    )] + [
+        # The extracted confdefs.h.
+        "build/src/confdefs.h",
+        # The generated builtin.inc.
+        "build/src/builtin.inc",
+    ],
+    cmd =
+        # Run ./configure && make src/builtin.inc.
+        "( " +
+        "  cd external/com_github_stedolan_jq; " +
+        "  ./configure > /dev/null 2> /dev/null; " +
+        "  make src/builtin.inc > /dev/null; " +
+        "); " +
+        # Extract confdefs.h from config.log.
+        "grep '^/\\* confdefs.h \\*/$$' external/com_github_stedolan_jq/config.log -A1000 " +
+        "  | head -n -1 > \"$(@D)\"/build/src/confdefs.h; " +
+        # Prefix each output file with an include of confdefs.h.
+        "OUTS=\"$(OUTS)\"; for FILE in $$OUTS; do " +
+        "  touch \"$$FILE\"; " +
+        "  BASENAME=\"$$(basename $$FILE)\"; " +
+        "  if [ \"$$BASENAME\" != \"confdefs.h\" ]; then " +
+        "    echo '#include \"confdefs.h\"' > \"$$FILE\"; " +
+        "  cat external/com_github_stedolan_jq/src/\"$$BASENAME\" >> \"$$FILE\"; " +
+        "  fi; " +
+        # Replace non-relative references.
+        "  sed -e 's|^#include \"src/|#include \"|g' -i \"$$FILE\"; " +
+        "done; " +
+        # Copy builtin.inc and version.h without modificaitons.
+        "cp external/com_github_stedolan_jq/src/builtin.inc \"$(@D)\"/build/src/builtin.inc; ",
+)
+
--- a/third_party/jq/README.md
+++ b/third_party/jq/README.md
@ -0,0 +1,6 @@
+third\_party: jq
+================
+
+Upstream: https://github.com/stedolan/jq
+
+Pulled in by WORKSPACE (com\_github\_stedolan\_jq), BUILD.external used as root BUILDfile for external repository.
--- a/third_party/oniguruma/BUILD
+++ b/third_party/oniguruma/BUILD
--- a/third_party/oniguruma/BUILD.external
+++ b/third_party/oniguruma/BUILD.external
@ -0,0 +1,441 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The Oniguruma library.
+cc_library(
+    name = "oniguruma",
+    hdrs = [
+        "src/oniguruma.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":ascii",
+        ":big5",
+        ":cp1251",
+        ":euc_jp",
+        ":euc_kr",
+        ":euc_tw",
+        ":gb18030",
+        ":iso8859",
+        ":koi8",
+        ":onig_init",
+        ":regcomp",
+        ":regenc",
+        ":regerror",
+        ":regexec",
+        ":regext",
+        ":reggnu",
+        ":regparse",
+        ":regposix",
+        ":regsyntax",
+        ":regtrav",
+        ":regversion",
+        ":sjis",
+        ":st",
+        ":unicode",
+        ":utf16",
+        ":utf32",
+        ":utf8",
+    ],
+)
+
+# The mktable binary.
+cc_binary(
+    name = "mktable",
+    srcs = [
+        "src/mktable.c",
+    ],
+    deps = [
+        ":oniguruma",
+    ],
+)
+
+# Other libraries linked in with :oniguruma.
+cc_library(
+    name = "ascii",
+    srcs = [
+        "src/ascii.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "big5",
+    srcs = [
+        "src/big5.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "cp1251",
+    srcs = [
+        "src/cp1251.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "euc_jp",
+    srcs = [
+        "src/euc_jp.c",
+        "src/euc_jp_prop.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "euc_kr",
+    srcs = [
+        "src/euc_kr.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "euc_tw",
+    srcs = [
+        "src/euc_tw.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "gb18030",
+    srcs = [
+        "src/gb18030.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "iso8859",
+    srcs = [
+        "src/iso8859_1.c",
+        "src/iso8859_10.c",
+        "src/iso8859_11.c",
+        "src/iso8859_13.c",
+        "src/iso8859_14.c",
+        "src/iso8859_15.c",
+        "src/iso8859_16.c",
+        "src/iso8859_2.c",
+        "src/iso8859_3.c",
+        "src/iso8859_4.c",
+        "src/iso8859_5.c",
+        "src/iso8859_6.c",
+        "src/iso8859_7.c",
+        "src/iso8859_8.c",
+        "src/iso8859_9.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "koi8",
+    srcs = [
+        "src/koi8.c",
+        "src/koi8_r.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "onig_init",
+    srcs = [
+        "src/onig_init.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regcomp",
+    srcs = [
+        "src/regcomp.c",
+    ],
+    deps = [
+        ":regparse",
+    ],
+    alwayslink = 1,
+    copts = [
+        "-Wno-maybe-uninitialized",
+    ],
+)
+
+cc_library(
+    name = "regenc",
+    srcs = [
+        "src/regenc.c",
+    ],
+    hdrs = [
+        "src/oniguruma.h",
+        "src/regenc.h",
+        "src/regint.h",
+    ],
+    deps = [
+        ":config",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regerror",
+    srcs = [
+        "src/regerror.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regexec",
+    srcs = [
+        "src/regexec.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regext",
+    srcs = [
+        "src/regext.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "reggnu",
+    srcs = [
+        "src/reggnu.c",
+    ],
+    hdrs = [
+        "src/oniggnu.h",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regparse",
+    srcs = [
+        "src/regparse.c",
+    ],
+    hdrs = [
+        "src/regparse.h",
+    ],
+    deps = [
+        ":st",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regposix",
+    srcs = [
+        "src/regposerr.c",
+        "src/regposix.c",
+    ],
+    hdrs = [
+        "src/onigposix.h",
+    ],
+    deps = [
+        ":config",
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regsyntax",
+    srcs = [
+        "src/regsyntax.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regtrav",
+    srcs = [
+        "src/regtrav.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "regversion",
+    srcs = [
+        "src/regversion.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "sjis",
+    srcs = [
+        "src/sjis.c",
+        "src/sjis_prop.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "st",
+    srcs = [
+        "src/st.c",
+    ],
+    hdrs = [
+        "src/st.h",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "unicode",
+    srcs = [
+        "src/unicode.c",
+        "src/unicode_fold1_key.c",
+        "src/unicode_fold2_key.c",
+        "src/unicode_fold3_key.c",
+        "src/unicode_unfold_key.c",
+    ],
+    deps = [
+        ":config",
+        ":data",
+        ":regenc",
+        ":st",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "utf8",
+    srcs = [
+        "src/utf8.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "utf16",
+    srcs = [
+        "src/utf16_be.c",
+        "src/utf16_le.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "utf32",
+    srcs = [
+        "src/utf32_be.c",
+        "src/utf32_le.c",
+    ],
+    deps = [
+        ":regenc",
+    ],
+    alwayslink = 1,
+)
+
+# Generated data files.
+cc_library(
+    name = "data",
+    hdrs = [
+        "src/unicode_egcb_data.c",
+        "src/unicode_fold_data.c",
+        "src/unicode_property_data.c",
+        "src/unicode_property_data_posix.c",
+        "src/unicode_wb_data.c",
+    ],
+)
+
+# CC library containing config.h.
+cc_library(
+    name = "config",
+    hdrs = [
+        ":config_h",
+    ],
+)
+
+genrule(
+    name = "config_h",
+    srcs = glob(["**"]),
+    outs = ["config.h"],
+    cmd =
+        "( " +
+        "  cd external/com_github_kkos_oniguruma; " +
+        "  ./configure > /dev/null; " +
+        "); " +
+        "cp external/com_github_kkos_oniguruma/src/config.h \"$@\"; ",
+)
+
--- a/third_party/oniguruma/README.md
+++ b/third_party/oniguruma/README.md
@ -0,0 +1,6 @@
+third\_party: oniguruma
+=======================
+
+Upstream: https://github.com/kkos/oniguruma
+
+Pulled in by WORKSPACE (com\_github\_kkos\_oniguruma), BUILD.external used as root BUILDfile for external repository.