diff --git a/.appveyor.yml b/.appveyor.yml
index eb43a0250..740e2f11c 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -4,19 +4,36 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# A p p V e y o r C I
+# ============================================================================ #
+
# https://www.appveyor.com/docs/appveyor-yml/
image: Ubuntu
+# workaround for default JDK9 have old CA certs:
+#
+# https://github.com/appveyor/ci/issues/3833
+#
+# https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/#configuring-language-stack
+#
+stack: jdk 15
+
+skip_commits:
+ files:
+ - docs/*
+ - '**/*.md'
+
# https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
environment:
APPVEYOR_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvihSRU+YjBKvKiacDfUoZ7ghoVMcwNh4cWIYUNFGZosXOzNtyOcBpIb71TCgLFhOd+aMWKXCEC67BpNSIjt+a/FLD27AwmgVHv6cPlE3G0JJ9zmIrNmx9511dshTsxUW2O0SbYG+3InuO7FUkSrld+kA1OucyjgmZU7/+Cs9shpAEOaIVYmGlpDGRucAHpwtckvdgRTtnA3WNZ/Qg1vU6Ik4Xm03vjrW6lSiuTffYO1kbdcMQ4IZBlzfmovOtXQ0PomvN5NMCpgOyQuoNlvyS11tOXoqNiWOkiLE15XEzAQth9hHbNiH8jHJbAtkHqWWh0KK4IUyNGvoL6QfNxsTlw== hari@anotherdimension
@@ -37,34 +54,36 @@ on_finish:
#
# https://github.com/appveyor/ci/pull/3385
#
- #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/install_openssh.sh' | bash -e -
+ #- sh: curl -sflL 'https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/install/install_openssh.sh' | bash -e -
+ #
+ # https://www.appveyor.com/docs/how-to/ssh-to-build-worker/
- sh: if [ "$APPVEYOR_SSH_BLOCK" = true ]; then curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -; fi
install:
-# workaround for:
-# Some packages could not be installed. This may mean that you have
-# requested an impossible situation or if you are using the unstable
-# distribution that some required packages have not yet been created
-# or been moved out of Incoming.
-# The following information may help to resolve the situation:
-#
-# The following packages have unmet dependencies:
-# mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed
-# E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.
-# devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed
-# make[2]: *** [apt-packages] Error 123
-# make[2]: Leaving directory '/home/appveyor/projects/pylib'
-# devops-python-tools/Makefile.in:212: recipe for target 'system-packages' failed
-#
-# adding "|| :" to the end of these commands causes them to be silently ignored!
-- sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list
-- sudo apt purge -qy --allow-change-held-packages mssql-server
-# this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411
-#- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages
-- setup/ci_bootstrap.sh
-- make
+ # workaround for:
+ # Some packages could not be installed. This may mean that you have
+ # requested an impossible situation or if you are using the unstable
+ # distribution that some required packages have not yet been created
+ # or been moved out of Incoming.
+ # The following information may help to resolve the situation:
+ #
+ # The following packages have unmet dependencies:
+ # mssql-server : Depends: libsasl2-modules-gssapi-mit but it is not going to be installed
+ # E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.
+ # DevOps-Python-tools/Makefile.in:272: recipe for target 'apt-packages' failed
+ # make[2]: *** [apt-packages] Error 123
+ # make[2]: Leaving directory '/home/appveyor/projects/pylib'
+ # DevOps-Python-tools/Makefile.in:212: recipe for target 'system-packages' failed
+ #
+ # adding "|| :" to the end of these commands causes them to be silently ignored!
+ - sudo sed -i '/https:\/\/packages.microsoft.com\/ubuntu\/.*\/mssql-server/d' /etc/apt/sources.list
+ - sudo apt purge -yq --allow-change-held-packages mssql-server
+ # this prevents conflicts installing default-jdk - see https://github.com/appveyor/ci/issues/3411
+ #- dpkg -l | awk '/openjdk/{print $2}' | DEBIAN_FRONTEND=noninteractive xargs sudo apt-get remove -y --allow-change-held-packages
+ - setup/ci_bootstrap.sh
+ - make
test_script:
-- make test
+ - make test
build: off
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 53dd7405b..eb0d641fd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,15 +4,19 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# B u i l d K i t e C I
+# ============================================================================ #
+
# BuildKite Pipeline
#
# add this command to the UI and it will read the rest of the steps from here:
diff --git a/.checkov.yaml b/.checkov.yaml
new file mode 100644
index 000000000..8962e2283
--- /dev/null
+++ b/.checkov.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-21 16:53:29 +0000 (Mon, 21 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v c o n f i g
+# ============================================================================ #
+
+# https://github.com/bridgecrewio/checkov#configuration-using-a-config-file
+#
+# This is not well documented but the fields seem to be the same as:
+#
+# checkov --help
+#
+# See master template at:
+#
+# https://github.com/HariSekhon/Templates/blob/master/.checkov.yaml
+
+---
+compact: true
+directory:
+ - .
+skip-path:
+ - bash-tools
+ - pylib
+ - sql
+ - templates
+docker-image: harisekhon/pytools
+download-external-modules: true # without this gets lots of annoying warning lines such as '2022-02-22 16:14:40,180 [MainThread ] [WARNI] Failed to download module x/y/z:n.n.n'
+framework:
+ - all
+no-guide: true
+output: cli
+quiet: true
+repo-id: HariSekhon/DevOps-Python-tools # what to report to Bridgecrew Cloud - without this gets annoying duplicate repos such as 'harisekhon_cli_repo/pytools'
+skip-suppressions: true
+soft-fail: true
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69f7a6179..312bf1020 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,36 +5,50 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-# https://circleci.com/docs/2.0/configuration-reference
+# ============================================================================ #
+# C i r c l e C I
+# ============================================================================ #
+
+# Master Template with more advanced config:
+#
+# https://github.com/HariSekhon/Templates/blob/master/circleci_config.yml
+
+# Reference:
+#
+# https://circleci.com/docs/2.0/configuration-reference
version: 2.1
+
+workflows:
+ version: 2
+ workflow:
+ jobs:
+ - build
+
jobs:
build:
- # technically a docker image is a better choice than machine
- # but we want to introduce some native environment variation
- # between build systems in order to test the repo's build automation is robust
- machine:
- image: default
- #image: ubuntu-1604:201903-01
- # set to an actual docker image when running locally using circle_ci_job.sh
- # docker image must have git installed to do the checkout
- # so using harisekhon/dev:ubuntu instead of base ubuntu image
- #image: harisekhon/dev:ubuntu
+ docker:
+ - image: cimg/base:2024.12
+ resource_class: small
steps:
+ # CLI is too old - config validate breaks in test - install new version to fix
+ # doesn't work - existing version is too old to update
+ #- run: circleci update
+ - run: |
+ curl -sSLf https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
- checkout
+ #- setup_remote_docker:
+ # version: 20.10.11
- run: setup/ci_bootstrap.sh
- run: make init
- run: make
- # to allow docker networking to work
- - run: sudo sysctl net.ipv4.ip_forward=1
- - run: sudo service docker restart
- run: make test
diff --git a/.cirrus.yml b/.cirrus.yml
index 46de8ac49..471dc55ba 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -4,21 +4,27 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# C i r r u s C I
+# ============================================================================ #
+
# https://cirrus-ci.org/guide/writing-tasks/
container:
image: ubuntu:18.04
task:
+ env:
+ TMPDIR: /var/tmp
script:
- setup/ci_bootstrap.sh
- make init
diff --git a/.concourse.yml b/.concourse.yml
deleted file mode 100644
index 2abb4ac17..000000000
--- a/.concourse.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# License: see accompanying Hari Sekhon LICENSE file
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-resources:
-- name: github
- icon: github-circle
- type: git
- source:
- uri: https://github.com/harisekhon/devops-python-tools
- branch: master
-#- name: daily
-# type: time
-# source:
-# interval: 1d
-
-# https://concourse-ci.org/golang-library-example.html
-
-jobs:
-- name: build
- public: false
- plan:
- - get: github # from resource above
- trigger: true
- #version: every # build every git commit, default: latest
- - task: build
- config:
- platform: linux
- image_resource:
- type: docker-image
- source:
- repository: ubuntu
- tag: latest
- inputs:
- - name: github
- path: code
- params:
- CONCOURSE: 1
- run:
- path: /bin/bash
- args:
- - -c
- - |
- cd code &&
- setup/ci_bootstrap.sh &&
- make init &&
- make ci test
diff --git a/.drone.yml b/.drone.yml
index beda97729..d105ab57a 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -1,18 +1,24 @@
+---
+# XXX: putting this separator further down with code causes a parsing bug in drone lint
#
# Author: Hari Sekhon
# Date: 2020-02-29 12:05:52 +0000 (Sat, 29 Feb 2020)
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# D r o n e C I
+# ============================================================================ #
+
# https://docs.drone.io/quickstart/cli/
#
# https://docs.drone.io/cli/install/
@@ -28,16 +34,16 @@ type: docker
name: default
steps:
-- name: build
- image: ubuntu:18.04
-# environment:
-# DEBUG: 1
- commands:
- - setup/ci_bootstrap.sh
- - make init
- - make ci
- - make test
+ - name: build
+ image: ubuntu:18.04
+ #environment:
+ # DEBUG: 1
+ commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
trigger:
branch:
- - master
+ - master
diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 000000000..462f53206
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,85 @@
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+# to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# http://EditorConfig.org
+
+# stop recursing upwards for other .editorconfig files
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.go]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_size = 4
+indent_style = tab
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[{*.md,*.hcl,*.tf,*.tfvars}]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml,*.yaml]
+indent_size = 2
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[.*]
+indent_size = 4
+indent_style = space
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# ============================================================================ #
+# Older Stuff, don't think I use this any more
+# ============================================================================ #
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+#[*.{js,py}]
+#charset = utf-8
+
+# Indentation override for all JS under lib directory
+#[lib/**.js]
+#indent_style = space
+#indent_size = 2
+
+# Matches the exact files either package.json or .travis.yml
+#[{package.json,.travis.yml}]
+#indent_style = space
+#indent_size = 2
+
+#[*.xml]
+#indent_style = space
+#indent_size = 2
diff --git a/.envrc b/.envrc
new file mode 100644
index 000000000..e4488d51f
--- /dev/null
+++ b/.envrc
@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# D i r E n v
+# ============================================================================ #
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+# See Also:
+#
+# .envrc-aws
+# .envrc-gcp
+# .envrc-kubernetes
+
+# direnv stdlib - loads .envrc from parent dir up to /
+#
+# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc.
+#
+# bypasses security authorization though - use with care
+#source_up
+#
+# source_up must be loaded before set -u otherwise gets this error:
+#
+# direnv: loading .envrc
+# /bin/bash: line 226: $1: unbound variable
+#
+# source_up causes this error is up .envrc is found in parent directories:
+#
+# direnv: No ancestor .envrc found
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+src="$(readlink -f "${BASH_SOURCE[0]}")"
+srcdir="$(cd "$(dirname "$src")" && pwd)"
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+# Automatically install Pre-Commit Git hooks if not already present
+
+if ! type -P pre-commit &>/dev/null; then
+ if uname -s | grep -q Darwin &&
+ type -P brew &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Homebrew..."
+ echo
+ brew install pre-commit
+ echo
+ elif type -P pip &>/dev/null; then
+ echo
+ echo "Pre-commit is not installed - installing now using Pip..."
+ echo
+ pip install pre-commit
+ fi
+fi
+
+if [ -f .pre-commit-config.yaml ] &&
+ type -P pre-commit &>/dev/null &&
+ git rev-parse --is-inside-work-tree &>/dev/null; then
+ hook="$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
+ if [ -L "$hook" ]; then
+ echo "Detected symlink hook: "
+ echo
+ ls -l "$hook"
+ echo
+ echo "Removing"
+ rm -f "$hook"
+ fi
+ if ! [ -f "$hook" ]; then
+ echo
+ echo "Pre-commit hook is not installed in local Git repo checkout - installing now..."
+ echo
+ pre-commit install
+ fi
+fi
+
+# ============================================================================ #
+# D o c k e r C o m p o s e
+# ============================================================================ #
+
+export COMPOSE_PROJECT_NAME="DevOps-Python-tools"
+
+# ============================================================================ #
+# G i t H u b
+# ============================================================================ #
+
+#export GITHUB_ORGANIZATION=HariSekhon
+
+# ============================================================================ #
+# A n s i b l e
+# ============================================================================ #
+
+# use the local repo's ansible.cfg rather than:
+#
+# $PWD/ansible.cfg
+# ~/.ansible.cfg
+# /etc/ansible/ansible.cfg
+#
+# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used
+#export ANSIBLE_CONFIG="/path/to/ansible.cfg"
+
+# ============================================================================ #
+# C l o u d f l a r e
+# ============================================================================ #
+
+#export CLOUDFLARE_EMAIL=hari@...
+#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens
+#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module
+
+# export the variables for terraform
+#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL"
+#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above
+
+# ============================================================================ #
+# Load External Envrc Files If Present
+# ============================================================================ #
+
+# XXX: safer to bring all these external .envrc inline if you're worried about changes
+# to it bypassing 'direnv allow' authorization
+load_if_exists(){
+ # first arg is a path to a .envrc
+ # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes
+ # to pass the context name 'docker-desktop' to switch to
+ local envrc="$1"
+ shift
+ if ! [[ "$envrc" =~ ^/ ]]; then
+ envrc="$srcdir/$envrc"
+ fi
+ if [ -f "$envrc" ]; then
+ # prevent looping on symlinks to this .envrc if given
+ if [ "$(readlink "$envrc")" = "$src" ]; then
+ return
+ fi
+ echo
+ echo "Loading $envrc"
+ # shellcheck disable=SC1090,SC1091
+ . "$envrc" "$@"
+ fi
+}
+
+# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc
+# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses
+# to install to system Python)
+#load_if_exists ~/.envrc
+
+# ============================================================================ #
+# P y t h o n
+# ============================================================================ #
+
+ #.envrc-aws \
+ #.envrc-gcp \
+ #.envrc-terraform \
+# shellcheck disable=SC2043
+for envrc in \
+ .envrc-python \
+ ; do
+ load_if_exists "$envrc"
+done
+
+# ============================================================================ #
+# A W S
+# ============================================================================ #
+
+if [[ "$PWD" =~ /aws/ ]]; then
+ load_if_exists .envrc-aws
+fi
+
+# ============================================================================ #
+# G C P
+# ============================================================================ #
+
+if [[ "$PWD" =~ /gcp/ ]]; then
+ load_if_exists .envrc-gcp
+fi
+
+# ============================================================================ #
+# T e r r a f o r m
+# ============================================================================ #
+
+if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then
+ load_if_exists .envrc-terraform
+fi
+
+# ============================================================================ #
+# K u b e r n e t e s
+# ============================================================================ #
+
+if [ -f "$srcdir/.envrc-kubernetes" ]; then
+ load_if_exists .envrc-kubernetes docker-desktop
+fi
+
+# ============================================================================ #
+# . E n v
+# ============================================================================ #
+
+echo
+# read .env too
+#dotenv
+
+load_if_exists .envrc.local
diff --git a/.envrc-python b/.envrc-python
new file mode 100644
index 000000000..6bc2d65b1
--- /dev/null
+++ b/.envrc-python
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: Mon Feb 22 17:42:01 2021 +0000
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y t h o n D i r E n v
+# ============================================================================ #
+
+# .envrc to auto-load the virtualenv inside the 'venv' directory if present
+
+# https://direnv.net/man/direnv-stdlib.1.html
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default
+for venv in "$PWD/venv" "$HOME/venv"; do
+ if [ -f "$venv/bin/activate" ]; then
+ echo
+ echo "Virtualenv directory found in: $venv"
+ echo
+ echo "Activating Virtualenv inside the directory: $venv"
+
+ # shellcheck disable=SC1091
+ source "$venv/bin/activate"
+ break
+ fi
+done
+
+# read .env too
+#dotenv
diff --git a/.flake8 b/.flake8
index e31fb949f..de51ee708 100644
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,32 @@
+#
+# Author: Hari Sekhon
+# Date: Mon Oct 21 15:57:10 2019 +0100
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F l a k e 8 C o n f i g
+# ============================================================================ #
+
+# https://flake8.pycqa.org/en/latest/user/configuration.html
+
[flake8]
-ignore = E265,E402,F401
+
max-line-length = 120
+
+ignore = E265,
+ E402,
+ F401
+
exclude = test*/*
+
max-complexity = 10
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..ad861088b
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,38 @@
+#
+# Author: Hari Sekhon
+# Date: 2021-11-09 15:14:59 +0000 (Tue, 09 Nov 2021)
+#
+# vim:ts=4:sts=4:sw=4:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Good in theory, to alert on PR changes to these code paths, but for public repos which may be forked and run .github/workflows/fork-update.yaml, this will result in a lot of spam
+
+# Tips:
+#
+# * includes changes under .github/
+# dir/* only matches first level file changes but doesn't recurse
+# dir/ recurses
+#
+# - CODEOWNERS in base branch of PR determines review request
+# - paths are case sensitive
+# - last match wins, use * at top for overall owner then override with more specific teams
+
+#* @harisekhon # username or email address
+#* @myorg/platform-engineering # team based is the way to go - team must have Write access to the repo regardless of if individuals have access
+#* @myorg/devops
+#k8s @myorg/devops @myorg/sre-team
+#apps/ @myorg/developers
+#apps/dir2 # ignores dir2 as no owner/team specified on this line
+#src/ @myorg/developers
+#docs/ docs@example.com
+#.github/workflows @ci-cd-team
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5f04c0a58..aa46b9847 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,3 +1,3 @@
Please be specific about your issue and include debug output from running with `-v -v -v` or for shell scripts after setting `export DEBUG=1` in your shell.
-You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-Tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-Tools) respectively.
+You can anonymize hostnames / FQDNs, IP / MAC addresses, Kerberos principals, email addresses and almost anything else using `anonymize.pl` or the newer `anonymize.py` available in the [DevOps Perl Tools](https://github.com/HariSekhon/DevOps-Perl-tools) and [DevOps Python Tools](https://github.com/HariSekhon/DevOps-Python-tools) respectively.
diff --git a/.github/workflows/*_centos.yaml.disabled b/.github/workflows/*_centos.yaml.disabled
new file mode 100644
index 000000000..51d70819e
--- /dev/null
+++ b/.github/workflows/*_centos.yaml.disabled
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (CentOS)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:centos
+ ghcr.io/harisekhon/pytools:centos
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-centos
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/alpine.yaml b/.github/workflows/alpine.yaml
index 6550283db..9bed93efb 100644
--- a/.github/workflows/alpine.yaml
+++ b/.github/workflows/alpine.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Alpine
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: alpine
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apk add --no-cache git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:latest
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/alpine_3.yaml b/.github/workflows/alpine_3.yaml
index 6c8eada80..f6c4f5ea9 100644
--- a/.github/workflows/alpine_3.yaml
+++ b/.github/workflows/alpine_3.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Alpine 3
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine_3.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: alpine:3
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apk add --no-cache git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: alpine:3
+ caches: apk pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos.yaml b/.github/workflows/centos.yaml
deleted file mode 100644
index 5c214e8fd..000000000
--- a/.github/workflows/centos.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: CentOS
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: centos
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: yum install -y git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/centos.yaml.disabled b/.github/workflows/centos.yaml.disabled
new file mode 100644
index 000000000..420b88392
--- /dev/null
+++ b/.github/workflows/centos.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:latest
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos6.yaml.disabled b/.github/workflows/centos6.yaml.disabled
deleted file mode 100644
index b78c9de3a..000000000
--- a/.github/workflows/centos6.yaml.disabled
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-# Not supporting RHEL6 any more because it doesn't have GNU parallel package
-
-name: CentOS 6
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: centos:6
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: yum install -y git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/centos7.yaml b/.github/workflows/centos7.yaml
deleted file mode 100644
index bb3c76ec6..000000000
--- a/.github/workflows/centos7.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: CentOS 7
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: centos:7
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: yum install -y git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/centos7.yaml.disabled b/.github/workflows/centos7.yaml.disabled
new file mode 100644
index 000000000..ec764d288
--- /dev/null
+++ b/.github/workflows/centos7.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 7
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos7.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:7
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/centos8.yaml b/.github/workflows/centos8.yaml
deleted file mode 100644
index 070024988..000000000
--- a/.github/workflows/centos8.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: CentOS 8
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: centos:8
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: yum install -y git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/centos8.yaml.disabled b/.github/workflows/centos8.yaml.disabled
new file mode 100644
index 000000000..76c476664
--- /dev/null
+++ b/.github/workflows/centos8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: CentOS 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/centos8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: centos:8
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml
new file mode 100644
index 000000000..209b3cef1
--- /dev/null
+++ b/.github/workflows/checkov.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C h e c k o v G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Static analysis of Terraform code - publishes report to GitHub Security tab
+
+# https://github.com/bridgecrewio/checkov-action
+
+---
+name: Checkov
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+jobs:
+ checkov:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Checkov
+ uses: HariSekhon/GitHub-Actions/.github/workflows/checkov.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml
new file mode 100644
index 000000000..168d4914c
--- /dev/null
+++ b/.github/workflows/codeowners.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o d e O w n e r s
+# ============================================================================ #
+
+---
+name: CodeOwners
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - CODEOWNERS
+ - .github/CODEOWNERS
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate CODEOWNERS
+ uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index d6aee201a..5e74ab863 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,22 +1,14 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
+---
name: "CodeQL"
on:
push:
- branches: [ master ]
+ branches:
+ - master
pull_request:
# The branches below must be a subset of the branches above
- branches: [ master ]
+ branches:
+ - master
schedule:
- cron: '37 15 * * 4'
@@ -32,39 +24,40 @@ jobs:
strategy:
fail-fast: false
matrix:
- language: [ 'python' ]
+ language:
+ - python
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://git.io/codeql-language-support
steps:
- - name: Checkout repository
- uses: actions/checkout@v2
-
- # Initializes the CodeQL tools for scanning.
- - name: Initialize CodeQL
- uses: github/codeql-action/init@v1
- with:
- languages: ${{ matrix.language }}
- # If you wish to specify custom queries, you can do so here or in a config file.
- # By default, queries listed here will override any specified in a config file.
- # Prefix the list here with "+" to use these queries and those in the config file.
- # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
- # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
- # If this step fails, then you should remove it and run the build manually (see below)
- - name: Autobuild
- uses: github/codeql-action/autobuild@v1
-
- # âšī¸ Command-line programs to run using the OS shell.
- # đ https://git.io/JvXDl
-
- # âī¸ If the Autobuild fails above, remove it and uncomment the following three lines
- # and modify them (or add more) to build your code if your project
- # uses a compiled language
-
- #- run: |
- # make bootstrap
- # make release
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v1
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v1
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+ # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v1
+
+ # Command-line programs to run using the OS shell.
+ # https://git.io/JvXDl
+
+ # If the Autobuild fails above, remove it and uncomment the following three lines
+ # and modify them (or add more) to build your code if your project
+ # uses a compiled language
+
+ #- run: |
+ # make bootstrap
+ # make release
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/debian.yaml b/.github/workflows/debian.yaml
index d729ae64e..d84cb16b1 100644
--- a/.github/workflows/debian.yaml
+++ b/.github/workflows/debian.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Debian
-#env:
-# DEBUG: 1
-
-on: # [push]
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: debian
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_10.yaml b/.github/workflows/debian_10.yaml
index 454fe8bb2..e857f280f 100644
--- a/.github/workflows/debian_10.yaml
+++ b/.github/workflows/debian_10.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Debian 10
-#env:
-# DEBUG: 1
-
-on: # [push]
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_10.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: debian:10 # -slim gets java install package conflicts
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:10
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_11.yaml b/.github/workflows/debian_11.yaml
new file mode 100644
index 000000000..ad765f1ad
--- /dev/null
+++ b/.github/workflows/debian_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_11.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:11
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_12.yaml b/.github/workflows/debian_12.yaml
new file mode 100644
index 000000000..dfbc4ec97
--- /dev/null
+++ b/.github/workflows/debian_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_12.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:12
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_6.yaml.disabled b/.github/workflows/debian_6.yaml.disabled
index 96673f75d..f3062b6f8 100644
--- a/.github/workflows/debian_6.yaml.disabled
+++ b/.github/workflows/debian_6.yaml.disabled
@@ -4,53 +4,47 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-name: CI Debian 6
+---
+name: Debian 6
-#env:
-# DEBUG: 1
-
-on: # [push]
+on:
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 10
- runs-on: ubuntu-latest
- container: debian:6
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: |
- ls -l /.dockerenv
- echo "pwd is $PWD"
- echo VERSION: ; cat /etc/*release /etc/*version 2>/dev/null || :
- apt-get update &&
- apt-get install -y git make
- - name: git clone
- run: |
- cd /tmp &&
- git clone "https://github.com/harisekhon/$repo"
- - name: build
- run: |
- cd "/tmp/$repo" &&
- make
- - name: test
- run: |
- cd "/tmp/$repo" &&
- make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:6
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_7.yaml.disabled b/.github/workflows/debian_7.yaml.disabled
index 622715687..1b0e00295 100644
--- a/.github/workflows/debian_7.yaml.disabled
+++ b/.github/workflows/debian_7.yaml.disabled
@@ -4,53 +4,47 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-name: CI Debian 7
+---
+name: Debian 7
-#env:
-# DEBUG: 1
-
-on: # [push]
+on:
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 10
- runs-on: ubuntu-latest
- container: debian:7-slim
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: |
- ls -l /.dockerenv
- echo "pwd is $PWD"
- cat /etc/*release
- apt-get update &&
- apt-get install -y git make
- - name: git clone
- run: |
- cd /tmp &&
- git clone "https://github.com/harisekhon/$repo"
- - name: build
- run: |
- cd "/tmp/$repo" &&
- make
- - name: test
- run: |
- cd "/tmp/$repo" &&
- make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:7
+ # causes nodejs errors
+ #caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_8.yaml b/.github/workflows/debian_8.yaml
deleted file mode 100644
index 99703fe57..000000000
--- a/.github/workflows/debian_8.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Debian 8
-
-#env:
-# DEBUG: 1
-
-on: # [push]
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: debian:8 # -slim gets java install package conflicts
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/debian_8.yaml.disabled b/.github/workflows/debian_8.yaml.disabled
new file mode 100644
index 000000000..b41ead7d3
--- /dev/null
+++ b/.github/workflows/debian_8.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 8
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_8.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:8
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/debian_9.yaml b/.github/workflows/debian_9.yaml
deleted file mode 100644
index db7819b07..000000000
--- a/.github/workflows/debian_9.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Debian 9
-
-#env:
-# DEBUG: 1
-
-on: # [push]
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: debian:9 # -slim gets java install package conflicts
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/debian_9.yaml.disabled b/.github/workflows/debian_9.yaml.disabled
new file mode 100644
index 000000000..451348091
--- /dev/null
+++ b/.github/workflows/debian_9.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Debian 9
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/debian_9.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: debian:9
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/docker_pytools_alpine.yaml b/.github/workflows/docker_pytools_alpine.yaml
new file mode 100644
index 000000000..7119d6852
--- /dev/null
+++ b/.github/workflows/docker_pytools_alpine.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Alpine)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:alpine
+ ghcr.io/harisekhon/pytools:alpine
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-alpine
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_debian.yaml b/.github/workflows/docker_pytools_debian.yaml
new file mode 100644
index 000000000..24f8f21dd
--- /dev/null
+++ b/.github/workflows/docker_pytools_debian.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Debian)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:debian
+ ghcr.io/harisekhon/pytools:debian
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-debian
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_fedora.yaml b/.github/workflows/docker_pytools_fedora.yaml
new file mode 100644
index 000000000..ad1a5ed8d
--- /dev/null
+++ b/.github/workflows/docker_pytools_fedora.yaml
@@ -0,0 +1,124 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Fedora)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:fedora
+ ghcr.io/harisekhon/pytools:fedora
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-fedora
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/docker_pytools_ubuntu.yaml b/.github/workflows/docker_pytools_ubuntu.yaml
new file mode 100644
index 000000000..884917919
--- /dev/null
+++ b/.github/workflows/docker_pytools_ubuntu.yaml
@@ -0,0 +1,126 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-01-27 18:55:16 +0000 (Thu, 27 Jan 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Docker Build (Ubuntu)
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ - .github/
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build.yaml@master
+ with:
+ repo_tags: |
+ harisekhon/pytools:latest
+ harisekhon/pytools:ubuntu
+ ghcr.io/harisekhon/pytools:latest
+ ghcr.io/harisekhon/pytools:ubuntu
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ debug: ${{ github.event.inputs.debug }}
+ secrets:
+ DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/fedora.yaml b/.github/workflows/fedora.yaml
index c7ca8aaae..5a4f9a9b4 100644
--- a/.github/workflows/fedora.yaml
+++ b/.github/workflows/fedora.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Fedora
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/fedora.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: fedora
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: yum install -y git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: fedora
+ caches: yum pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml
new file mode 100644
index 000000000..6b228ca33
--- /dev/null
+++ b/.github/workflows/fork-sync.yaml
@@ -0,0 +1,48 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k S y n c
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch
+
+---
+name: Fork Sync
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 */3 * * *'
+
+permissions:
+ contents: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_sync:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Sync
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-sync.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml
new file mode 100644
index 000000000..c74fa2a0b
--- /dev/null
+++ b/.github/workflows/fork-update-pr.yaml
@@ -0,0 +1,51 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# F o r k U p d a t e P R
+# ============================================================================ #
+
+# For a fork of the original repo, activate to keep its branches up to date via Pull Requests
+#
+# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date
+
+---
+name: Fork Update PR
+
+on: # yamllint disable-line rule:truthy
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 10 * * 1'
+
+permissions:
+ contents: write
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ fork_update_pr:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == true
+ if: github.repository_owner != 'HariSekhon'
+ name: Fork Update PR
+ uses: HariSekhon/GitHub-Actions/.github/workflows/fork-update-pr.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ghcr_python_ubuntu.yaml.disabled b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
new file mode 100644
index 000000000..c4503bc13
--- /dev/null
+++ b/.github/workflows/ghcr_python_ubuntu.yaml.disabled
@@ -0,0 +1,36 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-09 18:07:10 +0000 (Wed, 09 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: GHCR Build (Ubuntu)
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ workflow_dispatch:
+
+jobs:
+ docker_build:
+ name: Docker Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/docker_build_ghcr.yaml@master
+ with:
+ image: pytools
+ tags: ubuntu latest
+ dockerfile-repo: HariSekhon/Dockerfiles
+ context: Dockerfiles/devops-python-tools-ubuntu
+ if: github.ref_name == 'master' || github.ref_name == 'main' || github.ref_name == 'docker'
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml
new file mode 100644
index 000000000..1b0fa15b4
--- /dev/null
+++ b/.github/workflows/grype.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G r y p e
+# ============================================================================ #
+
+---
+name: Grype
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ Grype:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Grype
+ uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml
new file mode 100644
index 000000000..83fa5ef83
--- /dev/null
+++ b/.github/workflows/json.yaml
@@ -0,0 +1,58 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# J S O N
+# ============================================================================ #
+
+# Validate any JSON files found in the repo
+
+---
+name: JSON
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.json'
+ - .github/workflows/json.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_json:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check JSON
+ uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml
new file mode 100644
index 000000000..c389f5df0
--- /dev/null
+++ b/.github/workflows/kics.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s
+# ============================================================================ #
+
+---
+name: Kics
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ kics:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Kics
+ uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml
index 02cd075cd..438dd184e 100644
--- a/.github/workflows/mac.yaml
+++ b/.github/workflows/mac.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Mac
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: macos-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/cache@v1
- with:
- path: ~/Library/Caches/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: brew update
- run: which brew && brew update || echo
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-latest
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_10.15.yaml b/.github/workflows/mac_10.15.yaml
deleted file mode 100644
index 392688e0a..000000000
--- a/.github/workflows/mac_10.15.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Mac 10.15
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: macos-10.15
- steps:
- - uses: actions/checkout@v2
- - uses: actions/cache@v1
- with:
- path: ~/Library/Caches/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: brew update
- run: which brew && brew update || echo
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/mac_11.yaml b/.github/workflows/mac_11.yaml
new file mode 100644
index 000000000..cc392b105
--- /dev/null
+++ b/.github/workflows/mac_11.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-11
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/mac_12.yaml b/.github/workflows/mac_12.yaml
new file mode 100644
index 000000000..0bcd6e6fd
--- /dev/null
+++ b/.github/workflows/mac_12.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Mac 12
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/alpine.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ runs-on: macos-12
+ caches: brew pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml
new file mode 100644
index 000000000..72e301ba0
--- /dev/null
+++ b/.github/workflows/markdown.yaml
@@ -0,0 +1,54 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# M a r k D o w n
+# ============================================================================ #
+
+---
+name: Markdown
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.md'
+ - .mdlrc
+ - .mdl.rb
+ - .markdownlint.rb
+ - .github/workflows/markdown.yaml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ Markdown:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Markdown
+ uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master
diff --git a/.github/workflows/pypy2.yaml b/.github/workflows/pypy2.yaml
deleted file mode 100644
index f605de0f2..000000000
--- a/.github/workflows/pypy2.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: PyPy 2
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [pypy2]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/pypy2.yaml.disabled b/.github/workflows/pypy2.yaml.disabled
new file mode 100644
index 000000000..01080bcc8
--- /dev/null
+++ b/.github/workflows/pypy2.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 2
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy2
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy2
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/pypy3.yaml b/.github/workflows/pypy3.yaml
deleted file mode 100644
index 1cb9ef4f7..000000000
--- a/.github/workflows/pypy3.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: PyPy 3
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [pypy3]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/pypy3.yaml.disabled b/.github/workflows/pypy3.yaml.disabled
new file mode 100644
index 000000000..101b36751
--- /dev/null
+++ b/.github/workflows/pypy3.yaml.disabled
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: PyPy 3
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: PyPy3
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: pypy3
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python2.7.yaml b/.github/workflows/python2.7.yaml
deleted file mode 100644
index 485e4859b..000000000
--- a/.github/workflows/python2.7.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Python 2.7
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [2.7]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/python2.7.yaml.disabled b/.github/workflows/python2.7.yaml.disabled
new file mode 100644
index 000000000..1813931e7
--- /dev/null
+++ b/.github/workflows/python2.7.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 2.7
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python2.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 2.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 2.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.10.yaml b/.github/workflows/python3.10.yaml
new file mode 100644
index 000000000..8ec3008b9
--- /dev/null
+++ b/.github/workflows/python3.10.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.10
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.10.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.10
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.10"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.11.yaml b/.github/workflows/python3.11.yaml
new file mode 100644
index 000000000..0f27b0076
--- /dev/null
+++ b/.github/workflows/python3.11.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.11
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.11.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.11
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: "3.11"
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.5.yaml b/.github/workflows/python3.5.yaml
deleted file mode 100644
index 15fa37297..000000000
--- a/.github/workflows/python3.5.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Python 3.5
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [3.5]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/python3.6.yaml b/.github/workflows/python3.6.yaml
deleted file mode 100644
index 8eeb1a0e3..000000000
--- a/.github/workflows/python3.6.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Python 3.6
-
-#env:
-# DEBUG: 1
-
-on:
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [3.6]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
diff --git a/.github/workflows/python3.6.yaml.disabled b/.github/workflows/python3.6.yaml.disabled
new file mode 100644
index 000000000..91bd3e673
--- /dev/null
+++ b/.github/workflows/python3.6.yaml.disabled
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.6
+
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.6.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.6
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.6
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.7.yaml b/.github/workflows/python3.7.yaml
index 5bc6252f7..73a0045c7 100644
--- a/.github/workflows/python3.7.yaml
+++ b/.github/workflows/python3.7.yaml
@@ -4,50 +4,59 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Python 3.7
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.7.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [3.7]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.7
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.7
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.8.yaml b/.github/workflows/python3.8.yaml
index 0764a94dc..abcbc4c68 100644
--- a/.github/workflows/python3.8.yaml
+++ b/.github/workflows/python3.8.yaml
@@ -4,50 +4,59 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Python 3.8
-#env:
-# DEBUG: 1
-
-on:
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.8.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- #python-version: [2.7, 3.5, 3.6, 3.7, 3.8, pypy2, pypy3]
- python-version: [3.8]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # -${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.8
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.8
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/python3.9.yaml b/.github/workflows/python3.9.yaml
new file mode 100644
index 000000000..1ce7eb238
--- /dev/null
+++ b/.github/workflows/python3.9.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-02-04 21:40:04 +0000 (Tue, 04 Feb 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Python 3.9
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.py'
+ - pylib
+ - requirements.txt
+ - .github/workflows/python3.9.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Python 3.9
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ python-version: 3.9
+ caches: apt pip
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml
new file mode 100644
index 000000000..c2f614222
--- /dev/null
+++ b/.github/workflows/semgrep-cloud.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p C l o u d W o r k f l o w
+# ============================================================================ #
+
+# Logs results to https://semgrep.dev/
+
+---
+name: Semgrep Cloud
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep Cloud
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master
+ secrets:
+ SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml
new file mode 100644
index 000000000..04a3cf197
--- /dev/null
+++ b/.github/workflows/semgrep.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S e m g r e p G i t H u b W o r k f l o w
+# ============================================================================ #
+
+# Generates code scanning alerts in GitHub's Security tab -> Code scanning alerts
+
+# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#github-actions
+
+---
+name: Semgrep
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ semgrep:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Semgrep GitHub Security Tab
+ uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml
new file mode 100644
index 000000000..246087bd4
--- /dev/null
+++ b/.github/workflows/shellcheck.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# S h e l l C h e c k
+# ============================================================================ #
+
+# Validate any shell scripts found in the repo
+
+---
+name: ShellCheck
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.sh'
+ - .github/workflows/shellcheck.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ shellcheck:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: ShellCheck
+ uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml
new file mode 100644
index 000000000..95b2571ca
--- /dev/null
+++ b/.github/workflows/trivy.yaml
@@ -0,0 +1,62 @@
+#
+# Author: Hari Sekhon
+# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# T r i v y
+# ============================================================================ #
+
+# Scan files in the local repo
+
+---
+name: Trivy
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ trivy:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Trivy
+ uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml
index cac47c8a3..fe863f875 100644
--- a/.github/workflows/ubuntu.yaml
+++ b/.github/workflows/ubuntu.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Ubuntu
-#env:
-# DEBUG: 1
-
-on: # [push]
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: ubuntu:latest
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:latest
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_14.04.yaml b/.github/workflows/ubuntu_14.04.yaml
deleted file mode 100644
index 8e267edb4..000000000
--- a/.github/workflows/ubuntu_14.04.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Ubuntu 14.04
-
-#env:
-# DEBUG: 1
-
-on: # [push]
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: ubuntu:14.04
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/ubuntu_14.04.yaml.disabled b/.github/workflows/ubuntu_14.04.yaml.disabled
new file mode 100644
index 000000000..1442d675d
--- /dev/null
+++ b/.github/workflows/ubuntu_14.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 14.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_14.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:14.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_16.04.yaml b/.github/workflows/ubuntu_16.04.yaml
deleted file mode 100644
index d5db70c05..000000000
--- a/.github/workflows/ubuntu_16.04.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Ubuntu 16.04
-
-#env:
-# DEBUG: 1
-
-on: # [push]
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: ubuntu:16.04
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/ubuntu_16.04.yaml.disabled b/.github/workflows/ubuntu_16.04.yaml.disabled
new file mode 100644
index 000000000..a9655835d
--- /dev/null
+++ b/.github/workflows/ubuntu_16.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 16.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_16.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:16.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_18.04.yaml b/.github/workflows/ubuntu_18.04.yaml
deleted file mode 100644
index 2bf3b18a7..000000000
--- a/.github/workflows/ubuntu_18.04.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: Tue Feb 4 09:53:28 2020 +0000
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-name: Ubuntu 18.04
-
-#env:
-# DEBUG: 1
-
-on: # [push]
- push:
- branches:
- - master
- schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
-
-jobs:
- build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: ubuntu:18.04
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
diff --git a/.github/workflows/ubuntu_18.04.yaml.disabled b/.github/workflows/ubuntu_18.04.yaml.disabled
new file mode 100644
index 000000000..4b510ba1e
--- /dev/null
+++ b/.github/workflows/ubuntu_18.04.yaml.disabled
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 18.04
+
+on:
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_18.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:18.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_20.04.yaml b/.github/workflows/ubuntu_20.04.yaml
index bc0c83aac..47d0512ac 100644
--- a/.github/workflows/ubuntu_20.04.yaml
+++ b/.github/workflows/ubuntu_20.04.yaml
@@ -4,44 +4,82 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: Ubuntu 20.04
-#env:
-# DEBUG: 1
-
-on: # [push]
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_20.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- container: ubuntu:20.04
- env:
- repo: devops-python-tools
- steps:
- # untars repo in docker container so git submodule update fails
- #- uses: actions/checkout@v2
- - name: install git & make
- run: apt-get update -qq && apt-get install -qy git make
- - name: git clone
- run: cd /tmp && git clone "https://github.com/harisekhon/$repo"
- - name: init
- run: cd "/tmp/$repo" && git submodule update --init --recursive
- - name: build
- run: cd "/tmp/$repo" && make ci
- - name: test
- run: cd "/tmp/$repo" && make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:20.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_22.04.yaml b/.github/workflows/ubuntu_22.04.yaml
new file mode 100644
index 000000000..ba8a549b1
--- /dev/null
+++ b/.github/workflows/ubuntu_22.04.yaml
@@ -0,0 +1,85 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+---
+name: Ubuntu 22.04
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_22.04.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ container: ubuntu:22.04
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/ubuntu_github.yaml b/.github/workflows/ubuntu_github.yaml
index 08b8f24a3..0037e4ba3 100644
--- a/.github/workflows/ubuntu_github.yaml
+++ b/.github/workflows/ubuntu_github.yaml
@@ -4,42 +4,81 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+---
name: GitHub Actions Ubuntu
-#env:
-# DEBUG: 1
-
-on: # [push]
+on: # yamllint disable-line rule:truthy
push:
branches:
- master
+ paths-ignore:
+ - '**/*.md'
+ - '.github/workflows/*'
+ - '!.github/workflows/ubuntu_github.yaml'
+ - '**/Dockerfile'
+ - '**/Jenkinsfile'
+ - '**/.envrc*'
+ - .checkov.yaml
+ - .dockerignore
+ - .gcloudignore
+ - .editorconfig
+ - '.gitconfig*'
+ - .gitignore
+ - .grype.yaml
+ - .hound.yml
+ - .terraformignore
+ - Jenkinsfile
+ - .appveyor.yml
+ - .buildkite/pipeline.yml
+ - .circleci/config.yml
+ - .cirrus.yml
+ - .concourse.yml
+ - .drone.yml
+ - .gitlab-ci.yml
+ - .gocd.yml
+ - .scrutinizer.yml
+ - .semaphore/semaphore.yml
+ - .travis.yml
+ - .werckerignore
+ - azure-pipelines.yml
+ - bitbucket-pipelines.yml
+ - buddy.yml
+ - buildspec.yml
+ - cloudbuild.yaml
+ - codefresh.yml
+ - codeship.yml
+ - shippable.yml
+ - wercker.yml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
schedule:
- # * is a special character in YAML so you have to quote this string
- - cron: '0 7 * * *'
+ - cron: '0 7 * * *'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
build:
- #name: build
- timeout-minutes: 60
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-devops-python-tools # ${{ hashFiles('**/requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-devops-python-tools
- - name: init
- run: make init
- - name: build
- run: make ci
- - name: test
- run: make test
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Build
+ uses: HariSekhon/GitHub-Actions/.github/workflows/make.yaml@master
+ with:
+ caches: apt pip cpanm
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
new file mode 100644
index 000000000..ab6ab9638
--- /dev/null
+++ b/.github/workflows/validate.yaml
@@ -0,0 +1,60 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# V a l i d a t i o n
+# ============================================================================ #
+
+# Run all custom validations against files in the repo
+
+---
+name: Validation
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ pull_request:
+ branches:
+ - master
+ - main
+ paths-ignore:
+ - '**/*.md'
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ #schedule:
+ # - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ validate:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Validate
+ uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml
new file mode 100644
index 000000000..3f6265a6a
--- /dev/null
+++ b/.github/workflows/xml.yaml
@@ -0,0 +1,56 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# X M L
+# ============================================================================ #
+
+# Validate any XML files found in the repo
+
+---
+name: XML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.xml'
+ - .github/workflows/xml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+jobs:
+ check_xml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check XML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master
diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml
new file mode 100644
index 000000000..f2a4ca129
--- /dev/null
+++ b/.github/workflows/yaml.yaml
@@ -0,0 +1,64 @@
+#
+# Author: Hari Sekhon
+# Date: Tue Feb 4 09:53:28 2020 +0000
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# Y A M L
+# ============================================================================ #
+
+# Validate any YAML files found in the repo
+
+---
+name: YAML
+
+on: # yamllint disable-line rule:truthy
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ pull_request:
+ branches:
+ - master
+ - main
+ paths:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - .github/workflows/yaml.yaml
+ workflow_dispatch:
+ inputs:
+ debug:
+ type: boolean
+ required: false
+ default: false
+ schedule:
+ - cron: '0 0 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ check_yaml:
+ # github.event.repository context not available in scheduled workflows
+ #if: github.event.repository.fork == false
+ if: github.repository_owner == 'HariSekhon'
+ name: Check YAML
+ uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master
+ with:
+ debug: ${{ github.event.inputs.debug }}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index af2a6a22b..625220b32 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,18 +3,24 @@
# Author: Hari Sekhon
# Date: Sun Feb 23 19:02:10 2020 +0000
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# G i t L a b C I
+# ============================================================================ #
+
# https://docs.gitlab.com/ee/ci/yaml/README.html
+#include: '.gitlab/*.y*ml'
+
image: ubuntu:18.04
job:
diff --git a/.gitmodules b/.gitmodules
index f5871372d..30fa60a67 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,14 +1,16 @@
[submodule "pylib"]
path = pylib
- url = https://github.com/harisekhon/pylib
+ url = https://github.com/HariSekhon/pylib
branch = master
[submodule "bash-tools"]
path = bash-tools
- url = https://github.com/harisekhon/bash-tools
+ url = https://github.com/HariSekhon/DevOps-Bash-tools
branch = master
[submodule "sql"]
path = sql
url = https://github.com/HariSekhon/SQL-scripts
+ branch = master
[submodule "templates"]
path = templates
url = https://github.com/HariSekhon/Templates
+ branch = master
diff --git a/.gocd.yml b/.gocd.yml
deleted file mode 100644
index 54f7a3067..000000000
--- a/.gocd.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-# vim:ts=2:sts=2:sw=2:et
-#
-# Author: Hari Sekhon
-# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020)
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# License: see accompanying Hari Sekhon LICENSE file
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-# https://github.com/tomzo/gocd-yaml-config-plugin#setup
-
-# https://docs.gocd.org/current/configuration/configuration_reference.html
-
-format_version: 3
-pipelines:
- devops-python-tools:
- group: defaultGroup
- label_template: ${COUNT}
- lock_behavior: none
- display_order: -1
- materials:
- git:
- git: https://github.com/harisekhon/devops-python-tools
- shallow_clone: false
- auto_update: true
- branch: master
- stages:
- - build-and-test:
- fetch_materials: true
- keep_artifacts: false
- clean_workspace: false
- approval:
- type: success
- allow_only_on_success: false
- jobs:
-# apt-update:
-# timeout: 10
-# tasks:
-# - exec:
-# command: apt
-# arguments:
-# - update
-# run_if: passed
-# install-make:
-# timeout: 10
-# tasks:
-# - exec:
-# command: apt
-# arguments:
-# - install
-# - -qy
-# - git
-# - make
-# run_if: passed
- ci-bootstrap:
- timeout: 10
- tasks:
- - exec:
- command: setup/ci_bootstrap.sh
- run_if: passed
- init:
- timeout: 10
- tasks:
- - exec:
- command: make
- arguments:
- - init
- run_if: passed
- build:
- timeout: 60
- tasks:
- - exec:
- command: make
- arguments:
- - ci
- run_if: passed
- test:
- timeout: 60
- tasks:
- - exec:
- command: make
- arguments:
- - test
- run_if: passed
diff --git a/.ipython-notebook-pyspark.00-pyspark-setup.py b/.ipython-notebook-pyspark.00-pyspark-setup.py
index 4dde897da..1de53b3fb 100644
--- a/.ipython-notebook-pyspark.00-pyspark-setup.py
+++ b/.ipython-notebook-pyspark.00-pyspark-setup.py
@@ -24,4 +24,4 @@
sys.path.insert(0, os.path.join(spark_home, 'python'))
for lib in glob.glob(os.path.join(spark_home, 'python/lib/py4j-*-src.zip')):
sys.path.insert(0, lib)
-execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))
+execfile(os.path.join(spark_home, 'python/pyspark/shell.py')) # pylint: disable=undefined-variable
diff --git a/.mdl.rb b/.mdl.rb
new file mode 100644
index 000000000..f8f9b004c
--- /dev/null
+++ b/.mdl.rb
@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# vim:ts=4:sts=4:sw=4:et:filetype=ruby
+#
+# Author: Hari Sekhon
+# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024)
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+all
+#exclude_rule 'MD001'
+#exclude_rule 'MD003'
+#exclude_rule 'MD005'
+exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af
+#exclude_rule 'MD012'
+exclude_rule 'MD013' # long lines cannot be split if they are URLs
+#exclude_rule 'MD022'
+#exclude_rule 'MD025'
+#exclude_rule 'MD031'
+#exclude_rule 'MD032'
+exclude_rule 'MD033' # inline HTML is important for formatting
+exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines
+#exclude_rule 'MD039'
+#exclude_rule 'MD056'
diff --git a/.mdlrc b/.mdlrc
new file mode 100644
index 000000000..27e5b6895
--- /dev/null
+++ b/.mdlrc
@@ -0,0 +1,5 @@
+mdlrc_dir = File.expand_path('..', __FILE__)
+
+style_file = File.join(mdlrc_dir, '.mdl.rb')
+
+style style_file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..787fb79ea
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,74 @@
+#
+# Author: Hari Sekhon
+# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https///github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P r e - C o m m i t
+# ============================================================================ #
+
+---
+fail_fast: false
+#exclude: *.tmp$
+
+repos:
+
+ # will accept anything that 'git clone' understands
+ # this means you can set this to a local git repo to develop your own hook repos interactively
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.6.0
+ hooks:
+ - id: check-yaml
+ # Common errors
+ #- id: end-of-file-fixer # ruins .gitignore Icon\r
+ - id: trailing-whitespace
+ args: [--markdown-linebreak-ext=md]
+ # Git style
+ - id: check-added-large-files
+ - id: check-merge-conflict
+ - id: check-vcs-permalinks
+ #- id: forbid-new-submodules
+ # Cross platform
+ - id: check-case-conflict
+ - id: mixed-line-ending
+ args: [--fix=lf]
+ # Security
+ - id: detect-aws-credentials
+ args: ['--allow-missing-credentials']
+
+ # rewrites python files with useless changes like changing single quotes to double quotes
+ #- repo: https://github.com/psf/black
+ # rev: 24.8.0
+ # hooks:
+ # - id: black
+
+ # Git secrets Leaks
+ - repo: https://github.com/awslabs/git-secrets.git
+ # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error:
+ #
+ # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file
+ #
+ rev: 5357e18
+ hooks:
+ - id: git-secrets
+
+ - repo: https://github.com/markdownlint/markdownlint
+ rev: v0.12.0
+ hooks:
+ - id: markdownlint
+ name: Markdownlint
+ description: Run markdownlint on your Markdown files
+ entry: mdl
+ args: [-s, .mdl.rb]
+ language: ruby
+ files: \.(md|mdown|markdown)$
diff --git a/.pylintrc b/.pylintrc
index 65e97ab22..2006619da 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,378 +1,655 @@
-[MASTER]
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2006-06-28 23:25:09 +0100 (Wed, 28 Jun 2006)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# P y L i n t C o n f i g
+# ============================================================================ #
+
+# pylint --generate-rcfile >> .pylintrc
+
+[MAIN]
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Clear in-memory caches upon conclusion of linting. Useful if running pylint
+# in a server-like mode.
+clear-cache-post-run=no
+
+# Load and enable all available extensions. Use --list-extensions to see a list
+# all available extensions.
+#enable-all-extensions=
+
+# In error mode, messages with a category besides ERROR or FATAL are
+# suppressed, and no reports are done by default. Error mode is compatible with
+# disabling specific errors.
+#errors-only=
+
+# Always return a 0 (non-error) status code, even if lint errors are found.
+# This is primarily useful in continuous integration scripts.
+#exit-zero=
-# Specify a configuration file.
-#rcfile=
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-allow-list=
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
+# for backward compatibility.)
+extension-pkg-whitelist=
+
+# Return non-zero exit code if any of these messages/categories are detected,
+# even if score is above --fail-under value. Syntax same as enable. Messages
+# specified are enabled, while categories only check already-enabled messages.
+fail-on=
+
+# Specify a score threshold under which the program will exit with error.
+fail-under=10
+
+# Interpret the stdin as a python script, whose filename needs to be passed as
+# the module_or_package argument.
+#from-stdin=
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=CVS
+
+# Add files or directories matching the regular expressions patterns to the
+# ignore-list. The regex matches against paths and can be in Posix or Windows
+# format. Because '\\' represents the directory delimiter on Windows systems,
+# it can't be used as an escape character.
+ignore-paths=
+
+# Files or directories matching the regular expression patterns are skipped.
+# The regex matches against base names, not paths. The default value ignores
+# Emacs file locks
+ignore-patterns=^\.#
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use, and will cap the count on Windows to
+# avoid hangs.
+jobs=1
-# Pickle collected data for later comparisons.
-persistent=yes
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
-# List of plugins (as comma separated values of python modules names) to load,
+# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins=
-# Use multiple processes to speed up Pylint.
-jobs=1
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Minimum Python version to use for version dependent checks. Will default to
+# the version used to run pylint.
+py-version=3.11
+
+# Discover python modules and packages in the file system subtree.
+recursive=no
+
+# Add paths to the list of the source roots. Supports globbing patterns. The
+# source root is an absolute path or a path relative to the current working
+# directory used to determine a package namespace for modules located under the
+# source root.
+source-roots=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code
-extension-pkg-whitelist=
+# In verbose mode, extra non-checker-related info will be displayed.
+#verbose=
-# Allow optimization of some AST trees. This will activate a peephole AST
-# optimizer, which will apply various small optimizations. For instance, it can
-# be used to obtain the result of joining multiple strings with the addition
-# operator. Joining a lot of strings can lead to a maximum recursion error in
-# Pylint and this flag can prevent that. It has one side effect, the resulting
-# AST will be different than the one from reality.
-optimize-ast=no
+[BASIC]
-[MESSAGES CONTROL]
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-confidence=
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style. If left empty, argument names will be checked with the set
+# naming style.
+#argument-rgx=
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time. See also the "--disable" option for examples.
-#enable=
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,C0111
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style. If left empty, attribute names will be checked with the set naming
+# style.
+#attr-rgx=
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+ bar,
+ baz,
+ toto,
+ tutu,
+ tata
-[REPORTS]
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html. You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style. If left empty, class attribute names will be checked
+# with the set naming style.
+#class-attribute-rgx=
-# Tells whether to display a full report or only the messages
-reports=yes
+# Naming style matching correct class constant names.
+class-const-naming-style=UPPER_CASE
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+# Regular expression matching correct class constant names. Overrides class-
+# const-naming-style. If left empty, class constant names will be checked with
+# the set naming style.
+#class-const-rgx=
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+# Regular expression matching correct class names. Overrides class-naming-
+# style. If left empty, class names will be checked with the set naming style.
+#class-rgx=
-[BASIC]
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style. If left empty, constant names will be checked with the set naming
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style. If left empty, function names will be checked with the set
+# naming style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+ j,
+ k,
+ ex,
+ Run,
+ _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style. If left empty, inline iteration names will be checked
+# with the set naming style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,input
+# Regular expression matching correct method names. Overrides method-naming-
+# style. If left empty, method names will be checked with the set naming style.
+#method-rgx=
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
+# Naming style matching correct module names.
+module-naming-style=snake_case
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
+# Regular expression matching correct module names. Overrides module-naming-
+# style. If left empty, module names will be checked with the set naming style.
+#module-rgx=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
-# Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
-# Naming hint for function names
-function-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type alias names. If left empty, type
+# alias names will be checked with the set naming style.
+#typealias-rgx=
-# Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+# Regular expression matching correct type variable names. If left empty, type
+# variable names will be checked with the set naming style.
+#typevar-rgx=
-# Naming hint for variable names
-variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
-# Regular expression matching correct constant names
-const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style. If left empty, variable names will be checked with the set
+# naming style.
+#variable-rgx=
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-# Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
+[CLASSES]
-# Naming hint for attribute names
-attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
-# Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+ __new__,
+ setUp,
+ asyncSetUp,
+ __post_init__
-# Naming hint for argument names
-argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit
-# Regular expression matching correct class attribute names
-class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
-# Regular expression matching correct inline iteration names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+[DESIGN]
-# Regular expression matching correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
+# List of regular expressions of class ancestor names to ignore when counting
+# public methods (see R0903)
+exclude-too-few-public-methods=
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
+# List of qualified class names to ignore when counting class parents (see
+# R0901)
+ignored-parents=
-# Regular expression matching correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of arguments for function / method.
+max-args=5
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
-# Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
-# Naming hint for method names
-method-name-hint=[a-z_][a-z0-9_]{2,30}$
+# Maximum number of branch for function / method body.
+max-branches=12
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=^_
+# Maximum number of locals for function / method body.
+max-locals=15
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
-[ELIF]
+# Maximum number of return / yield for function / method body.
+max-returns=6
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
[FORMAT]
-# Maximum number of characters on a single line.
-max-line-length=120
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )??$
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,dict-separator
-# Maximum number of lines in a module
-max-module-lines=1000
+[IMPORTS]
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string=' '
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
-# Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
+# Allow explicit reexports by alias from a package __init__.
+allow-reexport-from-package=no
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=
+
+# Output a graph (.gv or any supported image format) of external dependencies
+# to the given file (report RP0402 must not be disabled).
+ext-import-graph=
+
+# Output a graph (.gv or any supported image format) of all (i.e. internal and
+# external) dependencies to the given file (report RP0402 must not be
+# disabled).
+import-graph=
+
+# Output a graph (.gv or any supported image format) of internal dependencies
+# to the given file (report RP0402 must not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
[LOGGING]
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
# Logging modules to check that the string format arguments are in logging
-# function parameter format
+# function parameter format.
logging-modules=logging
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+ CONTROL_FLOW,
+ INFERENCE,
+ INFERENCE_FAILURE,
+ UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+ bad-inline-option,
+ locally-disabled,
+ file-ignored,
+ suppressed-message,
+ useless-suppression,
+ deprecated-pragma,
+ use-symbolic-message-instead,
+ missing-class-docstring,
+ missing-function-docstring,
+ super-with-arguments,
+ consider-using-f-string
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[METHOD_ARGS]
+
+# List of qualified names (i.e., library.method) which require a timeout
+# parameter e.g. 'requests.api.get,requests.api.post'
+timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
+
+
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=FIXME,
+ XXX,
+ TODO
+# Regular expression of note tags to take in consideration.
+notes-rgx=
-[SIMILARITIES]
-# Minimum lines number of a similarity.
-min-similarity-lines=4
+[REFACTORING]
-# Ignore comments when computing similarities.
-ignore-comments=yes
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit,argparse.parse_error
-# Ignore imports when computing similarities.
-ignore-imports=no
+[REPORTS]
-[SPELLING]
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'fatal', 'error', 'warning', 'refactor',
+# 'convention', and 'info' which contain the number of messages in each
+# category, as well as 'statement' which is the total number of statements
+# analyzed. This score is used by the global evaluation report (RP0004).
+evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
-# Spelling dictionary name. Available dictionaries: none. To make it working
-# install python-enchant package.
-spelling-dict=
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+msg-template=
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+#output-format=
-# A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
+# Tells whether to display a full report or only the messages.
+reports=no
-# Tells whether to store unknown words to indicated private dictionary in
-# --spelling-private-dict-file option instead of raising a message.
-spelling-store-unknown-words=no
+# Activate the evaluation score.
+score=yes
-[TYPECHECK]
+[SIMILARITIES]
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
+# Comments are removed from the similarity computation
+ignore-comments=yes
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis. It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+# Docstrings are removed from the similarity computation
+ignore-docstrings=yes
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set). This supports can work
-# with qualified names.
-ignored-classes=
+# Imports are removed from the similarity computation
+ignore-imports=yes
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=
+# Signatures are removed from the similarity computation
+ignore-signatures=yes
+# Minimum lines number of a similarity.
+min-similarity-lines=4
-[VARIABLES]
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
+[SPELLING]
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=_$|dummy
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
+# Spelling dictionary name. No available dictionaries : You need to install
+# both the python package and the system dependency for enchant to work..
+spelling-dict=
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,_cb
+# List of comma separated words that should be considered directives if they
+# appear at the beginning of a comment and should not be checked.
+spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
-[CLASSES]
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
+[STRING]
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,_fields,_replace,_source,_make
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
-[DESIGN]
-# Maximum number of arguments for function / method
-max-args=5
+[TYPECHECK]
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
-# Maximum number of locals for function / method body
-max-locals=15
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
-# Maximum number of return / yield for function / method body
-max-returns=6
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
-# Maximum number of branch for function / method body
-max-branches=12
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
-# Maximum number of statements in function / method body
-max-statements=50
+# List of symbolic message names to ignore for Mixin members.
+ignored-checks-for-mixins=no-member,
+ not-async-context-manager,
+ not-context-manager,
+ attribute-defined-outside-init
-# Maximum number of parents for a class (see R0901).
-max-parents=7
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
-# Maximum number of boolean expressions in a if statement
-max-bool-expr=5
+# Regex pattern to define which classes are considered mixins.
+mixin-class-rgx=.*[Mm]ixin
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
-[IMPORTS]
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,TERMIOS,Bastion,rexec
+[VARIABLES]
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
+# List of names allowed to shadow builtins
+allowed-redefined-builtins=
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+ _cb
-[EXCEPTIONS]
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 6e5dc9f25..59aa80836 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -4,29 +4,34 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# S e m a p h o r e C I
+# ============================================================================ #
+
# https://docs.semaphoreci.com/reference/pipeline-yaml-reference/
version: v1.0
-name: devops-python-tools
+name: DevOps-Python-tools
agent:
+ # https://docs.semaphoreci.com/reference/machine-types#linux
machine:
type: e1-standard-2
- os_image: ubuntu1804
+ os_image: ubuntu2004
execution_time_limit:
hours: 3
blocks:
- name: Linux build
run:
- when: "branch = 'master'"
+ when: "branch = 'master' AND change_in('/', {exclude: ['**/*.md']})"
#execution_time_limit:
# hours: 2
task:
@@ -39,6 +44,21 @@ blocks:
prologue:
commands:
- cache restore
+ # prevents it getting stuck on config merge prompt on installing openssh-client pulling in openssh-server
+ #
+ # causes error:
+ #
+ # Not replacing deleted config file /etc/ssh/sshd_config
+ #
+ #- sudo rm -f /etc/ssh/sshd_config
+ - export DEBIAN_FRONTEND=noninteractive
+ - sudo -E apt-get update
+ - sudo -E apt-get upgrade -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confnew"
+ - sudo dpkg --configure -a --force-confmiss --force-confnew
+ #- echo "openssh-server openssh-server/conffile-diff select keep" | sudo debconf-set-selections
+ #- sudo dpkg --configure -a --force-confdef --force-confold
+ #- sudo apt-get upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"
+ - sudo apt-get install -y openssh-server
# each job is separate and could be run on a separate machine so all steps must be together
jobs:
- name: build
@@ -65,9 +85,10 @@ blocks:
#- name: DEBUG
# value: "1"
agent:
+ # https://docs.semaphoreci.com/reference/machine-types#macos
machine:
type: a1-standard-4
- os_image: macos-xcode12
+ os_image: macos-xcode15
prologue:
commands:
- cache restore
@@ -90,7 +111,7 @@ blocks:
# - for version in $(rbenv versions | grep -v system | sed 's/^\*//'); do yes | rbenv uninstall "$version"; rbenv install "$version"; done
#
# fix for python vs pip version mismatch
- - ln -svf /usr/local/bin/python3 /usr/local/bin/python
+ - ln -svf -- /usr/local/bin/python3 /usr/local/bin/python
jobs:
- name: build
commands:
diff --git a/.travis.yml b/.travis.yml
index a0b4f9044..47f2f80b5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,14 +3,14 @@
# Author: Hari Sekhon
# Date: 2014-11-29 01:02:47 +0000 (Sat, 29 Nov 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
@@ -19,6 +19,7 @@
# https://docs.travis-ci.com/user/customizing-the-build/
+---
version: ~> 1.0
# ============================
@@ -126,10 +127,10 @@ before_cache:
cache:
- pip
- directories:
- - $HOME/.cache
- - $HOME/.cpan
- - $HOME/.cpanm
- - $HOME/.gem
+ - $HOME/.cache
+ - $HOME/.cpan
+ - $HOME/.cpanm
+ - $HOME/.gem
# ==============================================
# https://docs.travis-ci.com/user/job-lifecycle/
diff --git a/.yamllint b/.yamllint
index b5e6f8448..e29153b7f 100644
--- a/.yamllint
+++ b/.yamllint
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2019-02-26 14:42:07 +0000 (Tue, 26 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
---
diff --git a/Jenkinsfile b/Jenkinsfile
index 7739f92e3..d582bd112 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,16 +1,15 @@
-#!/usr/bin/env groovy
// vim:ts=4:sts=4:sw=4:et:filetype=groovy:syntax=groovy
//
// Author: Hari Sekhon
// Date: 2017-06-28 12:39:02 +0200 (Wed, 28 Jun 2017)
//
-// https://github.com/harisekhon/devops-python-tools
+// https://github.com/HariSekhon/DevOps-Python-tools
//
// License: see accompanying Hari Sekhon LICENSE file
//
// If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
//
-// https://www.linkedin.com/in/harisekhon
+// https://www.linkedin.com/in/HariSekhon
//
// ========================================================================== //
@@ -49,7 +48,7 @@ pipeline {
stages {
stage ('Checkout') {
steps {
- checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/harisekhon/devops-python-tools']]])
+ checkout([$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '', url: 'https://github.com/HariSekhon/DevOps-Python-tools']]])
}
}
diff --git a/LICENSE b/LICENSE
index 03b527051..4860c1cf5 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,45 +1,7 @@
-=======================================
-HARI SEKHON LICENSE Revision 2013112300
-=======================================
+Copyright 2015 Hari Sekhon
-Copyright (c) 2006 onwards, Hari Sekhon
-All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-Redistribution and use in source and binary forms, with or without
-modification, is permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-3. All advertising materials mentioning features or use of this software
- must display the following acknowledgement:
- This product includes software developed by Hari Sekhon.
-4. Neither the name Hari Sekhon nor any affiliates may be used to endorse or
- promote products derived from this software without specific prior written
- permission.
-5. Modifications may be released to the public only with prior written permission
- from Hari Sekhon. Forking on GitHub is permitted for the purpose of creating
- patch pull requests back to the original repository.
-6. Private modifications may be made to suit requirements, but any modifications
- to this work, whether publicly disclosed or not, must be sent back to
- Hari Sekhon via GitHub (https://github.com/harisekhon/devops-python-tools) or
- LinkedIn (https://www.linkedin.com/in/harisekhon)
- and must come under this same license. Any such modifications may be
- reincorporated for the improvement of this software.
-7. This work may not be sold without prior written permission from Hari Sekhon
-8. This license may change at any time and the latest revision supersedes
- all prior revisions.
-9. Alternative licensing must be agreed in writing with Hari Sekhon prior
- to public availability.
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-THIS SOFTWARE IS PROVIDED BY Hari Sekhon ''AS IS'' AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL Hari Sekhon OR ANY AFFILIATED BODY BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
index 66b95292f..1d98ac524 100644
--- a/Makefile
+++ b/Makefile
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules to non-system python otherwise they're not found by programs.
@@ -21,15 +21,15 @@
#
# Alpine:
#
-# apk add --no-cache git make && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make
+# apk add --no-cache git make && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
#
# Debian / Ubuntu:
#
-# apt-get update && apt-get install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make
+# apt-get update && apt-get install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
#
# RHEL / CentOS:
#
-# yum install -y make git && git clone https://github.com/harisekhon/devops-python-tools pytools && cd pytools && make
+# yum install -y make git && git clone https://github.com/HariSekhon/DevOps-Python-tools pytools && cd pytools && make
# ===================
@@ -86,12 +86,12 @@ init:
# @$(MAKE) $@c
%.pyc:: %.py
@# this utility script supports taking .pyc or .pyo names and still does the right thing
- @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
python -m py_compile $< && \
echo && \
echo Generated $@
%.pyo:: %.py
- @bash-tools/python_pip_install_for_script.sh $@ --exclude harisekhon && \
+ @PIP=$(PIP) bash-tools/python/python_pip_install_for_script.sh $@ --exclude harisekhon && \
python -O -m py_compile $< && \
echo && \
echo Generated $@
@@ -116,20 +116,20 @@ python: pylib
fi
@# only install pip packages not installed via system packages
- @#$(SUDO_PIP) pip install --upgrade -r requirements.txt
- @#$(SUDO_PIP) pip install -r requirements.txt
- @PIP_OPTS="--ignore-installed" bash-tools/python_pip_install_if_absent.sh requirements.txt
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
+ @#$(SUDO_PIP) $(PIP) install -r requirements.txt
+ @PIP=$(PIP) PIP_OPTS="--ignore-installed" bash-tools/python/python_pip_install_if_absent.sh requirements.txt
@# python-krbV dependency doesn't build on Mac any more and is unmaintained and not ported to Python 3
@# python_pip_install_if_absent.sh would import snakebite module and not trigger to build the enhanced snakebite with [kerberos] bit
- bash-tools/setup/python_install_snakebite.sh
+ PIP=$(PIP) bash-tools/setup/python_install_snakebite.sh || :
# Python >= 3.4 - try but accept failure in case we're not on the right version of Python
- @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then bash-tools/python_pip_install.sh "avro-python3"; fi
- bash-tools/python_pip_install.sh "avro-python3" || :
+ @#if [ "$$(echo "$$(python -V 2>&1 | grep -Eo '[[:digit:]]+\.[[:digit:]]+') >= 3.4" | bc -l)" = 1 ]; then PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3"; fi
+ PIP=$(PIP) bash-tools/python/python_pip_install.sh "avro-python3" || :
@# for impyla
- @#$(SUDO_PIP) pip install --upgrade setuptools || :
+ @#$(SUDO_PIP) $(PIP) install --upgrade setuptools || :
@#
@# snappy may fail to install on Mac not finding snappy-c.h - workaround:
@#
@@ -140,21 +140,21 @@ python: pylib
@# /usr/local/include/snappy-c.h
@#
@# sudo su
- @# LD_RUN_PATH=/usr/local/include pip install snappy
+ @# LD_RUN_PATH=/usr/local/include $(PIP) install snappy
@#
- @#$(SUDO_PIP) pip install --upgrade -r requirements.txt
+ @#$(SUDO_PIP) $(PIP) install --upgrade -r requirements.txt
@# for ipython-notebook-pyspark.py
- @#$(SUDO_PIP) pip install jinja2
+ @#$(SUDO_PIP) PIP=$(PIP) install jinja2
@# HiveServer2
- @#$(SUDO_PIP) pip install pyhs2
+ @#$(SUDO_PIP) $(PIP) install pyhs2
@# Impala
- @#$(SUDO_PIP) pip install impyla
+ @#$(SUDO_PIP) $(PIP) install impyla
@# must downgrade happybase library to work on Python 2.6
- @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) pip install --upgrade "happybase==0.9"; fi
+ @#if [ "$$(python -c 'import sys; sys.path.append("pylib"); import harisekhon; print(harisekhon.utils.getPythonVersion())')" = "2.6" ]; then $(SUDO_PIP) $(PIP) install --upgrade "happybase==0.9"; fi
@# Python >= 2.7 - won't build on 2.6, handle separately and accept failure
- @bash-tools/python_pip_install.sh "ipython[notebook]" || :
+ @PIP=$(PIP) bash-tools/python/python_pip_install.sh "ipython[notebook]" || :
@echo
$(MAKE) pycompile
@echo
@@ -165,7 +165,7 @@ python: pylib
.PHONY: parquet-tools
parquet-tools:
- @BIN='.' bash-tools/setup/install_parquet-tools.sh
+ @BIN='.' bash-tools/install/install_parquet-tools.sh
.PHONY: apk-packages-extra
apk-packages-extra:
@@ -175,7 +175,7 @@ apk-packages-extra:
.PHONY: apt-packages-extra
apt-packages-extra:
- if [ -z "$(NOJAVA)" ]; then which java || $(SUDO) apt-get install -y default-jdk; fi
+ if [ -z "$(NOJAVA)" ]; then which java || bash-tools/packages/apt_install_packages.sh default-jdk; fi
# for validate_multimedia.py
# available in Alpine 2.6, 2.7 and 3.x
@@ -189,8 +189,8 @@ apk-packages-multimedia:
# Debian 9 Stretch onwards, not available in Debian 8 Jessie
.PHONY: apt-packages-multimedia
apt-packages-multimedia:
- $(SUDO) apt-get update
- $(SUDO) apt-get install -y --no-install-recommends ffmpeg
+ $(SUDO) apt-get update -o DPkg::Lock::Timeout=1200
+ $(SUDO) apt-get install -o DPkg::Lock::Timeout=1200 -y --no-install-recommends ffmpeg
# for validate_multimedia.py
.PHONY: yum-packages-multimedia
@@ -200,9 +200,10 @@ yum-packages-multimedia:
.PHONY: jython
jython:
- if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
- if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
- if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ @#if [ -x /sbin/apk ]; then apk add --no-cache wget expect; fi
+ @#if [ -x /usr/bin/apt-get ]; then apt-get install -y wget expect; fi
+ @#if [ -x /usr/bin/yum ]; then yum install -y wget expect; fi
+ bash-tools/packages/install_packages.sh wget expect
sh jython_install.sh
.PHONY: test-lib
@@ -210,12 +211,13 @@ test-lib:
cd pylib && $(MAKE) test
.PHONY: test
-test: test-lib
+#test: test-lib
+test:
tests/all.sh
.PHONY: basic-test
basic-test: test-lib
- bash-tools/check_all.sh
+ bash-tools/checks/check_all.sh
.PHONY: install
install: build
diff --git a/README.md b/README.md
index f0d4b1863..5e1fe9bca 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,12 @@
-Hari Sekhon - DevOps Python Tools
-=================================
-
-[](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard)
-[](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools)
-[](https://app.codiga.io/project/8839/dashboard)
-[](https://app.codiga.io/project/8839/dashboard)
-[](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/context:python)
-[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
-[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
-[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
-[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
-[](https://lgtm.com/projects/g/HariSekhon/DevOps-Python-tools/alerts/)
-[](https://github.com/harisekhon/devops-python-tools/stargazers)
-[](https://github.com/harisekhon/devops-python-tools/network)
+# Hari Sekhon - DevOps Python Tools
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/stargazers)
+[](https://github.com/HariSekhon/DevOps-Python-tools/network)
+[](https://github.com/boyter/scc/)
+[](https://github.com/boyter/scc/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/LICENSE)
+[](https://www.linkedin.com/in/HariSekhon/)
[](https://github.com/HariSekhon/DevOps-Python-tools/commits/master)
-[](https://github.com/HariSekhon/DevOps-Python-tools)
@@ -24,11 +16,19 @@ Hari Sekhon - DevOps Python Tools
[](https://pyup.io/repos/github/HariSekhon/DevOps-Python-tools/)
-->
+[](https://www.codacy.com/gh/HariSekhon/DevOps-Python-tools/dashboard)
+[](https://www.codefactor.io/repository/github/harisekhon/DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/dashboard?id=HariSekhon_DevOps-Python-tools)
+[](https://sonarcloud.io/summary/new_code?id=HariSekhon_DevOps-Python-tools)
+
[](https://github.com/HariSekhon/DevOps-Python-tools)
[](https://github.com/HariSekhon/DevOps-Python-tools)
[](https://hub.docker.com/r/harisekhon/github/)
[](https://github.com/HariSekhon/Dockerfiles)
-[](https://hub.docker.com/r/harisekhon/github)
+[](https://hub.docker.com/r/harisekhon/pytools)
[](https://hub.docker.com/r/harisekhon/pytools/)
[](https://seladb.github.io/StarTrack-js/#/preload?r=HariSekhon,Nagios-Plugins&r=HariSekhon,Dockerfiles&r=HariSekhon,DevOps-Python-tools&r=HariSekhon,DevOps-Perl-tools&r=HariSekhon,DevOps-Bash-tools&r=HariSekhon,HAProxy-configs&r=HariSekhon,SQL-scripts)
[](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/STARCHARTS.md)
@@ -37,88 +37,87 @@ Hari Sekhon - DevOps Python Tools
[](http://microbadger.com/#/images/harisekhon/pytools)
-->
-[](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md)
+[](https://harisekhon.github.io/CI-CD/)
[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/Jenkinsfile)
-[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.concourse.yml)
-[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.gocd.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.concourse.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/.gocd.yml)
[](https://github.com/HariSekhon/TeamCity-CI)
-[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml)
+[](https://circleci.com/gh/HariSekhon/DevOps-Python-tools)
+[](https://buildkite.com/hari-sekhon/devops-python-tools)
[](https://ci.appveyor.com/project/HariSekhon/DevOps-Python-tools/branch/master)
[](https://cloud.drone.io/HariSekhon/DevOps-Python-tools)
-[](https://circleci.com/gh/HariSekhon/DevOps-Python-tools)
-[](https://app.codeship.com/projects/387251)
-[](https://app.shippable.com/github/HariSekhon/DevOps-Python-tools/dashboard/jobs)
[](https://g.codefresh.io/pipelines/edit/new/builds?id=5e58e2e6353f5d1ada385bf2&pipeline=DevOps-Python-tools&projects=GitHub&projectId=5e52ca8ea284e00f882ea992&context=github&filter=page:1;pageSize:10;timeFrameStart:week)
-[](https://buildkite.com/hari-sekhon/devops-python-tools)
[](https://cirrus-ci.com/github/HariSekhon/DevOps-Python-tools)
[](https://harisekhon.semaphoreci.com/projects/DevOps-Python-tools)
-[](https://app.wercker.com/harisekhon/DevOps-Python-tools/runs)
[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buddy.yml)
-
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/shippable.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/.travis.yml)
[](https://dev.azure.com/harisekhon/GitHub/_build/latest?definitionId=8&branchName=master)
-[](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines)
-[](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/)
-[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/buildspec.yml)
-[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cloudbuild.yaml)
+[](https://gitlab.com/HariSekhon/DevOps-Python-tools/pipelines)
+[](https://bitbucket.org/harisekhon/devops-python-tools/addon/pipelines/home#!/)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/buildspec.yml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/blob/master/cicd/cloudbuild.yaml)
-[](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools)
[](https://github.com/HariSekhon/DevOps-Python-tools)
[](https://gitlab.com/HariSekhon/DevOps-Python-tools)
+[](https://dev.azure.com/harisekhon/GitHub/_git/DevOps-Python-tools)
[](https://bitbucket.org/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/shellcheck.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/json.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/yaml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/xml.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/markdown.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/validate.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/kics.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/grype.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/semgrep-cloud.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/trivy.yaml)
+
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_alpine.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_debian.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_fedora.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/docker_pytools_ubuntu.yaml)
+
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22GitHub+Actions+Ubuntu%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Mac+10.15%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_11.yaml)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions/workflows/mac_12.yaml)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+14.04%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+16.04%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+18.04%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+20.04%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Ubuntu+22.04%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+8%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+9%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+10%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+7%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22CentOS+8%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+11%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Debian+12%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Fedora%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Alpine+3%22)
[](https://github.com/HariSekhon/DevOps-Python-tools)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+2.7%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.5%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.6%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.7%22)
[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.8%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+2%22)
-[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22PyPy+3%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.9%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.10%22)
+[](https://github.com/HariSekhon/DevOps-Python-tools/actions?query=workflow%3A%22Python+3.11%22)
[git.io/pytools](https://git.io/pytools)
-### AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools
+## AWS, Docker, Spark, Hadoop, HBase, Hive, Impala, Python & Linux Tools
DevOps, Cloud, Big Data, NoSQL, Python & Linux tools. All programs have `--help`.
-See Also:
-
- - [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools)
- - [DevOps Perl Tools](https://github.com/harisekhon/devops-perl-tools)
- - [DevOps Golang Tools](https://github.com/HariSekhon/DevOps-Golang-tools)
- - [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins)
-
-repos which contains hundreds more scripts and programs for Cloud, Big Data, SQL, NoSQL, Web and Linux.
-
Hari Sekhon
Cloud & Big Data Contractor, United Kingdom
-[](https://www.linkedin.com/in/harisekhon/)
-###### (you're welcome to connect with me on LinkedIn)
+[](https://www.linkedin.com/in/HariSekhon/)
+
*(you're welcome to connect with me on LinkedIn)*
-##### Make sure you run ```make update``` if updating and not just ```git pull``` as you will often need the latest library submodule and possibly new upstream libraries. #####
+**Make sure you run `make update` if updating and not just `git pull` as you will often need the latest library submodule and possibly new upstream libraries**
## Quick Start
@@ -127,47 +126,57 @@ Cloud & Big Data Contractor, United Kingdom
All programs and their pre-compiled dependencies can be found ready to run on [DockerHub](https://hub.docker.com/r/harisekhon/pytools/).
List all programs:
-```
+
+```shell
docker run harisekhon/pytools
```
+
Run any given program:
-```
+
+```shell
docker run harisekhon/pytools
```
### Automated Build from source
installs git, make, pulls the repo and build the dependencies:
-```
+
+```shell
curl -L https://git.io/python-bootstrap | sh
```
or manually:
-```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
make
```
-To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension instead of `.py`:
-```
+To only install pip dependencies for a single script, you can just type make and the filename with a `.pyc` extension
+instead of `.py`:
+
+```shell
make anonymize.pyc
```
-Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/devops-python-tools#detailed-build-instructions) further down for more information.
+Make sure to read [Detailed Build Instructions](https://github.com/HariSekhon/DevOps-Python-tools#detailed-build-instructions) further down for more information.
-Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/harisekhon/devops-python-tools#jython-for-hadoop-utils) for details.
+Some Hadoop tools with require Jython, see [Jython for Hadoop Utils](https://github.com/HariSekhon/DevOps-Python-tools#jython-for-hadoop-utils) for details.
### Usage
-All programs come with a ```--help``` switch which includes a program description and the list of command line options.
+All programs come with a `--help` switch which includes a program description and the list of command line options.
-Environment variables are supported for convenience and also to hide credentials from being exposed in the process list eg. ```$PASSWORD```, ```$TRAVIS_TOKEN```. These are indicated in the ```--help``` descriptions in brackets next to each option and often have more specific overrides with higher precedence eg. ```$AMBARI_HOST```, ```$HBASE_HOST``` take priority over ```$HOST```.
+Environment variables are supported for convenience and also to hide credentials from being exposed in the process list
+eg. `$PASSWORD`, `$TRAVIS_TOKEN`. These are indicated in the `--help` descriptions in brackets next to each option and
+often have more specific overrides with higher precedence eg. `$AMBARI_HOST`, `$HBASE_HOST` take priority over `$HOST`.
### DevOps Python Tools - Inventory
- Linux:
- - ```anonymize.py``` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing lists)
+ - `anonymize.py` - anonymizes your configs / logs from files or stdin (for pasting to Apache Jira tickets or mailing
+ - lists)
- anonymizations include these and more:
- hostnames / domains / FQDNs
- email addresses
@@ -176,115 +185,201 @@ Environment variables are supported for convenience and also to hide credentials
- Kerberos principals
- LDAP sensitive fields (eg. CN, DN, OU, UID, sAMAccountName, member, memberOf...)
- Cisco & Juniper ScreenOS configurations passwords, shared keys and SNMP strings
- - ```anonymize_custom.conf``` - put regex of your Name/Company/Project/Database/Tables to anonymize to ``````
- - placeholder tokens indicate what was stripped out (eg. ``````, ``````, ``````)
- - ```--ip-prefix``` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes communicating with each other to compare configs and log communications
- - ```--hash-hostnames``` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support teams can differentiate hosts in clusters
- - ```anonymize_parallel.sh``` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation important for anonymization rules, as well as maintaining file content order. On servers this parallelization can result in a 30x speed up for large log files
- - ```find_duplicate_files.py``` - finds duplicate files in one or more directory trees via multiple methods including file basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
- - ```find_active_server.py``` - finds fastest responding healthy server or active master in high availability deployments, useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc). Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See further down for more details and sub-programs that simplify usage for many of the most common cluster technologies
- - ```welcome.py``` - cool spinning welcome message greeting your username and showing last login time and user to put in your shell's ```.profile``` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
+ - `anonymize_custom.conf` - put regex of your Name/Company/Project/Database/Tables to anonymize to ``
+ - placeholder tokens indicate what was stripped out (eg. ``, ``, ``)
+ - `--ip-prefix` leaves the last IP octect to aid in cluster debugging to still see differentiated nodes
+ communicating with each other to compare configs and log communications
+ - `--hash-hostnames` - hashes hostnames to look like Docker temporary container ID hostnames so that vendors support
+ teams can differentiate hosts in clusters
+ - `anonymize_parallel.sh` - splits files in to multiple parts and runs `anonymize.py` on each part in parallel
+ before re-joining back in to a file of the same name with a `.anonymized` suffix. Preserves order of evaluation
+ important for anonymization rules, as well as maintaining file content order. On servers this parallelization can
+ result in a 30x speed up for large log files
+ - `find_duplicate_files.py` - finds duplicate files in one or more directory trees via multiple methods including file
+ basename, size, MD5 comparison of same sized files, or bespoke regex capture of partial file basename
+ - `find_active_server.py` - finds fastest responding healthy server or active master in high availability deployments,
+ useful for scripting against clustered technologies (eg. Elasticsearch, Hadoop, HBase, Cassandra etc).
+ Multi-threaded for speed and highly configurable - socket, http, https, ping, url and/or regex content match. See
+ further down for more details and sub-programs that simplify usage for many of the most common cluster technologies
+ - `welcome.py` - cool spinning welcome message greeting your username and showing last login time and user to put in
+ your shell's `.profile` (there is also a perl version in my [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) repo)
- [Amazon Web Services](https://aws.amazon.com/):
- - ```aws_users_access_key_age.py``` - lists all users access keys, status, date of creation and age in days. Optionally filters for active keys and older than N days (for key rotation governance)
- - ```aws_users_unused_access_keys.py``` - lists users access keys that haven't been used in the last N days or that have never been used (these should generally be removed/disabled). Optionally filters for only active keys
- - ```aws_users_last_used.py``` - lists all users and their days since last use across both passwords and access keys. Optionally filters for users not used in the last N days to find old accounts to remove
- - ```aws_users_pw_last_used.py``` - lists all users and dates since their passwords were last used. Optionally filters for users with passwords not used in the last N days
+ - `aws_users_access_key_age.py` - lists all users access keys, status, date of creation and age in days. Optionally
+ filters for active keys and older than N days (for key rotation governance)
+ - `aws_users_unused_access_keys.py` - lists users access keys that haven't been used in the last N days or that have
+ never been used (these should generally be removed/disabled). Optionally filters for only active keys
+ - `aws_users_last_used.py` - lists all users and their days since last use across both passwords and access keys.
+ Optionally filters for users not used in the last N days to find old accounts to remove
+ - `aws_users_pw_last_used.py` - lists all users and dates since their passwords were last used. Optionally filters for
+ users with passwords not used in the last N days
- [Google Cloud Platform](https://cloud.google.com/):
- [GCF](https://cloud.google.com/functions) - Google Cloud Functions written in Python:
- - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is triggered by [Cloud Scheduler](https://cloud.google.com/scheduler)
- - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and [Cloud Scheduler](https://cloud.google.com/scheduler) jobs
- - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP address - use this to test your VPC connector public routing, comparison with firewall rules etc.
- - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via VPC connector routing
- - ```gcp_service_account_credential_keys.py``` - lists all GCP service account credential keys for a given project with their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days
+ - [gcp_cloud_function_sql_export/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_sql_export) - runs [Cloud SQL](https://cloud.google.com/sql) export backups to
+ [GCS](https://cloud.google.com/storage), subscribing to [PubSub](https://cloud.google.com/pubsub) topic that is
+ triggered by [Cloud Scheduler](https://cloud.google.com/scheduler)
+ - see the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools/) repo for several related GCP SQL to set up service account permissions and
+ [Cloud Scheduler](https://cloud.google.com/scheduler) jobs
+ - [gcp_cloud_function_ifconfig/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_ifconfig) - debug your cloud function public networking by determining its public IP
+ address - use this to test your VPC connector public routing, comparison with firewall rules etc.
+ - [gcp_cloud_function_proxy/](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/gcp_cloud_function_proxy) - debug your cloud function networking by querying a given URL to check its
+ accessibility, returning the HTTP status code and content. Use this to validate access through firewall rules via
+ VPC connector routing
+ - `gcp_service_account_credential_keys.py` - lists all GCP service account credential keys for a given project with
+ their age and expiry details, optionally filtering by non-expiring, already expired, or will expire within N days
- [Docker](https://www.docker.com/):
- - ```docker_registry_show_tags.py``` / ```dockerhub_show_tags.py``` / ```quay_show_tags.py``` - shows tags for docker repos in a docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests across versions in a simple bash for loop, eg. ```docker_pull_all_tags.sh```
- - ```dockerhub_search.py``` - search DockerHub with a configurable number of returned results (older official `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned to the termainal and how many DockerHub has in total (use ```-q / --quiet``` to return only the image names for easy shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg. ```docker_pull_all_images.sh``` and can be chained with ```dockerhub_show_tags.py``` to download all tagged versions for all docker images eg. ```docker_pull_all_images_all_tags.sh```
- - ```dockerfiles_check_git*.py``` - check Git tags & branches align with the containing Dockerfile's ```ARG *_VERSION```
+ - `docker_registry_show_tags.py` / `dockerhub_show_tags.py` / `quay_show_tags.py` - shows tags for docker repos in a
+ docker registry or on [DockerHub](https://hub.docker.com/u/harisekhon/) or [Quay.io](https://quay.io/) - Docker CLI doesn't support this yet but it's a very
+ useful thing to be able to see live on the command line or use in shell scripts (use `-q`/`--quiet` to return only
+ the tags for easy shell scripting). You can use this to pre-download all tags of a docker image before running tests
+ across versions in a simple bash for loop, eg. `docker_pull_all_tags.sh`
+ - `dockerhub_search.py` - search DockerHub with a configurable number of returned results (older official
+ `docker search` was limited to only 25 results), using `--verbose` will also show you how many results were returned
+ to the termainal and how many DockerHub has in total (use `-q / --quiet` to return only the image names for easy
+ shell scripting). This can be used to download all of my DockerHub images in a simple bash for loop eg.
+ `docker_pull_all_images.sh` and can be chained with `dockerhub_show_tags.py` to download all tagged versions for all
+ docker images eg. `docker_pull_all_images_all_tags.sh`
+ - `dockerfiles_check_git*.py` - check Git tags & branches align with the containing Dockerfile's `ARG *_VERSION`
- [Spark](https://spark.apache.org/) & Data Format Converters:
- - ```spark_avro_to_parquet.py``` - PySpark Avro => Parquet converter
- - ```spark_parquet_to_avro.py``` - PySpark Parquet => Avro converter
- - ```spark_csv_to_avro.py``` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
- - ```spark_csv_to_parquet.py``` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
- - ```spark_json_to_avro.py``` - PySpark JSON => Avro converter
- - ```spark_json_to_parquet.py``` - PySpark JSON => Parquet converter
- - ```xml_to_json.py``` - XML to JSON converter
- - ```json_to_xml.py``` - JSON to XML converter
- - ```json_to_yaml.py``` - JSON to YAML converter
- - ```json_docs_to_bulk_multiline.py``` - converts json files to bulk multi-record one-line-per-json-document format for pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json / directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard output for convenient command line chaining and redirection, optionally continues on error, collects broken records to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
- - ```yaml_to_json.py``` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON)
- - see also ```validate_*.py``` further down for all these formats and more
+ - `spark_avro_to_parquet.py` - PySpark Avro => Parquet converter
+ - `spark_parquet_to_avro.py` - PySpark Parquet => Avro converter
+ - `spark_csv_to_avro.py` - PySpark CSV => Avro converter, supports both inferred and explicit schemas
+ - `spark_csv_to_parquet.py` - PySpark CSV => Parquet converter, supports both inferred and explicit schemas
+ - `spark_json_to_avro.py` - PySpark JSON => Avro converter
+ - `spark_json_to_parquet.py` - PySpark JSON => Parquet converter
+ - `xml_to_json.py` - XML to JSON converter
+ - `json_to_xml.py` - JSON to XML converter
+ - `json_to_yaml.py` - JSON to YAML converter
+ - `json_docs_to_bulk_multiline.py` - converts json files to bulk multi-record one-line-per-json-document format for
+ pre-processing and loading to big data systems like [Hadoop](http://hadoop.apache.org/) and
+ [MongoDB](https://www.mongodb.com/), can recurse directory trees, and mix json-doc-per-file / bulk-multiline-json /
+ directories / standard input, combines all json documents and outputs bulk-one-json-document-per-line to standard
+ output for convenient command line chaining and redirection, optionally continues on error, collects broken records
+ to standard error for logging and later reprocessing for bulk batch jobs, even supports single quoted json while not
+ technically valid json is used by MongoDB and even handles embedded double quotes in 'single quoted json'
+ - `yaml_to_json.py` - YAML to JSON converter (because some APIs like GitLab CI Validation API require JSON)
+ - see also `validate_*.py` further down for all these formats and more
- [Hadoop](http://hadoop.apache.org/) ecosystem & NoSQL:
- [Ambari](https://hortonworks.com/apache/ambari/):
- - ```ambari_blueprints.py``` - Blueprint cluster templating and deployment tool using Ambari API
+ - `ambari_blueprints.py` - Blueprint cluster templating and deployment tool using Ambari API
- list blueprints
- fetch all blueprints or a specific blueprint to local json files
- blueprint an existing cluster
- create a new cluster using a blueprint
- - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for proper revision control
+ - sorts and prettifies the resulting JSON template for deterministic config and line-by-line diff necessary for
+ proper revision control
- optionally strips out the excessive and overly specific configs to create generic more reusable templates
- - see the ```ambari_blueprints/``` directory for a variety of Ambari blueprint templates generated by and deployable using this tool
- - ```ambari_ams_*.sh``` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
- - ```ambari_cancel_all_requests.sh``` - cancel all ongoing operations using the Ambari API
- - ```ambari_trigger_service_checks.py``` - trigger service checks using the Ambari API
+ - see the `ambari_blueprints/` directory for a variety of Ambari blueprint templates generated by and deployable
+ using this tool
+ - `ambari_ams_*.sh` - query the Ambari Metrics Collector API for a given metrics, list all metrics or hosts
+ - `ambari_cancel_all_requests.sh` - cancel all ongoing operations using the Ambari API
+ - `ambari_trigger_service_checks.py` - trigger service checks using the Ambari API
- [Hadoop](http://hadoop.apache.org/) HDFS:
- - ```hdfs_find_replication_factor_1.py``` - finds HDFS files with replication factor 1, optionally resetting them to replication factor 3 to avoid missing block alerts during datanode maintenance windows
- - ```hdfs_time_block_reads.jy``` - HDFS per-block read timing debugger with datanode and rack locations for a given file or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
- - ```hdfs_files_native_checksums.jy``` - fetches native HDFS checksums for quicker file comparisons (about 100x faster than doing hdfs dfs -cat | md5sum)
- - ```hdfs_files_stats.jy``` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree showing block size, replication factor, underfilled blocks and small files
+ - `hdfs_find_replication_factor_1.py` - finds HDFS files with replication factor 1, optionally resetting them to
+ replication factor 3 to avoid missing block alerts during datanode maintenance windows
+ - `hdfs_time_block_reads.jy` - HDFS per-block read timing debugger with datanode and rack locations for a given file
+ or directory tree. Reports the slowest Hadoop datanodes in descending order at the end. Helps find cluster data
+ layer bottlenecks such as slow datanodes, faulty hardware or misconfigured top-of-rack switch ports.
+ - `hdfs_files_native_checksums.jy` - fetches native HDFS checksums for quicker file comparisons (about 100x faster
+ than doing `hdfs dfs -cat | md5sum`)
+ - `hdfs_files_stats.jy` - fetches HDFS file stats. Useful to generate a list of all files in a directory tree
+ showing block size, replication factor, underfilled blocks and small files
- [Hive](https://hive.apache.org/) / [Impala](https://impala.apache.org/):
- - ```hive_schemas_csv.py``` / ```impala_schemas_csv.py``` - dumps all databases, tables, columns and types out in CSV format to standard output
+ - `hive_schemas_csv.py` / `impala_schemas_csv.py` - dumps all databases, tables, columns and types out in CSV format
+ to standard output
The following programs can all optionally filter by database / table name regex:
- - ```hive_foreach_table.py``` / ```impala_foreach_table.py``` - execute any query or statement against every Hive / Impala table
- - ```hive_tables_row_counts.py``` / ```impala_tables_row_counts.py``` - outputs tables row counts. Useful for reconciliation between cluster migrations
- - ```hive_tables_column_counts.py``` / ```impala_tables_column_counts.py``` - outputs tables column counts. Useful for finding unusually wide tables
- - ```hive_tables_row_column_counts.py``` / ```impala_tables_row_column_counts.py``` - outputs tables row and column counts. Useful for finding unusually big tables
- - ```hive_tables_row_counts_any_nulls.py``` / ```impala_tables_row_counts_any_nulls.py``` - outputs tables row counts where any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or subtle ETL bugs
- - ```hive_tables_null_columns.py``` / ```impala_tables_null_columns.py``` - outputs tables columns containing only NULLs. Useful for catching data quality problems or subtle ETL bugs
- - ```hive_tables_null_rows.py``` / ```impala_tables_null_rows.py``` - outputs tables row counts where all fields contain NULLs. Useful for catching data quality problems or subtle ETL bugs
- - ```hive_tables_metadata.py``` / ```impala_tables_metadata.py``` - outputs for each table the matching regex metadata DDL property from describe table
- - ```hive_tables_locations.py``` / ```impala_tables_locations.py``` - outputs for each table its data location
+ - `hive_foreach_table.py` / `impala_foreach_table.py` - execute any query or statement against every Hive / Impala
+ table
+ - `hive_tables_row_counts.py` / `impala_tables_row_counts.py` - outputs tables row counts. Useful for reconciliation
+ between cluster migrations
+ - `hive_tables_column_counts.py` / `impala_tables_column_counts.py` - outputs tables column counts. Useful for
+ finding unusually wide tables
+ - `hive_tables_row_column_counts.py` / `impala_tables_row_column_counts.py` - outputs tables row and column counts.
+ Useful for finding unusually big tables
+ - `hive_tables_row_counts_any_nulls.py` / `impala_tables_row_counts_any_nulls.py` - outputs tables row counts where
+ any field is NULL. Useful for reconciliation between cluster migrations or catching data quality problems or
+ subtle ETL bugs
+ - `hive_tables_null_columns.py` / `impala_tables_null_columns.py` - outputs tables columns containing only NULLs.
+ Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_null_rows.py` / `impala_tables_null_rows.py` - outputs tables row counts where all fields contain
+ NULLs. Useful for catching data quality problems or subtle ETL bugs
+ - `hive_tables_metadata.py` / `impala_tables_metadata.py` - outputs for each table the matching regex metadata DDL
+ property from describe table
+ - `hive_tables_locations.py` / `impala_tables_locations.py` - outputs for each table its data location
- [HBase](https://hbase.apache.org/):
- - ```hbase_generate_data.py``` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table, with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
- - ```hbase_show_table_region_ranges.py``` - dumps HBase table region ranges information, useful when pre-splitting tables
- - ```hbase_table_region_row_distribution.py``` - calculates the distribution of rows across regions in an HBase table, giving per region row counts and % of total rows for the table as well as median and quartile row counts per regions
- - ```hbase_table_row_key_distribution.py``` - calculates the distribution of row keys by configurable prefix length in an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row counts per prefix
- - ```hbase_compact_tables.py``` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating on all tables or takes an optional regex and compacts only matching tables.
- - ```hbase_flush_tables.py``` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an optional regex and flushes only matching tables.
- - ```hbase_regions_by_*size.py``` - queries given RegionServers JMX to lists topN regions by storeFileSize or memStoreSize, ascending or descending
- - ```hbase_region_requests.py``` - calculates requests per second per region across all given RegionServers or average since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes / total requests per second. Useful for watching more granular region stats to detect region hotspotting
- - ```hbase_regionserver_requests.py``` - calculates requests per regionserver second across all given regionservers or average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read, write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular RegionServer stats to detect RegionServer hotspotting
- - ```hbase_regions_least_used.py``` - finds topN biggest/smallest regions across given RegionServers than have received the least requests (requests below a given threshold)
+ - `hbase_generate_data.py` - inserts random generated data in to a given [HBase](https://hbase.apache.org/) table,
+ with optional skew support with configurable skew percentage. Useful for testing region splitting, balancing, CI
+ tests etc. Outputs stats for number of rows written, time taken, rows per sec and volume per sec written.
+ - `hbase_show_table_region_ranges.py` - dumps HBase table region ranges information, useful when pre-splitting
+ tables
+ - `hbase_table_region_row_distribution.py` - calculates the distribution of rows across regions in an HBase table,
+ giving per region row counts and % of total rows for the table as well as median and quartile row counts per
+ regions
+ - `hbase_table_row_key_distribution.py` - calculates the distribution of row keys by configurable prefix length in
+ an HBase table, giving per prefix row counts and % of total rows for the table as well as median and quartile row
+ counts per prefix
+ - `hbase_compact_tables.py` - compacts HBase tables (for off-peak compactions). Defaults to finding and iterating
+ on all tables or takes an optional regex and compacts only matching tables.
+ - `hbase_flush_tables.py` - flushes HBase tables. Defaults to finding and iterating on all tables or takes an
+ optional regex and flushes only matching tables.
+ - `hbase_regions_by_*size.py` - queries given RegionServers JMX to lists topN regions by storeFileSize or
+ memStoreSize, ascending or descending
+ - `hbase_region_requests.py` - calculates requests per second per region across all given RegionServers or average
+ since RegionServer startup, configurable intervals and count, can filter to any combination of reads / writes /
+ total requests per second. Useful for watching more granular region stats to detect region hotspotting
+ - `hbase_regionserver_requests.py` - calculates requests per regionserver second across all given regionservers or
+ average since regionserver(s) startup(s), configurable interval and count, can filter to any combination of read,
+ write, total, rpcScan, rpcMutate, rpcMulti, rpcGet, blocked per second. Useful for watching more granular
+ RegionServer stats to detect RegionServer hotspotting
+ - `hbase_regions_least_used.py` - finds topN biggest/smallest regions across given RegionServers than have received
+ the least requests (requests below a given threshold)
- [OpenTSDB](http://opentsdb.net/):
- - ```opentsdb_import_metric_distribution.py``` - calculates metric distribution in bulk import file(s) to find data skew and help avoid HBase region hotspotting
- - ```opentsdb_list_metrics*.sh``` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase tables with optionally their created date, sorted ascending
+ - `opentsdb_import_metric_distribution.py` - calculates metric distribution in bulk import file(s) to find data skew
+ and help avoid HBase region hotspotting
+ - `opentsdb_list_metrics*.sh` - lists OpenTSDB metric names, tagk or tagv via OpenTSDB API or directly from HBase
+ tables with optionally their created date, sorted ascending
- [Pig](https://pig.apache.org/)
- - ```pig-text-to-elasticsearch.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Elasticsearch](https://www.elastic.co/products/elasticsearch)
- - ```pig-text-to-solr.pig``` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
- - ```pig_udfs.jy``` - Pig Jython UDFs for Hadoop
-- ```find_active_server.py``` - returns first available healthy server or active master in high availability deployments, useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex content match, multi-threaded for speed. Designed to extend tools that only accept a single ```--host``` option but for which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you want to query cluster wide information available from any online peer (eg. Elasticsearch)
- - The following are simplified specialisations of the above program, just pass host arguments, all the details have been baked in, no switches required
- - ```find_active_hadoop_namenode.py``` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
- - ```find_active_hadoop_resource_manager.py``` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
- - ```find_active_hbase_master.py``` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
- - ```find_active_hbase_thrift.py``` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run multiple of these for load balancing)
- - ```find_active_hbase_stargate.py``` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server (run multiple of these for load balancing)
- - ```find_active_apache_drill.py``` - returns first available [Apache Drill](https://drill.apache.org/) node
- - ```find_active_cassandra.py``` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node
- - ```find_active_impala*.py``` - returns first available [Impala](https://impala.apache.org/) node of either Impalad, Catalog or Statestore
- - ```find_active_presto_coordinator.py``` - returns first available [Presto](https://prestodb.io/) Coordinator
- - ```find_active_kubernetes_api.py``` - returns first available [Kubernetes](https://kubernetes.io/) API server
- - ```find_active_oozie.py``` - returns first active [Oozie](http://oozie.apache.org/) server
- - ```find_active_solrcloud.py``` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
- - ```find_active_elasticsearch.py``` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
- - see also: [Advanced HAProxy configurations](https://github.com/harisekhon/haproxy-configs) which are part of the [Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins)
+ - `pig-text-to-elasticsearch.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Elasticsearch](https://www.elastic.co/products/elasticsearch)
+ - `pig-text-to-solr.pig` - bulk index unstructured files in [Hadoop](http://hadoop.apache.org/) to
+ [Solr](http://lucene.apache.org/solr/) / [SolrCloud clusters](https://wiki.apache.org/solr/SolrCloud)
+ - `pig_udfs.jy` - Pig Jython UDFs for Hadoop
+- `find_active_server.py` - returns first available healthy server or active master in high availability deployments,
+ useful for chaining with single argument tools. Configurable tests include socket, http, https, ping, url and/or regex
+ content match, multi-threaded for speed. Designed to extend tools that only accept a single `--host` option but for
+ which the technology has later added multi-master support or active-standby masters (eg. Hadoop, HBase) or where you
+ want to query cluster wide information available from any online peer (eg. Elasticsearch)
+ - The following are simplified specialisations of the above program, just pass host arguments, all the details have
+ been baked in, no switches required
+ - `find_active_hadoop_namenode.py` - returns active [Hadoop](http://hadoop.apache.org/) Namenode in HDFS HA
+ - `find_active_hadoop_resource_manager.py` - returns active [Hadoop](http://hadoop.apache.org/) Resource Manager in Yarn HA
+ - `find_active_hbase_master.py` - returns active [HBase](https://hbase.apache.org/) Master in HBase HA
+ - `find_active_hbase_thrift.py` - returns first available [HBase](https://hbase.apache.org/) Thrift Server (run
+ multiple of these for load balancing)
+ - `find_active_hbase_stargate.py` - returns first available [HBase](https://hbase.apache.org/) Stargate rest server
+ (run multiple of these for load balancing)
+ - `find_active_apache_drill.py` - returns first available [Apache Drill](https://drill.apache.org/) node
+ - `find_active_cassandra.py` - returns first available [Apache Cassandra](https://cassandra.apache.org/) node
+ - `find_active_impala*.py` - returns first available [Impala](https://impala.apache.org/) node of either Impalad,
+ Catalog or Statestore
+ - `find_active_presto_coordinator.py` - returns first available [Presto](https://prestodb.io/) Coordinator
+ - `find_active_kubernetes_api.py` - returns first available [Kubernetes](https://kubernetes.io/) API server
+ - `find_active_oozie.py` - returns first active [Oozie](http://oozie.apache.org/) server
+ - `find_active_solrcloud.py` - returns first available [Solr](http://lucene.apache.org/solr/) / [SolrCloud](https://wiki.apache.org/solr/SolrCloud) node
+ - `find_active_elasticsearch.py` - returns first available [Elasticsearch](https://www.elastic.co/products/elasticsearch) node
+ - see also: [Advanced HAProxy configurations](https://github.com/HariSekhon/HAProxy-configs) which are part of the
+ [Advanced Nagios Plugins Collection](https://github.com/HariSekhon/Nagios-Plugins)
- [Travis CI](https://travis-ci.org/):
- - ```travis_last_log.py``` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo - useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
- - ```travis_debug_session.py``` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot debug launcher for Travis CI
-- ```selenium_hub_browser_test.py``` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as Chrome and Firefox to fetch a given URL and content/regex match the result
+ - `travis_last_log.py` - fetches [Travis CI](https://travis-ci.org/) latest running / completed / failed build log for given repo -
+ useful for quickly getting the log of the last failed build when CCMenu or BuildNotify applets turn red
+ - `travis_debug_session.py` - launches a [Travis CI](https://travis-ci.org/) interactive debug build session via Travis API, tracks
+ session creation and drops user straight in to the SSH shell on the remote Travis build, very convenient one shot
+ debug launcher for Travis CI
+- `selenium_hub_browser_test.py` - checks [Selenium Grid Hub / Selenoid](https://www.selenium.dev/documentation/en/grid/) is working by calling browsers such as
+ Chrome and Firefox to fetch a given URL and content/regex match the result
- Data Validation (useful in CI):
- - ```validate_*.py``` - validate files, directory trees and/or standard input streams
+ - `validate_*.py` - validate files, directory trees and/or standard input streams
- supports the following file formats:
- Avro
- CSV
@@ -294,40 +389,44 @@ Environment variables are supported for convenience and also to hide credentials
- Parquet
- XML
- YAML
- - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`, `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
- - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data & Linux technologies
+ - directories are recursed, testing any files with relevant matching extensions (`.avro`, `.csv`, `json`, `parquet`,
+ `.ini`/`.properties`, `.ldif`, `.xml`, `.yml`/`.yaml`)
+ - used for Continuous Integration tests of various adjacent Spark data converters as well as configuration files for
+ things like Presto, Ambari, Apache Drill etc found in my [DockerHub](https://hub.docker.com/u/harisekhon/) images
+ [Dockerfiles master repo](https://github.com/HariSekhon/Dockerfiles) which contains docker builds and configurations for many open source Big Data &
+ Linux technologies
### Detailed Build Instructions
-##### Python VirtualEnv localized installs
-
-The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages instead of installing from PyPI then follow the Manual Build section below.
+#### Python VirtualEnv localized installs
+The automated build will use 'sudo' to install required Python PyPI libraries to the system unless running as root or it
+detects being inside a VirtualEnv. If you want to install some of the common Python libraries using your OS packages
+instead of installing from PyPI then follow the Manual Build section below.
-#### Manual Setup
+### Manual Setup
Enter the pytools directory and run git submodule init and git submodule update to fetch my library repo:
-```
-git clone https://github.com/harisekhon/devops-python-tools pytools
+```shell
+git clone https://github.com/HariSekhon/DevOps-Python-tools pytools
cd pytools
git submodule init
git submodule update
sudo pip install -r requirements.txt
```
-
-#### Offline Setup
+### Offline Setup
Download the DevOps Python Tools and Pylib git repos as zip files:
-https://github.com/HariSekhon/devops-python-tools/archive/master.zip
+
-https://github.com/HariSekhon/pylib/archive/master.zip
+
-Unzip both and move Pylib to the ```pylib``` folder under DevOps Python Tools.
+Unzip both and move Pylib to the `pylib` folder under DevOps Python Tools.
-```
+```shell
unzip devops-python-tools-master.zip
unzip pylib-master.zip
@@ -336,165 +435,246 @@ mv -v pylib-master pylib
mv -vf pylib pytools/
```
-Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
+Proceed to install PyPI modules for whichever programs you want to use using your usual procedure - usually an internal
+mirror or proxy server to PyPI, or rpms / debs (some libraries are packaged by Linux distributions).
All PyPI modules are listed in the `requirements.txt` and `pylib/requirements.txt` files.
Internal Mirror example ([JFrog Artifactory](https://jfrog.com/artifactory/) or similar):
-```
+```shell
sudo pip install --index https://host.domain.com/api/pypi/repo/simple --trusted host.domain.com -r requirements.txt
```
Proxy example:
-```
+```shell
sudo pip install --proxy hari:mypassword@proxy-host:8080 -r requirements.txt
```
-##### Mac OS X
+#### Mac OS X
-The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing `git` is enough to trigger Xcode install).
+The automated build also works on Mac OS X but you'll need to install [Apple XCode](https://developer.apple.com/download/) (on recent Macs just typing
+`git` is enough to trigger Xcode install).
-I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
+I also recommend you get [HomeBrew](https://brew.sh/) to install other useful tools and libraries you may need like OpenSSL for
+development headers and tools such as wget (these are installed automatically if Homebrew is detected on Mac OS X):
-```
-bash-tools/setup/install_homebrew.sh
+```shell
+bash-tools/install/install_homebrew.sh
```
-```
+```shell
brew install openssl wget
```
If failing to build an OpenSSL lib dependency, just prefix the build command like so:
-```
+```shell
sudo OPENSSL_INCLUDE=/usr/local/opt/openssl/include OPENSSL_LIB=/usr/local/opt/openssl/lib ...
```
-You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is caused by pip 10 vs pip 9 and downgrading will work around it:
+You may get errors trying to install to Python library paths even as root on newer versions of Mac, sometimes this is
+caused by pip 10 vs pip 9 and downgrading will work around it:
-```
+```shell
sudo pip install --upgrade pip==9.0.1
make
sudo pip install --upgrade pip
make
```
-### Jython for Hadoop Utils ###
+### Jython for Hadoop Utils
The 3 Hadoop utility programs listed below require Jython (as well as Hadoop to be installed and correctly configured)
-```
+```shell
hdfs_time_block_reads.jy
hdfs_files_native_checksums.jy
hdfs_files_stats.jy
```
Run like so:
-```
+
+```shell
jython -J-cp $(hadoop classpath) hdfs_time_block_reads.jy --help
```
-The ```-J-cp $(hadoop classpath)``` part dynamically inserts the current Hadoop java classpath required to use the Hadoop APIs.
+The `-J-cp $(hadoop classpath)` part dynamically inserts the current Hadoop java classpath required to use the Hadoop
+APIs.
See below for procedure to install Jython if you don't already have it.
-##### Automated Jython Install
+#### Automated Jython Install
This will download and install jython to /opt/jython-2.7.0:
-```
+```shell
make jython
```
-##### Manual Jython Install
+#### Manual Jython Install
-Jython is a simple download and unpack and can be fetched from http://www.jython.org/downloads.html
+Jython is a simple download and unpack and can be fetched from
Then add the Jython install bin directory to the $PATH or specify the full path to the `jython` binary, eg:
-```
+```shell
/opt/jython-2.7.0/bin/jython hdfs_time_block_reads.jy ...
```
+### Configuration for Strict Domain / FQDN validation
-#### Configuration for Strict Domain / FQDN validation ####
+Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my
+[PyLib](https://github.com/HariSekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`,
+`.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal
+environments).
-Strict validations include host/domain/FQDNs using TLDs which are populated from the official IANA list is done via my [PyLib](https://github.com/harisekhon/pylib) library submodule - see there for details on configuring this to permit custom TLDs like `.local`, `.intranet`, `.vm`, `.cloud` etc. (all already included in there because they're common across companies internal environments).
-
-#### Python SSL certificate verification problems
+### Python SSL certificate verification problems
If you end up with an error like:
-```
+
+```shell
./dockerhub_show_tags.py centos ubuntu
[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:765)
```
-It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One quick fix is to do the following:
-```
+
+It can be caused by an issue with the underlying Python + libraries due to changes in OpenSSL and certificates. One
+quick fix is to do the following:
+
+```shell
sudo pip uninstall -y certifi &&
sudo pip install certifi==2015.04.28
```
-### Updating ###
+### Updating
+
+Run:
+
+```shell
+make update
+```
-Run ```make update```. This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
+This will git pull and then git submodule update which is necessary to pick up corresponding library updates.
-If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each time then run ```make update-no-recompile``` (will miss new library dependencies - do full ```make update``` if you encounter issues).
+If you update often and want to just quickly git pull + submodule update but skip rebuilding all those dependencies each
+time then run `make update-no-recompile` (will miss new library dependencies - do full `make update` if you encounter
+issues).
### Testing
[Continuous Integration](https://travis-ci.org/HariSekhon/devops-python-tools) is run on this repo with tests for success and failure scenarios:
-- unit tests for the custom supporting [python library](https://github.com/harisekhon/pylib)
+
+- unit tests for the custom supporting [python library](https://github.com/HariSekhon/pylib)
- integration tests of the top level programs using the libraries for things like option parsing
-- [functional tests](https://github.com/HariSekhon/devops-python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
+- [functional tests](https://github.com/HariSekhon/DevOps-Python-tools/tree/master/tests) for the top level programs using local test data and [Docker containers](https://hub.docker.com/u/harisekhon/)
To trigger all tests run:
-```
+```shell
make test
```
-which will start with the underlying libraries, then move on to top level integration tests and functional tests using docker containers if docker is available.
+which will start with the underlying libraries, then move on to top level integration tests and functional tests using
+docker containers if docker is available.
-### Contributions ###
+### Contributions
Patches, improvements and even general feedback are welcome in the form of GitHub pull requests and issue tickets.
-### See Also ###
+You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another
+Hortonworks guy Jonas Straub:
-- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 550+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies
+
-- [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery
+## Star History
-- [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc.
+[](https://star-history.com/#HariSekhon/DevOps-Python-tools&Date)
-- [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments
+[git.io/python-tools](https://git.io/python-tools)
+
+[git.io/pytools](https://git.io/pytools)
-- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux)
+## More Core Repos
-- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux...
+
-- [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc.
+### Knowledge
-- [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) - 50+ DockerHub public images for Docker & Kubernetes - Hadoop, Kafka, ZooKeeper, HBase, Cassandra, Solr, SolrCloud, Presto, Apache Drill, Nifi, Spark, Mesos, Consul, Riak, OpenTSDB, Jython, Advanced Nagios Plugins & DevOps Tools repos on Alpine, CentOS, Debian, Fedora, Ubuntu, Superset, H2O, Serf, Alluxio / Tachyon, FakeS3
+[](https://github.com/HariSekhon/Knowledge-Base)
+[](https://github.com/HariSekhon/Diagrams-as-Code)
+
+
+
+### DevOps Code
+
+[](https://github.com/HariSekhon/DevOps-Bash-tools)
+[](https://github.com/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/DevOps-Perl-tools)
+[](https://github.com/HariSekhon/DevOps-Golang-tools)
-You might also be interested in the following really nice Jupyter notebook for HDFS space analysis created by another Hortonworks guy Jonas Straub:
+### Containerization
-- https://github.com/mr-jstraub/HDFSQuota/blob/master/HDFSQuota.ipynb
+[](https://github.com/HariSekhon/Kubernetes-configs)
+[](https://github.com/HariSekhon/Dockerfiles)
-### Stargazers over time
+### CI/CD
-[](https://starchart.cc/HariSekhon/DevOps-Python-tools)
+[](https://github.com/HariSekhon/GitHub-Actions)
+[](https://github.com/HariSekhon/Jenkins)
-[git.io/python-tools](https://git.io/python-tools)
+### Databases - DBA - SQL
-[git.io/pytools](https://git.io/pytools)
+[](https://github.com/HariSekhon/SQL-scripts)
+
+### DevOps Reloaded
+
+[](https://github.com/HariSekhon/HAProxy-configs)
+[](https://github.com/HariSekhon/Terraform)
+[](https://github.com/HariSekhon/Packer)
+[](https://github.com/HariSekhon/Ansible)
+[](https://github.com/HariSekhon/Environments)
+
+### Monitoring
+
+[](https://github.com/HariSekhon/Nagios-Plugins)
+[](https://github.com/HariSekhon/Nagios-Plugin-Kafka)
+[](https://github.com/HariSekhon/Prometheus)
+
+### Templates
+
+[](https://github.com/HariSekhon/Templates)
+[](https://github.com/HariSekhon/Template-repo)
+
+### Desktop
+
+[](https://github.com/HariSekhon/TamperMonkey)
+[](https://github.com/HariSekhon/Hammerspoon)
+[](https://github.com/HariSekhon/MPV-Scripts)
+
+### Spotify
+
+[](https://github.com/HariSekhon/Spotify-tools)
+[](https://github.com/HariSekhon/Spotify-playlists)
+
+The rest of my original source repos are
+[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers).
+
+Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/)
+and can be re-generated using the my [Dockerfiles](https://github.com/HariSekhon/Dockerfiles) repo.
+
+
diff --git a/ambari_ams_metrics.sh b/ambari_ams_metrics.sh
index 85f24ee53..64f20f0a2 100755
--- a/ambari_ams_metrics.sh
+++ b/ambari_ams_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-16 22:14:34 +0100 (Mon, 16 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# There were 2346 metrics last I checked but this probably varies a lot based on what services are deployed
diff --git a/ambari_blueprints.py b/ambari_blueprints.py
index 82b796ef2..e5dd52217 100755
--- a/ambari_blueprints.py
+++ b/ambari_blueprints.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-08 14:09:50 +0000 (Sun, 08 Nov 2015)
# (re-instantiated from a Perl version in 2014)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/ambari_cancel_all_requests.sh b/ambari_cancel_all_requests.sh
index 15e079525..7d2df4425 100755
--- a/ambari_cancel_all_requests.sh
+++ b/ambari_cancel_all_requests.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-09-27 17:25:36 +0100 (Tue, 27 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/ambari_trigger_service_checks.py b/ambari_trigger_service_checks.py
index edf98b2bf..2afe64207 100755
--- a/ambari_trigger_service_checks.py
+++ b/ambari_trigger_service_checks.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-23 15:45:28 +0200 (Fri, 23 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/anonymize.py b/anonymize.py
index 339376c13..ebcb20ab0 100755
--- a/anonymize.py
+++ b/anonymize.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
@@ -6,16 +6,16 @@
# Date: 2018-08-08 19:02:02 +0100 (Wed, 08 Aug 2018)
# Original Date: 2013-07-18 21:17:41 +0100 (Thu, 18 Jul 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
-# ported from Perl version from DevOps Perl Tools repo (https://github.com/harisekhon/devops-perl-tools)
+# ported from Perl version from DevOps Perl Tools repo (https://github.com/HariSekhon/DevOps-Perl-tools)
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
@@ -36,7 +36,7 @@
Ignore phrases are in a similar file anonymize_ignore.conf, also adjacent to this program.
-Based on Perl Anonymize.pl from https://github.com/harisekhon/devops-perl-tools
+Based on Perl Anonymize.pl from https://github.com/HariSekhon/DevOps-Perl-tools
The Perl version is incredibly faster than Python due to the better regex engine
@@ -95,7 +95,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.10.12'
+__version__ = '0.11.0'
ip_regex = r'(?!127\.0\.0\.)' + ip_regex
subnet_mask_regex = r'(?!127\.0\.0\.)' + subnet_mask_regex
@@ -792,46 +792,47 @@ def run(self):
# allow to easily switch pre-compilation on/off for testing
# testing shows on a moderate sized file that it is a couple secs quicker to use pre-compiled regex
def compile(self, name, regex):
+ log.debug(f"compiling regex '{name}' = '{regex}'")
self.regex[name] = re.compile(regex, re.I)
#self.regex[name] = regex
def prepare_regex(self):
self.compile('hostname',
- r'(? 2018-01-:00:00
- r'(?!\d+T\d+:\d+)' + \
- r'(?!\d+[^A-Za-z0-9]|' + \
- self.custom_ignores_raw + ')' + \
- '(' + hostname_regex + ')' + \
+ r'(?!\d+T\d+:\d+)' +
+ r'(?!\d+[^A-Za-z0-9]|' +
+ self.custom_ignores_raw + ')' +
+ '(' + hostname_regex + ')' +
self.negative_host_lookbehind + r':(\d{1,5}(?!\.?\w))',
)
self.compile('domain',
# don't match java -some.net.property
- #r'(?/dev/null || :
echo
- "$srcdir/bash-tools/split.sh" --parts "$parallelism" "$filename"
+ "$srcdir/bash-tools/bin/split.sh" --parts "$parallelism" "$filename"
echo "Anonymizing parts"
for file_part in "$filename".*; do
cmd="$srcdir/anonymize.py -a $file_part > $file_part.anonymized"
diff --git a/aws_s3_presign.py b/aws_s3_presign.py
index 2fbb1331f..e1addfbbc 100755
--- a/aws_s3_presign.py
+++ b/aws_s3_presign.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-01-14 17:45:38 +0000 (Tue, 14 Jan 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/aws_users_access_key_age.py b/aws_users_access_key_age.py
index c0026570d..be218c2db 100755
--- a/aws_users_access_key_age.py
+++ b/aws_users_access_key_age.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-13 17:24:40 +0000 (Fri, 13 Dec 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -35,9 +35,9 @@
See also:
aws_users_access_key_age.sh - simpler version in the adjacent DevOps Bash Tools repo without age filtering
- - https://github.com/harisekhon/devops-bash-tools
+ - https://github.com/HariSekhon/DevOps-Bash-tools
-Advanced Nagios Plugins (https://github.com/harisekhon/nagios-plugins)
+Advanced Nagios Plugins (https://github.com/HariSekhon/Nagios-Plugins)
check_aws_access_keys_age.py
check_aws_access_keys_disabled.py
diff --git a/aws_users_last_used.py b/aws_users_last_used.py
index 9a93b7bad..e06a7bcdb 100755
--- a/aws_users_last_used.py
+++ b/aws_users_last_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
#
-# https://github.com/harisekhon/nagios-plugins
+# https://github.com/HariSekhon/Nagios-Plugins
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -35,8 +35,8 @@
See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
-- https://github.com/harisekhon/devops-bash-tools
-- https://github.com/harisekhon/nagios-plugins
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
"""
diff --git a/aws_users_pw_last_used.py b/aws_users_pw_last_used.py
index 7e00ee4e0..6810abd17 100755
--- a/aws_users_pw_last_used.py
+++ b/aws_users_pw_last_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-19 11:43:25 +0000 (Thu, 19 Dec 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -31,9 +31,9 @@
See also the DevOps Bash Tools repo and The Advanced Nagios Plugins Collection for similar tools
-https://github.com/harisekhon/devops-bash-tools
+https://github.com/HariSekhon/DevOps-Bash-tools
-https://github.com/harisekhon/nagios-plugins
+https://github.com/HariSekhon/Nagios-Plugins
"""
diff --git a/aws_users_unused_access_keys.py b/aws_users_unused_access_keys.py
index fcb244e07..befb152ee 100755
--- a/aws_users_unused_access_keys.py
+++ b/aws_users_unused_access_keys.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-16 11:37:15 +0000 (Mon, 16 Dec 2019)
#
-# https://github.com/harisekhon/nagios-plugins
+# https://github.com/HariSekhon/Nagios-Plugins
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -33,8 +33,8 @@
See also the DevOps Bash Tools and Advanced Nagios Plugins Collection repos which have more similar AWS tools
-- https://github.com/harisekhon/devops-bash-tools
-- https://github.com/harisekhon/nagios-plugins
+- https://github.com/HariSekhon/DevOps-Bash-tools
+- https://github.com/HariSekhon/Nagios-Plugins
"""
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d8f0e09ac..b430c58dd 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -3,47 +3,68 @@
# Author: Hari Sekhon
# Date: Sun Feb 23 19:02:10 2020 +0000
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# A z u r e D e v O p s P i p e l i n e
+# ============================================================================ #
+
# https://aka.ms/yaml
trigger:
-- master
+ - master
+
+variables:
+ # ubuntu version
+ os_version: '22.04'
pool:
# there is no /dev/stderr on this azure build!
#vmImage: 'ubuntu-latest'
- # Ubuntu 16.04 required for docker container support, looks like 18.04 works too
- vmImage: 'ubuntu-18.04'
+ #vmImage: 'ubuntu-22.04'
+ vmImage: 'ubuntu-$(os_version)'
# unprivileged container without sudo, cannot install dependencies
-#container: ubuntu:18.04
+#container: ubuntu:22.04
steps:
-# requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName'
-- script: env | sort
- displayName: env
+ - script: cat /etc/*-release
+ displayName: OS Release
+
+ # requires script as first key, otherwise parsing breaks with error message: Unexpected value 'displayName'
+ - script: env | sort
+ displayName: Environment
-# doesn't work in container due to unprivileged execution and lack of sudo
-#- script: sudo apt-get update && sudo apt-get install -y git make
-# displayName: install git & make
+ # doesn't work in container due to unprivileged execution and lack of sudo
+ #- script: sudo apt-get update && sudo apt-get install -y git make
+ # displayName: install git & make
-#- script: make
-# displayName: build
+ #- script: make
+ # displayName: build
-# doesn't work in vmImage build due to lack of access to normal /dev/stderr device
-# tee: /dev/stderr: No such device or address
-#- script: make test
-# displayName: test
+ # doesn't work in vmImage build due to lack of access to normal /dev/stderr device
+ # tee: /dev/stderr: No such device or address
+ #- script: make test
+ # displayName: test
-# hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-(
-- script: sudo docker run -v "$PWD":/code ubuntu:18.04 /bin/bash -c 'set -ex && cd /code && setup/ci_bootstrap.sh && make init && make ci test'
- displayName: docker build
+ # hacky workaround to Azure Pipelines ubuntu environment limitations of unprivileged container and no /dev/stderr in vmImage :-(
+ - script: |
+ sudo docker run -v "$PWD":/code "ubuntu:$(os_version)" /bin/bash -c '
+ set -ex
+ cd /code
+ setup/ci_bootstrap.sh
+ if [ -x setup/ci_git_set_dir_safe.sh ]; then
+ setup/ci_git_set_dir_safe.sh
+ fi
+ make init
+ make ci test
+ '
+ displayName: docker build
diff --git a/bash-tools b/bash-tools
index 9e172499f..11dee29ce 160000
--- a/bash-tools
+++ b/bash-tools
@@ -1 +1 @@
-Subproject commit 9e172499fad04aff95a06af08750187adbc82293
+Subproject commit 11dee29cea607445270d8bd675d1bcdecb069c14
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index a7ec1b183..cdf5243ee 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -4,18 +4,27 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-# https://confluence.atlassian.com/x/5Q4SMw
-# Only use spaces to indent your .yml configuration.
-# -----
+# ============================================================================ #
+# B i t b u c k e t C I / C D P i p e l i n e
+# ============================================================================ #
+
+# Reference:
+#
+# https://support.atlassian.com/bitbucket-cloud/docs/configure-bitbucket-pipelinesyml/
+
+# Languages:
+#
+# https://confluence.atlassian.com/x/5Q4SMw
+
# You can specify a custom docker image from Docker Hub as your build environment.
image: atlassian/default-image:2
diff --git a/buddy.yml b/buddy.yml
index aed9fc3bc..e0b57451f 100644
--- a/buddy.yml
+++ b/buddy.yml
@@ -4,40 +4,45 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# B u d d y C I
+# ============================================================================ #
+
# https://buddy.works/docs/yaml/yaml-schema
+---
- pipeline: "Build"
trigger_mode: "ON_EVERY_PUSH"
ref_name: "master"
ref_type: "BRANCH"
- target_site_url: "https://github.com/harisekhon/devops-python-tools"
+ target_site_url: "https://github.com/HariSekhon/DevOps-Python-tools"
trigger_condition: "ALWAYS"
actions:
- - action: "Execute: make ci test"
- type: "BUILD"
- working_directory: "/buddy/devops-python-tools"
- docker_image_name: "library/ubuntu"
- docker_image_tag: "18.04"
- #setup_commands:
- # this step gets cached, which results in
- # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
- # - apt update
- # - apt install -qy git make
- execute_commands:
- - setup/ci_bootstrap.sh
- - make init
- - make ci
- - make test
- volume_mappings:
- - "/:/buddy/devops-python-tools"
- shell: "BASH"
- trigger_condition: "ALWAYS"
+ - action: "Execute: make ci test"
+ type: "BUILD"
+ working_directory: "/buddy/DevOps-Python-tools"
+ docker_image_name: "library/ubuntu"
+ docker_image_tag: "18.04"
+ #setup_commands:
+ # this step gets cached, which results in
+ # E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?
+ # - apt update
+ # - apt install -qy git make
+ execute_commands:
+ - setup/ci_bootstrap.sh
+ - make init
+ - make ci
+ - make test
+ volume_mappings:
+ - "/:/buddy/DevOps-Python-tools"
+ shell: "BASH"
+ trigger_condition: "ALWAYS"
diff --git a/center.py b/center.py
index 489d15f8d..379353e68 100755
--- a/center.py
+++ b/center.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
# args: -s <<< "Auth & Config"
# args: -s <<< "GKE Clusters"
@@ -6,14 +6,14 @@
# Author: Hari Sekhon
# Date: 2016-01-29 21:05:38 +0000 (Fri, 29 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -44,7 +44,8 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.4.2'
+__version__ = '0.5.1'
+
class Center(CLI):
@@ -56,7 +57,9 @@ def __init__(self):
# this doesn't put enough spaces around ampersands, eg. in "Auth & Config"
#self.re_bound = re.compile(r'(\b)')
self.re_spaces = re.compile(r'(\s)')
+ self.re_multiple_spaces = re.compile(r'(\s){2}')
self.re_chars = re.compile(r'([^\s])(?!\s)')
+ self.re_chars_spaced = re.compile(r'([^\s])\s')
self.timeout_default = None
def add_options(self):
@@ -66,11 +69,16 @@ def add_options(self):
help='No comment prefix handling')
self.add_opt('-s', '--space', action='store_true', default=False,
help='Space all chars out, makes bigger headings')
+ self.add_opt('-u', '--unspace', action='store_true', default=False,
+ help='Removes spaces betweeen chars out, the inverse of --space')
def run(self):
log_option('width', self.get_opt('width'))
log_option('no comment prefix', self.get_opt('no_comment'))
log_option('space chars', self.get_opt('space'))
+ log_option('unspace chars', self.get_opt('unspace'))
+ if self.get_opt('space') and self.get_opt('unspace'):
+ self.usage("--space and --unspace are mutually exclusive!")
if self.args:
self.process_line(' '.join(self.args))
else:
@@ -83,6 +91,11 @@ def space(self, line):
line = self.re_chars.sub(r'\1 ', line)
return line
+ def unspace(self, line):
+ line = self.re_chars_spaced.sub(r'\1', line)
+ line = self.re_multiple_spaces.sub(r'\1', line)
+ return line
+
def process_line(self, line):
char = ''
if not line:
@@ -91,7 +104,7 @@ def process_line(self, line):
char = ' '
# preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it
line = line.strip()
- if isChars(line[0], '#'):
+ if line and isChars(line[0], '#'):
char = line[0]
line = line.lstrip(char)
elif len(line) > 1 and isChars(line[0:1], '/'):
@@ -102,9 +115,12 @@ def process_line(self, line):
line = line.lstrip(char)
if self.get_opt('space'):
line = self.space(line)
+ if self.get_opt('unspace'):
+ line = self.unspace(line)
line = line.strip()
side = int(max((self.get_opt('width') - len(line)) / 2, 0))
print(char + ' ' * side + line)
+
if __name__ == '__main__':
Center().main()
diff --git a/cicd/.concourse.yml b/cicd/.concourse.yml
new file mode 100644
index 000000000..92847121e
--- /dev/null
+++ b/cicd/.concourse.yml
@@ -0,0 +1,65 @@
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:06:48 +0000 (Sat, 21 Mar 2020)
+#
+# vim:ts=2:sts=2:sw=2:et
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# C o n c o u r s e C I
+# ============================================================================ #
+
+# https://concourse-ci.org/golang-library-example.html
+
+# https://resource-types.concourse-ci.org/
+# https://concourse-ci.org/resource-types.html
+resources:
+ - name: github
+ icon: github-circle
+ type: git
+ source:
+ uri: https://github.com/HariSekhon/DevOps-Python-tools
+ branch: master
+ #- name: daily
+ # type: time
+ # source:
+ # interval: 1d
+
+# https://concourse-ci.org/jobs.html
+jobs:
+ - name: build
+ public: false
+ plan:
+ - get: github # from resource above
+ trigger: true
+ #version: every # build every git commit, default: latest
+ - task: build
+ config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ repository: ubuntu
+ tag: latest
+ inputs:
+ - name: github
+ path: code
+ params:
+ CONCOURSE: 1
+ run:
+ path: /bin/bash
+ args:
+ - -c
+ - |
+ cd code &&
+ setup/ci_bootstrap.sh &&
+ make init &&
+ make ci test
diff --git a/cicd/.gocd.yml b/cicd/.gocd.yml
new file mode 100644
index 000000000..65583a818
--- /dev/null
+++ b/cicd/.gocd.yml
@@ -0,0 +1,94 @@
+# vim:ts=2:sts=2:sw=2:et
+#
+# Author: Hari Sekhon
+# Date: 2020-03-21 11:14:07 +0000 (Sat, 21 Mar 2020)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# G o C D
+# ============================================================================ #
+
+# https://github.com/tomzo/gocd-yaml-config-plugin#setup
+
+# https://docs.gocd.org/current/configuration/configuration_reference.html
+
+---
+format_version: 3
+pipelines:
+ DevOps-Python-tools:
+ group: defaultGroup
+ label_template: ${COUNT}
+ lock_behavior: none
+ display_order: -1
+ materials:
+ git:
+ git: https://github.com/HariSekhon/DevOps-Python-tools
+ shallow_clone: false
+ auto_update: true
+ branch: master
+ stages:
+ - build-and-test:
+ fetch_materials: true
+ keep_artifacts: false
+ clean_workspace: false
+ approval:
+ type: success
+ allow_only_on_success: false
+ jobs:
+ #apt-update:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - update
+ # run_if: passed
+ #install-make:
+ # timeout: 10
+ # tasks:
+ # - exec:
+ # command: apt
+ # arguments:
+ # - install
+ # - -qy
+ # - git
+ # - make
+ # run_if: passed
+ ci-bootstrap:
+ timeout: 10
+ tasks:
+ - exec:
+ command: setup/ci_bootstrap.sh
+ run_if: passed
+ init:
+ timeout: 10
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - init
+ run_if: passed
+ build:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - ci
+ run_if: passed
+ test:
+ timeout: 60
+ tasks:
+ - exec:
+ command: make
+ arguments:
+ - test
+ run_if: passed
diff --git a/buildspec.yml b/cicd/buildspec.yml
similarity index 97%
rename from buildspec.yml
rename to cicd/buildspec.yml
index ab099ea97..f4e887953 100644
--- a/buildspec.yml
+++ b/cicd/buildspec.yml
@@ -2,7 +2,7 @@
# Author: Hari Sekhon
# Date: 2020-12-19 15:32:28 +0000 (Sat, 19 Dec 2020)
#
-# https://github.com/HariSekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/cloudbuild.yaml b/cicd/cloudbuild.yaml
similarity index 95%
rename from cloudbuild.yaml
rename to cicd/cloudbuild.yaml
index 9212c4a5e..61e439fa3 100644
--- a/cloudbuild.yaml
+++ b/cicd/cloudbuild.yaml
@@ -2,7 +2,7 @@
# Author: Hari Sekhon
# Date: 2020-12-19 16:27:26 +0000 (Sat, 19 Dec 2020)
#
-# https://github.com/HariSekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/cloudera_navigator_tables_used.py b/cloudera_navigator_tables_used.py
index 645d344b4..bbd94dbb3 100755
--- a/cloudera_navigator_tables_used.py
+++ b/cloudera_navigator_tables_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-03-09 11:35:47 +0000 (Mon, 09 Mar 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/cloudera_navigator_tables_used_postgres.py b/cloudera_navigator_tables_used_postgres.py
index 47bf1b419..972716236 100755
--- a/cloudera_navigator_tables_used_postgres.py
+++ b/cloudera_navigator_tables_used_postgres.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-03-16 19:21:24 +0000 (Mon, 16 Mar 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/codefresh.yml b/codefresh.yml
index 542c323d8..7cddcaf7c 100644
--- a/codefresh.yml
+++ b/codefresh.yml
@@ -4,15 +4,19 @@
#
# vim:ts=2:sts=2:sw=2:et
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
+# ============================================================================ #
+# C o d e f r e s h C I
+# ============================================================================ #
+
# https://codefresh.io/docs/docs/codefresh-yaml/
version: "1.0"
diff --git a/codeship.yml b/codeship.yml
deleted file mode 100644
index 53efd9424..000000000
--- a/codeship.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2021-04-12 18:33:44 +0100 (Mon, 12 Apr 2021)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/HariSekhon/devops-python-tools
-#
-# License: see accompanying Hari Sekhon LICENSE file
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
-#
-# https://www.linkedin.com/in/HariSekhon
-#
-
-# ============================================================================ #
-# C o d e S h i p
-# ============================================================================ #
-
-# 3rd party way of doing IaC on CodeShip CI as the free edition doesn't support this
-
-# https://github.com/painless-software/codeship-yaml
-
-# Requires setting up the CodeShip commands like so:
-#
-# pip install codeship-yaml
-# codeship-yaml
-#
-# or seaparately in sections:
-#
-# Project Settings > Test Settings > Setup Commands:
-#
-# pip install codeship-yaml
-# codeship-yaml install
-#
-# Project Settings > Test Settings > Test Commands:
-#
-# codeship-yaml before_script script
-#
-# Project Settings > Deployment > (branch name)
-#
-# codeship-yaml after_success
-
----
-install:
- # these cause package installation breakages due to GPG or 403 errors, old addresses etc.
- - sudo rm -fv /etc/apt/sources.list.d/cli_assets_heroku_com_branches_stable_apt.list
- - sudo rm -fv /etc/apt/sources.list.d/apache_bintray_com_couchdb_deb.list
- - sudo rm -fv /etc/apt/sources.list.d/www_apache_org_dist_cassandra_debian.list
- - make
-#before_script:
-# - somecommand
-script:
- - make test
-#after_success:
-# - echo "Now we can deploy"
diff --git a/crunch_accounting_csv_statement_converter.py b/crunch_accounting_csv_statement_converter.py
index 6819d6ace..2e675c253 100755
--- a/crunch_accounting_csv_statement_converter.py
+++ b/crunch_accounting_csv_statement_converter.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-02-04 23:24:30 +0000 (Mon, 04 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
index 708ab7a66..61a1235bd 100755
--- a/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
+++ b/crunch_accounting_csv_statement_converter_scripts/barclaycard_crunch_accounting_csv_statement_converter.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
index 704b9defd..f708fec0c 100755
--- a/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
+++ b/crunch_accounting_csv_statement_converter_scripts/barclays_crunch_accounting_csv_statement_converter.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/crunch_accounting_csv_statement_converter_scripts/lib.sh b/crunch_accounting_csv_statement_converter_scripts/lib.sh
index 65a21926b..e77e66834 100755
--- a/crunch_accounting_csv_statement_converter_scripts/lib.sh
+++ b/crunch_accounting_csv_statement_converter_scripts/lib.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2020-05-29 12:35:16 +0100 (Fri, 29 May 2020)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_pull_all_images.sh b/docker_pull_all_images.sh
index 430ac70d5..e6d39fd0d 100755
--- a/docker_pull_all_images.sh
+++ b/docker_pull_all_images.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_pull_all_images_all_tags.sh b/docker_pull_all_images_all_tags.sh
index 3656395cb..d12c3e25b 100755
--- a/docker_pull_all_images_all_tags.sh
+++ b/docker_pull_all_images_all_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_pull_all_tags.sh b/docker_pull_all_tags.sh
index e555af230..229f3fe68 100755
--- a/docker_pull_all_tags.sh
+++ b/docker_pull_all_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-29 14:57:23 +0200 (Tue, 29 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/docker_registry_show_tags.py b/docker_registry_show_tags.py
index 50d1af743..8c18d3cdc 100755
--- a/docker_registry_show_tags.py
+++ b/docker_registry_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/dockerfiles_check_git_branches.py b/dockerfiles_check_git_branches.py
index b2e407f4b..662dce3e2 100755
--- a/dockerfiles_check_git_branches.py
+++ b/dockerfiles_check_git_branches.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
diff --git a/dockerfiles_check_git_tags.py b/dockerfiles_check_git_tags.py
index 996f787b8..3e0b627bd 100755
--- a/dockerfiles_check_git_tags.py
+++ b/dockerfiles_check_git_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-20 20:24:12 +0100 (Fri, 20 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
diff --git a/dockerhub_search.py b/dockerhub_search.py
index 9a8623b34..ba7034145 100755
--- a/dockerhub_search.py
+++ b/dockerhub_search.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
# args: harisekhon
#
# Author: Hari Sekhon
# Date: 2016-05-27 13:15:30 +0100 (Fri, 27 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -33,7 +33,7 @@
dockerhub_search.sh
-in the DevOps Bash tools repo - https://github.com/harisekhon/devops-python-tools
+in the DevOps Bash tools repo - https://github.com/HariSekhon/DevOps-Python-tools
"""
diff --git a/dockerhub_show_tags.py b/dockerhub_show_tags.py
index e0dfecbb8..59cd6cb5f 100755
--- a/dockerhub_show_tags.py
+++ b/dockerhub_show_tags.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-10 11:26:49 +0100 (Tue, 10 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_apache_drill.py b/find_active_apache_drill.py
index ff4461960..1e89fb676 100755
--- a/find_active_apache_drill.py
+++ b/find_active_apache_drill.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_apache_drill2.py b/find_active_apache_drill2.py
index 4dcdf8219..364b0eda3 100755
--- a/find_active_apache_drill2.py
+++ b/find_active_apache_drill2.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_cassandra.py b/find_active_cassandra.py
index 67360ccc3..c16ee217c 100755
--- a/find_active_cassandra.py
+++ b/find_active_cassandra.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_elasticsearch.py b/find_active_elasticsearch.py
index 42d98d4b8..be449bc79 100755
--- a/find_active_elasticsearch.py
+++ b/find_active_elasticsearch.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop2_namenode.py b/find_active_hadoop2_namenode.py
index bdb7ee5e4..7aad7e77d 100755
--- a/find_active_hadoop2_namenode.py
+++ b/find_active_hadoop2_namenode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop_namenode.py b/find_active_hadoop_namenode.py
index ff578d9ef..1f6f2d44d 100755
--- a/find_active_hadoop_namenode.py
+++ b/find_active_hadoop_namenode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hadoop_yarn_resource_manager.py b/find_active_hadoop_yarn_resource_manager.py
index d1179c7cc..f2f0e48e3 100755
--- a/find_active_hadoop_yarn_resource_manager.py
+++ b/find_active_hadoop_yarn_resource_manager.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 6 14:44:38 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_master.py b/find_active_hbase_master.py
index e89c87613..cc7de0222 100755
--- a/find_active_hbase_master.py
+++ b/find_active_hbase_master.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_stargate.py b/find_active_hbase_stargate.py
index c62b1756c..79dc122fa 100755
--- a/find_active_hbase_stargate.py
+++ b/find_active_hbase_stargate.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_hbase_thrift.py b/find_active_hbase_thrift.py
index a83594730..06e44a414 100755
--- a/find_active_hbase_thrift.py
+++ b/find_active_hbase_thrift.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala.py b/find_active_impala.py
index cce6eb399..ae086f46a 100755
--- a/find_active_impala.py
+++ b/find_active_impala.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_catalog.py b/find_active_impala_catalog.py
index 7fc7a143e..cb6ea0de3 100755
--- a/find_active_impala_catalog.py
+++ b/find_active_impala_catalog.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_impala_statestore.py b/find_active_impala_statestore.py
index 3ec124494..049ed78f5 100755
--- a/find_active_impala_statestore.py
+++ b/find_active_impala_statestore.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_kubernetes_api.py b/find_active_kubernetes_api.py
index 0ef8165f8..a93b0bc81 100755
--- a/find_active_kubernetes_api.py
+++ b/find_active_kubernetes_api.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_oozie.py b/find_active_oozie.py
index 21b9271d6..156bb51a9 100755
--- a/find_active_oozie.py
+++ b/find_active_oozie.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_presto_coordinator.py b/find_active_presto_coordinator.py
index 13c8f6546..9b0050a47 100755
--- a/find_active_presto_coordinator.py
+++ b/find_active_presto_coordinator.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Wed Sep 13 13:58:21 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_active_server.py b/find_active_server.py
index 1a127cd54..30810a448 100755
--- a/find_active_server.py
+++ b/find_active_server.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-29 15:00:36 +0100 (Thu, 29 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
r"""
@@ -96,7 +96,7 @@
See also Advanced HAProxy configurations (part of the Advanced Nagios Plugins Collection) at:
- https://github.com/harisekhon/haproxy-configs
+ https://github.com/HariSekhon/HAProxy-configs
"""
diff --git a/find_active_solrcloud.py b/find_active_solrcloud.py
index b6683fafb..659eceb59 100755
--- a/find_active_solrcloud.py
+++ b/find_active_solrcloud.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: Tue Sep 5 10:49:49 CEST 2017
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_duplicate_files.py b/find_duplicate_files.py
index 54eb70a96..4c452b03f 100755
--- a/find_duplicate_files.py
+++ b/find_duplicate_files.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-14 09:50:03 +0100 (Sun, 14 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/find_missing_files_in_sequence.py b/find_missing_files_in_sequence.py
index 6d2d8b0db..f982cbddd 100755
--- a/find_missing_files_in_sequence.py
+++ b/find_missing_files_in_sequence.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-07-31 11:03:17 +0100 (Fri, 31 Jul 2020)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/gcp_cloud_function_ifconfig/.gcloudignore b/gcp_cloud_function_ifconfig/.gcloudignore
index 8f605abc2..42de3fd00 100644
--- a/gcp_cloud_function_ifconfig/.gcloudignore
+++ b/gcp_cloud_function_ifconfig/.gcloudignore
@@ -4,7 +4,7 @@
#
# vim:ts=4:sts=4:sw=4:et
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_ifconfig/deploy.sh b/gcp_cloud_function_ifconfig/deploy.sh
index 6f7fc0ab3..39aeba85e 100755
--- a/gcp_cloud_function_ifconfig/deploy.sh
+++ b/gcp_cloud_function_ifconfig/deploy.sh
@@ -4,7 +4,7 @@
# Author: Hari Sekhon
# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_ifconfig/main.py b/gcp_cloud_function_ifconfig/main.py
index 309bc58c0..c9e50b0e7 100755
--- a/gcp_cloud_function_ifconfig/main.py
+++ b/gcp_cloud_function_ifconfig/main.py
@@ -5,7 +5,7 @@
# Author: Hari Sekhon
# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_proxy/.gcloudignore b/gcp_cloud_function_proxy/.gcloudignore
index 8f605abc2..42de3fd00 100644
--- a/gcp_cloud_function_proxy/.gcloudignore
+++ b/gcp_cloud_function_proxy/.gcloudignore
@@ -4,7 +4,7 @@
#
# vim:ts=4:sts=4:sw=4:et
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_proxy/deploy.sh b/gcp_cloud_function_proxy/deploy.sh
index a583c1d2e..d8e04e9b5 100755
--- a/gcp_cloud_function_proxy/deploy.sh
+++ b/gcp_cloud_function_proxy/deploy.sh
@@ -4,7 +4,7 @@
# Author: Hari Sekhon
# Date: 2021-05-24 16:34:19 +0100 (Mon, 24 May 2021)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_proxy/main.py b/gcp_cloud_function_proxy/main.py
index 9e525e72b..8680d455a 100755
--- a/gcp_cloud_function_proxy/main.py
+++ b/gcp_cloud_function_proxy/main.py
@@ -5,7 +5,7 @@
# Author: Hari Sekhon
# Date: 2021-05-24 16:03:30 +0100 (Mon, 24 May 2021)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_sql_export/.gcloudignore b/gcp_cloud_function_sql_export/.gcloudignore
index 8f605abc2..42de3fd00 100644
--- a/gcp_cloud_function_sql_export/.gcloudignore
+++ b/gcp_cloud_function_sql_export/.gcloudignore
@@ -4,7 +4,7 @@
#
# vim:ts=4:sts=4:sw=4:et
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_sql_export/deploy.sh b/gcp_cloud_function_sql_export/deploy.sh
index 3dd2c44f5..c5c3c3249 100755
--- a/gcp_cloud_function_sql_export/deploy.sh
+++ b/gcp_cloud_function_sql_export/deploy.sh
@@ -4,7 +4,7 @@
# Author: Hari Sekhon
# Date: 2020-10-16 10:12:26 +0100 (Fri, 16 Oct 2020)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/gcp_cloud_function_sql_export/main.py b/gcp_cloud_function_sql_export/main.py
index f67135583..6e3765667 100755
--- a/gcp_cloud_function_sql_export/main.py
+++ b/gcp_cloud_function_sql_export/main.py
@@ -5,7 +5,7 @@
# Author: Hari Sekhon
# Date: 2020-10-14 15:29:38 +0100 (Wed, 14 Oct 2020)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
@@ -93,6 +93,11 @@ def main(event, context):
}
try:
+ logging.info("Requesting project '%s' database instance '%s' runs a backup export to bucket '%s' path '%s'",
+ project,
+ instance,
+ bucket,
+ backup_uri)
request = service.instances().export(
project=project,
instance=instance,
diff --git a/gcp_service_account_credential_keys.py b/gcp_service_account_credential_keys.py
index bd4460f35..f58f86e98 100755
--- a/gcp_service_account_credential_keys.py
+++ b/gcp_service_account_credential_keys.py
@@ -1,11 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-10-29 18:02:14 +0000 (Thu, 29 Oct 2020)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/getent.py b/getent.py
index ce77aefcf..d8b701cd6 100755
--- a/getent.py
+++ b/getent.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-11-20 12:35:49 +0000 (Sun, 20 Nov 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/git_check_branches_upstream.py b/git_check_branches_upstream.py
index 0ab2e177b..27ec09865 100755
--- a/git_check_branches_upstream.py
+++ b/git_check_branches_upstream.py
@@ -1,24 +1,24 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-07-21 16:19:19 +0100 (Thu, 21 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
Tool to check Git branches have their upstream origin branch set consistently and auto-fix if necessary
-Mainly written for my https://github.com/harisekhon/Dockerfiles repo
+Mainly written for my https://github.com/HariSekhon/Dockerfiles repo
which has over 100 branches which get merged, pulled and pushed around
"""
diff --git a/hbase_compact_tables.py b/hbase_compact_tables.py
index 60f246328..7bfca40b2 100755
--- a/hbase_compact_tables.py
+++ b/hbase_compact_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-04-27 20:49:23 +0100 (Wed, 27 Apr 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_flush_tables.py b/hbase_flush_tables.py
index 04d51bc75..71be25c17 100755
--- a/hbase_flush_tables.py
+++ b/hbase_flush_tables.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-12 13:44:24 +0200 (Mon, 12 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# NOTE: 'flush' is not supported in Happybase as it's not in the Thrift API,
diff --git a/hbase_generate_data.py b/hbase_generate_data.py
index 133fa42dc..3ed31a644 100755
--- a/hbase_generate_data.py
+++ b/hbase_generate_data.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-09-14 15:19:35 +0200 (Wed, 14 Sep 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_region_requests.py b/hbase_region_requests.py
index 2af24c219..d182431c8 100755
--- a/hbase_region_requests.py
+++ b/hbase_region_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_by_memstore_size.py b/hbase_regions_by_memstore_size.py
index 47165a7a9..295653a81 100755
--- a/hbase_regions_by_memstore_size.py
+++ b/hbase_regions_by_memstore_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_by_size.py b/hbase_regions_by_size.py
index 49f7419d8..c67184c8b 100755
--- a/hbase_regions_by_size.py
+++ b/hbase_regions_by_size.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regions_least_used.py b/hbase_regions_least_used.py
index 45727dd34..752be4ed5 100755
--- a/hbase_regions_least_used.py
+++ b/hbase_regions_least_used.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_regionserver_requests.py b/hbase_regionserver_requests.py
index f61b40cbd..a8c96ebb9 100755
--- a/hbase_regionserver_requests.py
+++ b/hbase_regionserver_requests.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_scan_table_column_names.sh b/hbase_scan_table_column_names.sh
index f2d62ad44..026f7d5a8 100755
--- a/hbase_scan_table_column_names.sh
+++ b/hbase_scan_table_column_names.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-06-29 19:01:22 +0100 (Fri, 29 Jun 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/hbase_show_table_region_ranges.py b/hbase_show_table_region_ranges.py
index 2a5647fb9..585e11cf9 100755
--- a/hbase_show_table_region_ranges.py
+++ b/hbase_show_table_region_ranges.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-05 13:57:37 +0100 (Wed, 05 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_region_row_distribution.py b/hbase_table_region_row_distribution.py
index 87b4d2e34..0dfe6e024 100755
--- a/hbase_table_region_row_distribution.py
+++ b/hbase_table_region_row_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-06 10:42:59 +0100 (Thu, 06 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_table_regions_by_regionserver.sh b/hbase_table_regions_by_regionserver.sh
index 5e603e289..8017f406a 100755
--- a/hbase_table_regions_by_regionserver.sh
+++ b/hbase_table_regions_by_regionserver.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-08-24 19:34:33 +0100 (Fri, 24 Aug 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/hbase_table_regionserver_distribution.sh b/hbase_table_regionserver_distribution.sh
index 252a9a3d4..58699a877 100755
--- a/hbase_table_regionserver_distribution.sh
+++ b/hbase_table_regionserver_distribution.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:36:23 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Useful to see table region skew across regionservers
diff --git a/hbase_table_row_key_distribution.py b/hbase_table_row_key_distribution.py
index 8784d5952..6059763d7 100755
--- a/hbase_table_row_key_distribution.py
+++ b/hbase_table_row_key_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-08 09:02:01 +0100 (Sat, 08 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hbase_uncompacted_regions.sh b/hbase_uncompacted_regions.sh
index 7b092c06d..00a3302c5 100755
--- a/hbase_uncompacted_regions.sh
+++ b/hbase_uncompacted_regions.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-11 19:26:21 +0100 (Wed, 11 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Find regions that require major_compact after region migration to achieve data locality again
diff --git a/hdfs_files_native_checksums.jy b/hdfs_files_native_checksums.jy
index 6aa6e6ff2..9c558401a 100755
--- a/hdfs_files_native_checksums.jy
+++ b/hdfs_files_native_checksums.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-20 18:21:02 +0100 (Thu, 20 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -50,7 +50,7 @@ def usage(*msg):
if msg:
printerr("".join(msg))
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/hdfs_files_stats.jy b/hdfs_files_stats.jy
index 69132daee..587b3b390 100755
--- a/hdfs_files_stats.jy
+++ b/hdfs_files_stats.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -55,7 +55,7 @@ def usage(*msg):
if msg:
printerr(msg)
die("""
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/hdfs_find_replication_factor_1.py b/hdfs_find_replication_factor_1.py
index 0a38a9316..1f348556d 100755
--- a/hdfs_find_replication_factor_1.py
+++ b/hdfs_find_replication_factor_1.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2018-11-28 16:37:00 +0000 (Wed, 28 Nov 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hdfs_time_block_reads.jy b/hdfs_time_block_reads.jy
index e340cff7c..ab1812a23 100755
--- a/hdfs_time_block_reads.jy
+++ b/hdfs_time_block_reads.jy
@@ -3,7 +3,7 @@
# Author: Hari Sekhon
# Date: 2013-06-08 22:06:27 +0100 (Sat, 08 Jun 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -28,7 +28,7 @@ __version__ = '0.9.1'
import os
usage_msg = """
-Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
================================================================================
%s - version %s
diff --git a/headtail.py b/headtail.py
index ce9a43e43..ec21a3f5c 100755
--- a/headtail.py
+++ b/headtail.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-07 22:57:18 +0000 (Thu, 07 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hexanonymize.py b/hexanonymize.py
index 91728e7fc..cac35e6a0 100755
--- a/hexanonymize.py
+++ b/hexanonymize.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-01-02 17:08:32 +0000 (Thu, 02 Jan 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_compute_table_stats.py b/hive_compute_table_stats.py
index dfd1d51e5..d416f9379 100755
--- a/hive_compute_table_stats.py
+++ b/hive_compute_table_stats.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_foreach_table.py b/hive_foreach_table.py
index 1b40f4434..9d5abf6d6 100755
--- a/hive_foreach_table.py
+++ b/hive_foreach_table.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_schemas_csv.py b/hive_schemas_csv.py
index 5daf009d6..ea41f3753 100755
--- a/hive_schemas_csv.py
+++ b/hive_schemas_csv.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_column_counts.py b/hive_tables_column_counts.py
index 2902f23dd..d6cb4ce68 100755
--- a/hive_tables_column_counts.py
+++ b/hive_tables_column_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_list.py b/hive_tables_list.py
index 07c415780..64675c61f 100755
--- a/hive_tables_list.py
+++ b/hive_tables_list.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_locations.py b/hive_tables_locations.py
index af17aaed8..403e07443 100755
--- a/hive_tables_locations.py
+++ b/hive_tables_locations.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_metadata.py b/hive_tables_metadata.py
index 886c3eed8..d15d62feb 100755
--- a/hive_tables_metadata.py
+++ b/hive_tables_metadata.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_null_columns.py b/hive_tables_null_columns.py
index 5874a1565..207fcba3f 100755
--- a/hive_tables_null_columns.py
+++ b/hive_tables_null_columns.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_null_rows.py b/hive_tables_null_rows.py
index 82082d358..f1a5752f1 100755
--- a/hive_tables_null_rows.py
+++ b/hive_tables_null_rows.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_row_column_counts.py b/hive_tables_row_column_counts.py
index 660fef904..c2a5ec7e9 100755
--- a/hive_tables_row_column_counts.py
+++ b/hive_tables_row_column_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_row_counts.py b/hive_tables_row_counts.py
index b37059c9b..27aaa1a9c 100755
--- a/hive_tables_row_counts.py
+++ b/hive_tables_row_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/hive_tables_row_counts_any_nulls.py b/hive_tables_row_counts_any_nulls.py
index a1de5e461..8ea7e6b75 100755
--- a/hive_tables_row_counts_any_nulls.py
+++ b/hive_tables_row_counts_any_nulls.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_compute_table_stats.py b/impala_compute_table_stats.py
index 62f71cd4e..04d462e16 100755
--- a/impala_compute_table_stats.py
+++ b/impala_compute_table_stats.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_foreach_table.py b/impala_foreach_table.py
index a47d846cc..f4e324ac9 100755
--- a/impala_foreach_table.py
+++ b/impala_foreach_table.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_schemas_csv.py b/impala_schemas_csv.py
index d6555b542..0ac8491dd 100755
--- a/impala_schemas_csv.py
+++ b/impala_schemas_csv.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_column_counts.py b/impala_tables_column_counts.py
index b26b0ebcc..e62027542 100755
--- a/impala_tables_column_counts.py
+++ b/impala_tables_column_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_list.py b/impala_tables_list.py
index feb4e73ca..a45194ad3 100755
--- a/impala_tables_list.py
+++ b/impala_tables_list.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_locations.py b/impala_tables_locations.py
index 3be05d6f8..383bb1154 100755
--- a/impala_tables_locations.py
+++ b/impala_tables_locations.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_metadata.py b/impala_tables_metadata.py
index ba8a66cbc..501b82c93 100755
--- a/impala_tables_metadata.py
+++ b/impala_tables_metadata.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_null_columns.py b/impala_tables_null_columns.py
index ad968fbc1..2b70caf3f 100755
--- a/impala_tables_null_columns.py
+++ b/impala_tables_null_columns.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_null_rows.py b/impala_tables_null_rows.py
index c70484540..4fea8a82e 100755
--- a/impala_tables_null_rows.py
+++ b/impala_tables_null_rows.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_row_column_counts.py b/impala_tables_row_column_counts.py
index b2e867734..8cf4d6b82 100755
--- a/impala_tables_row_column_counts.py
+++ b/impala_tables_row_column_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_row_counts.py b/impala_tables_row_counts.py
index 3d748ded2..decc2982c 100755
--- a/impala_tables_row_counts.py
+++ b/impala_tables_row_counts.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/impala_tables_row_counts_any_nulls.py b/impala_tables_row_counts_any_nulls.py
index ab6a6c638..d7b49a3b2 100755
--- a/impala_tables_row_counts_any_nulls.py
+++ b/impala_tables_row_counts_any_nulls.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-11-26 10:08:52 +0000 (Tue, 26 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/ipython_notebook_pyspark.py b/ipython_notebook_pyspark.py
index 013d61ff2..4c4505015 100755
--- a/ipython_notebook_pyspark.py
+++ b/ipython_notebook_pyspark.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
#
# Author: Hari Sekhon
# Date: 6/8/2014
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
@@ -93,7 +93,7 @@
# TODO: rewrite from here as a CLI class, not using globals
if len(sys.argv) > 1:
- printerr("""Hari Sekhon - https://github.com/harisekhon/devops-python-tools
+ printerr("""Hari Sekhon - https://github.com/HariSekhon/DevOps-Python-tools
usage: %s
diff --git a/json_docs_to_bulk_multiline.py b/json_docs_to_bulk_multiline.py
index 98f4d4db9..eb49bcbeb 100755
--- a/json_docs_to_bulk_multiline.py
+++ b/json_docs_to_bulk_multiline.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-07-28 17:08:47 +0200 (Fri, 28 Jul 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/json_to_xml.py b/json_to_xml.py
index 03f5b27fa..54c0d6395 100755
--- a/json_to_xml.py
+++ b/json_to_xml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: GNU GPL version 2 (this file only), rest of this repo is licensed as per the adjacent LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/json_to_yaml.py b/json_to_yaml.py
index 657209550..674f6729f 100755
--- a/json_to_yaml.py
+++ b/json_to_yaml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-19 17:54:21 +0000 (Thu, 19 Dec 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/jython_autoinstall.exp b/jython_autoinstall.exp
index 1230a03a1..342488e6a 100755
--- a/jython_autoinstall.exp
+++ b/jython_autoinstall.exp
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: Fri Jun 17 15:12:17 2016 +0100
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set force_conservative 0 ;# set to 1 to force conservative mode even if
diff --git a/jython_install.sh b/jython_install.sh
index b56c94cac..4d3f420ee 100755
--- a/jython_install.sh
+++ b/jython_install.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-01 10:17:55 +0100 (Mon, 01 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/kics.config b/kics.config
new file mode 100644
index 000000000..51ac5712e
--- /dev/null
+++ b/kics.config
@@ -0,0 +1,44 @@
+#
+# Author: Hari Sekhon
+# Date: 2023-05-05 18:05:53 +0100 (Fri, 05 May 2023)
+#
+# vim:ts=2:sts=2:sw=2:et:filetype=yaml
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# ============================================================================ #
+# K i c s C o n f i g
+# ============================================================================ #
+
+# https://github.com/Checkmarx/kics/blob/master/docs/configuration-file.md
+
+---
+#path: assets/iac_samples
+verbose: true
+log-file: true
+#type:
+# - Dockerfile
+# - Kubernetes
+#queries-path: "assets/queries"
+exclude-paths:
+ # ignore submodules - handle them in the source repos only
+ - bash-tools/
+ - github-actions/
+ - haproxy-configs/
+ - jenkins/
+ - kubernetes-templates/
+ - lib/
+ - pylib/
+ - spotify-tools/
+ - sql/
+ - sql-keywords/
+ - templates/
+ - terraform-templates/
+#output-path: "results"
diff --git a/lib/hive_impala_cli.py b/lib/hive_impala_cli.py
index 7515abddd..958ccec69 100755
--- a/lib/hive_impala_cli.py
+++ b/lib/hive_impala_cli.py
@@ -4,14 +4,14 @@
# Author: Hari Sekhon
# Date: 2019-11-07 14:52:38 +0000 (Thu, 07 Nov 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
from __future__ import absolute_import
diff --git a/lib/postgres_cli.py b/lib/postgres_cli.py
index 7ccd5b617..e84416d98 100755
--- a/lib/postgres_cli.py
+++ b/lib/postgres_cli.py
@@ -4,14 +4,14 @@
# Author: Hari Sekhon
# Date: 2020-03-12 17:39:57 +0000 (Thu, 12 Mar 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
from __future__ import absolute_import
diff --git a/opentsdb_import_metric_distribution.py b/opentsdb_import_metric_distribution.py
index 1331b68d2..3b3d3df09 100755
--- a/opentsdb_import_metric_distribution.py
+++ b/opentsdb_import_metric_distribution.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-10-10 11:47:12 +0100 (Mon, 10 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/opentsdb_list_metrics.sh b/opentsdb_list_metrics.sh
index 6365ffc35..a119bd9fa 100755
--- a/opentsdb_list_metrics.sh
+++ b/opentsdb_list_metrics.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/opentsdb_list_metrics_hbase.sh b/opentsdb_list_metrics_hbase.sh
index 945b108ff..86142e4c0 100755
--- a/opentsdb_list_metrics_hbase.sh
+++ b/opentsdb_list_metrics_hbase.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2018-07-13 22:36:14 +0100 (Fri, 13 Jul 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -91,6 +91,7 @@ print_metrics(){
print_metrics_by_age(){
# hackish but convenient - forking to the date command thousands or hundreds of thousands of times can take hours, python takes 10 secs even for 250,000+ metrics
tmp_python_script=$(mktemp)
+ # shellcheck disable=SC1117
cat > "$tmp_python_script" <=2.0.0,<3.0.0
# for Elasticsearch 1.x
#elasticsearch>=1.0.0,<2.0.0
+
# fails on requiring newer version of setuptools
#Flask==0.10.1
GitPython==2.1.15
+
# this GCP API is surprisingly awful, not using
#google-api-python-client==1.11.0
-happybase==1.0.0
+
+# XXX: install broken on new M1 Mac Python 3.9
+#happybase==1.0.0
+
humanize==0.5.1
-impyla==0.16.0
+impyla==0.19.0
jinja2==2.11.3
#kazoo==2.2.1
ldif3==3.2.2
#MarkupSafe==0.23
#Markdown==2.6.8
+
+matplotlib==3.7.5
+
# Python 3.5+
#numpy==1.17.2
-numpy==1.16.5
+# XXX: install broken on new M1 Mac Python 3.9
+#numpy==1.16.5
+
+# for plot_uk_marriage_rates.py
+openpyxl==3.1.5
+pandas==2.0.3
+
# requires pg_config to build from source
#psycopg2==2.8.4
-psycopg2-binary==2.8.4
+# XXX: install broken on new M1 Mac Python 3.9
+#psycopg2-binary==2.8.4
+
python-cson==1.0.9
psutil==5.7.0
+
# parquet support in pyarrow is weaker, gone back to using parquet-tools
#pyarrow==0.6.0
#PyHive==0.6.1
+
+# doesn't work with non-trivial code
#PyInstaller==3.3.1
-python-ldap==3.2.0
+
+# gcc compile error on Alpine, don't think this is used either
+#python-ldap==3.2.0
+
#python-jenkins==0.4.13
# pulled in automatically by snakebite[kerberos]
#python-krbV==1.0.90
# needed by avro
-python-snappy==0.5
-sasl==0.2.1
+
+# XXX: install broken on new M1 Mac Python 3.9
+#python-snappy==0.5
+
+# XXX: install broken on new M1 Mac Python 3.9
+#sasl==0.2.1
+
sh==1.12.14
selenium==3.141.0
+
# pulls in python-KrbV as a dependency which doesn't build on Mac any more
# relies on python-krbV is unmaintained and unported to Python 3
# - moved to Makefile as best effort
#snakebite[kerberos]==2.11.0
#snakebite==2.11.0
-thrift-sasl==0.2.1
-thrift==0.9.3
-thriftpy==0.3.9
+
+# XXX: install broken on new M1 Mac Python 3.9
+#thrift-sasl==0.2.1
+#thrift==0.9.3
+#thriftpy==0.3.9
+
toml==0.10.0
xmltodict==0.10.2
yamllint==1.15.0
+
#pyyaml>=5.4 # not directly required, pinned by Snyk to avoid a vulnerability. update: this breaks Python 3.5 build where this requirement is not found
diff --git a/selenium_hub_browser_test.py b/selenium_hub_browser_test.py
index 8b0242b83..c3d117e53 100755
--- a/selenium_hub_browser_test.py
+++ b/selenium_hub_browser_test.py
@@ -1,11 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2021-05-12 09:55:01 +0100 (Wed, 12 May 2021)
#
-# https://github.com/HariSekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
diff --git a/serf_event_handler.py b/serf_event_handler.py
index 38a0df1b4..c0d5cacbb 100755
--- a/serf_event_handler.py
+++ b/serf_event_handler.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-16 15:44:16 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -61,7 +61,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.2.2'
+__version__ = '0.2.3'
class SerfEventHandler(CLI):
@@ -95,7 +95,7 @@ def add_options(self):
def enable_commands(self):
if self.event in ['query', 'event']:
- cmd = None
+ cmd = ''
if self.event == 'query':
cmd = self.query_name
elif self.event == 'user':
diff --git a/setup/apk-packages-dev.txt b/setup/apk-packages-dev.txt
index efef4f8a5..a8fe03ea8 100644
--- a/setup/apk-packages-dev.txt
+++ b/setup/apk-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/apk-packages-pip.txt b/setup/apk-packages-pip.txt
index 01130d497..1b9ee9d77 100644
--- a/setup/apk-packages-pip.txt
+++ b/setup/apk-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/apk-packages.txt b/setup/apk-packages.txt
index bccca7733..267f430e5 100644
--- a/setup/apk-packages.txt
+++ b/setup/apk-packages.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh
index 5383ae1f0..d7bedf92a 100755
--- a/setup/bootstrap.sh
+++ b/setup/bootstrap.sh
@@ -4,30 +4,30 @@
# Author: Hari Sekhon
# Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Alpine / Wget:
#
-# wget https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh && sh bootstrap.sh
+# wget -O- https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
#
# Curl:
#
-# curl https://raw.githubusercontent.com/HariSekhon/devops-python-tools/master/setup/bootstrap.sh | sh
+# curl https://raw.githubusercontent.com/HariSekhon/DevOps-Python-tools/master/setup/bootstrap.sh | sh
set -eu
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
-repo="https://github.com/HariSekhon/devops-python-tools"
+repo="https://github.com/HariSekhon/DevOps-Python-tools"
-directory="python-tools"
+directory="pytools"
sudo=""
[ "$(whoami)" = "root" ] || sudo=sudo
@@ -40,16 +40,24 @@ if [ "$(uname -s)" = Darwin ]; then
elif [ "$(uname -s)" = Linux ]; then
echo "Bootstrapping on Linux: $repo"
if type apk >/dev/null 2>&1; then
- $sudo apk --no-cache add bash git make curl
+ $sudo apk --no-cache add bash git make curl wget
elif type apt-get >/dev/null 2>&1; then
- opts=""
+ if [ -n "${CI:-}" ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ fi
+ opts="-o DPkg::Lock::Timeout=1200"
if [ -z "${PS1:-}" ]; then
- opts="-qq"
+ opts="$opts -qq"
fi
- $sudo apt-get update $opts
- $sudo apt-get install $opts -y git make curl
+ $sudo apt-get update $opts
+ $sudo apt-get install $opts -y git make curl wget --no-install-recommends
elif type yum >/dev/null 2>&1; then
- $sudo yum install -y git make curl
+ if grep -qi 'NAME=.*CentOS' /etc/*release; then
+ echo "CentOS EOL detected, replacing yum base URL to vault to re-enable package installs"
+ $sudo sed -i 's/^[[:space:]]*mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Linux-*
+ $sudo sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-Linux-*
+ fi
+ $sudo yum install -y git make curl wget
else
echo "Package Manager not found on Linux, cannot bootstrap"
exit 1
@@ -73,4 +81,6 @@ else
fi
fi
-make
+if [ -z "${NO_MAKE:-}" ]; then
+ make
+fi
diff --git a/setup/brew-packages.txt b/setup/brew-packages.txt
index 03a0a78a5..401e24b97 100644
--- a/setup/brew-packages.txt
+++ b/setup/brew-packages.txt
@@ -3,11 +3,11 @@
# Author: Hari Sekhon
# Date: 2018-05-22 11:15:16 +0100 (Tue, 22 May 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh
index 691e04a2f..b01a77d04 100755
--- a/setup/ci_bootstrap.sh
+++ b/setup/ci_bootstrap.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding
@@ -52,7 +52,7 @@ retry(){
if [ "$(uname -s)" = Darwin ]; then
echo "Bootstrapping Mac"
# removing adjacent dependency to be able to curl from github to avoid submodule circular dependency (git / submodule / install git & make)
- #retry "$srcdir/install_homebrew.sh"
+ #retry "$srcdir/../install/install_homebrew.sh"
if command -v brew 2>&1; then
# fix for CI runners on Mac with shallow homebrew clone - which is failing all the BuildKite builds
for git_root in /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask; do
@@ -72,8 +72,9 @@ elif [ "$(uname -s)" = Linux ]; then
retry $sudo apk update
retry $sudo apk add --no-progress bash git make
elif type apt-get >/dev/null 2>&1; then
- retry $sudo apt-get update -q
- retry $sudo apt-get install -qy git make
+ opts="-q -o DPkg::Lock::Timeout=1200"
+ retry $sudo apt-get update $opts
+ retry $sudo apt-get install $opts -y git make
elif type yum >/dev/null 2>&1; then
#retry $sudo yum makecache
retry $sudo yum install -qy git make
diff --git a/setup/ci_git_set_dir_safe.sh b/setup/ci_git_set_dir_safe.sh
new file mode 100755
index 000000000..66a53903a
--- /dev/null
+++ b/setup/ci_git_set_dir_safe.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# vim:ts=4:sts=4:sw=4:et
+#
+# Author: Hari Sekhon
+# Date: 2022-08-03 20:07:09 +0100 (Wed, 03 Aug 2022)
+#
+# https://github.com/HariSekhon/DevOps-Python-tools
+#
+# License: see accompanying Hari Sekhon LICENSE file
+#
+# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
+#
+# https://www.linkedin.com/in/HariSekhon
+#
+
+# Necessary for some CI/CD systems like Azure DevOps Pipelines which have incorrect ownership on the git checkout dir triggering this error:
+#
+# fatal: detected dubious ownership in repository at '/code/sql'
+
+# standalone script without lib dependency so it can be called directly from bootstrapped CI before submodules, since that is the exact problem that needs to be solved to allow CI/CD systems with incorrect ownership of the checkout directory to be able to checkout the necessary git submodules
+
+set -euo pipefail
+[ -n "${DEBUG:-}" ] && set -x
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+dir="${1:-$srcdir/..}"
+
+cd "$dir"
+
+echo "Setting directory as safe: $PWD"
+git config --global --add safe.directory "$PWD"
+
+while read -r submodule_dir; do
+ dir="$PWD/$submodule_dir"
+ echo "Setting directory as safe: $dir"
+ git config --global --add safe.directory "$dir"
+done < <(git submodule | awk '{print $2}')
+
+echo "Done"
diff --git a/setup/deb-packages-dev.txt b/setup/deb-packages-dev.txt
index ae1a456ce..015439134 100644
--- a/setup/deb-packages-dev.txt
+++ b/setup/deb-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/deb-packages-optional.txt b/setup/deb-packages-optional.txt
index 038ae8802..cd797f747 100644
--- a/setup/deb-packages-optional.txt
+++ b/setup/deb-packages-optional.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/deb-packages-pip.txt b/setup/deb-packages-pip.txt
index 8c6215957..da4eea8c8 100644
--- a/setup/deb-packages-pip.txt
+++ b/setup/deb-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/deb-packages.txt b/setup/deb-packages.txt
index d7dc245df..4a028e4b1 100644
--- a/setup/deb-packages.txt
+++ b/setup/deb-packages.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/gocd_config_repo.json b/setup/gocd_config_repo.json
index a3a8beb73..ce63bf63d 100644
--- a/setup/gocd_config_repo.json
+++ b/setup/gocd_config_repo.json
@@ -1,10 +1,10 @@
{
- "id": "devops-python-tools",
+ "id": "DevOps-Python-tools",
"plugin_id": "yaml.config.plugin",
"material": {
"type": "git",
"attributes": {
- "url": "https://github.com/harisekhon/devops-python-tools",
+ "url": "https://github.com/HariSekhon/DevOps-Python-tools",
"branch": "master",
"auto_update": true
}
@@ -12,7 +12,7 @@
"configuration": [
{
"key": "file_pattern",
- "value": "*.gocd.y*ml"
+ "value": "cicd/*.gocd.y*ml"
}
],
"rules": [
diff --git a/setup/jenkins-job.xml b/setup/jenkins-job.xml
index 4469d2afe..2c9da2b6b 100644
--- a/setup/jenkins-job.xml
+++ b/setup/jenkins-job.xml
@@ -17,7 +17,7 @@
- https://github.com/harisekhon/devops-python-tools/
+ https://github.com/HariSekhon/DevOps-Python-tools/
@@ -37,7 +37,7 @@
2
- https://github.com/harisekhon/devops-python-tools
+ https://github.com/HariSekhon/DevOps-Python-tools
diff --git a/setup/rpm-packages-dev.txt b/setup/rpm-packages-dev.txt
index 6dd8fa477..f36770476 100644
--- a/setup/rpm-packages-dev.txt
+++ b/setup/rpm-packages-dev.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/rpm-packages-optional.txt b/setup/rpm-packages-optional.txt
index cb889fb12..b1bb97d9e 100644
--- a/setup/rpm-packages-optional.txt
+++ b/setup/rpm-packages-optional.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/rpm-packages-pip.txt b/setup/rpm-packages-pip.txt
index 1a879d376..2a33f4bc5 100644
--- a/setup/rpm-packages-pip.txt
+++ b/setup/rpm-packages-pip.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/setup/rpm-packages.txt b/setup/rpm-packages.txt
index cf5f0222b..7f6263457 100644
--- a/setup/rpm-packages.txt
+++ b/setup/rpm-packages.txt
@@ -2,11 +2,11 @@
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying LICENSE file
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# ============================================================================ #
diff --git a/shippable.yml b/shippable.yml
deleted file mode 100644
index 71c6c3c76..000000000
--- a/shippable.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-23 23:20:54 +0000 (Sun, 23 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# License: see accompanying Hari Sekhon LICENSE file
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-# http://docs.shippable.com/platform/workflow/config/
-
-# http://docs.shippable.com/ci/advancedOptions/environmentVariables/
-
-language: none
-
-branches:
- only:
- - master
-
-build:
- ci:
- # workaround to broken repos
- # W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: https://downloads.apache.org/cassandra/debian 311x InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E91335D77E3E87CB
- # W: GPG error: http://dl.yarnpkg.com/debian stable Release: The following signatures were invalid: KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1580619281 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983 KEYEXPIRED 1507181400 KEYEXPIRED 1546376218 KEYEXPIRED 1546372003 KEYEXPIRED 1580619281 KEYEXPIRED 1580607983
- # E: The repository 'http://dl.yarnpkg.com/debian stable Release' is no longer signed.
- # devops-python-tools/Makefile.in:272: recipe for target 'apt-packages' failed
- - rm -fv /etc/apt/sources.list.d/cassandra.sources.list*
- - rm -fv /etc/apt/sources.list.d/yarn.list*
- # Basho repo is giving a '402 payment required' error
- # https://github.com/Shippable/support/issues/5172
- - rm -fv /etc/apt/sources.list.d/basho_riak.list
- #- shippable_retry make
- - setup/ci_bootstrap.sh
- - make init
- - make ci
- - make test
-
-integrations:
- notifications:
- - integrationName: email
- type: email
- on_success: never
- on_failure: never
- on_pull_request: never
diff --git a/sonar-project.properties b/sonar-project.properties
index 75e1bf83c..319cc6897 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -1,32 +1,42 @@
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
-# Date: 2016-07-19 18:18:27 +0100 (Tue, 19 Jul 2016)
+# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
-sonar.projectName=DevOps Python Tools
-sonar.projectKey=pytools
+# ============================================================================ #
+# S o n a r Q u b e
+# ============================================================================ #
+
+sonar.host.url=https://sonarcloud.io
+
+# Required metadata
+sonar.organization=harisekhon
+sonar.projectName=DevOps-Python-tools
+sonar.projectKey=HariSekhon_DevOps-Python-tools
sonar.projectVersion=1.0
-sonar.projectDescription=Python / Jython Tools
+sonar.projectDescription=DevOps-Python-tools
-sonar.links.homepage=https://github.com/harisekhon/devops-python-tools
-sonar.links.scm=https://github.com/harisekhon/devops-python-tools
-sonar.links.issue=https://github.com/harisekhon/devops-python-tools/issues
-sonar.links.ci=https://travis-ci.org/HariSekhon/devops-python-tools
+sonar.links.homepage=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.scm=https://github.com/HariSekhon/DevOps-Python-tools
+sonar.links.issue=https://github.com/HariSekhon/DevOps-Python-tools/issues
+sonar.links.ci=https://github.com/HariSekhon/DevOps-Python-tools/actions
+# directories to scan (defaults to sonar-project.properties dir otherwise)
sonar.sources=.
#sonar.language=py
sonar.sourceEncoding=UTF-8
-sonar.exclusions=**/tests/spark*/**/*
+#sonar.exclusions=**/tests/**
+sonar.exclusions=**/zookeeper-*/**/*
diff --git a/spark_avro_to_parquet.py b/spark_avro_to_parquet.py
index 10fd55e88..8528af1a6 100755
--- a/spark_avro_to_parquet.py
+++ b/spark_avro_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_csv_to_avro.py b/spark_csv_to_avro.py
index efcbc1279..d41717b4f 100755
--- a/spark_csv_to_avro.py
+++ b/spark_csv_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -57,6 +57,7 @@
'com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
@@ -166,7 +167,6 @@ def create_struct(arg):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
@@ -198,5 +198,6 @@ def create_struct(arg):
# the databricks avro driver
df.write.format('com.databricks.spark.avro').save(avro_dir)
+
if __name__ == '__main__':
SparkCSVToAvro().main()
diff --git a/spark_csv_to_parquet.py b/spark_csv_to_parquet.py
index 3bbb9c18a..e70f014f6 100755
--- a/spark_csv_to_parquet.py
+++ b/spark_csv_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -49,6 +49,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-csv_2.11:1.5.0 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
@@ -157,9 +158,7 @@ def create_struct(arg):
if not isVersionLax(spark_version):
die("Spark version couldn't be determined. " + support_msg('pytools'))
- # pylint: disable=invalid-name
-
- df = None
+ # pylint: disable=invalid-name
if isMinVersion(spark_version, 1.4):
if has_header and not schema:
log.info('inferring schema from CSV headers')
diff --git a/spark_json_to_avro.py b/spark_json_to_avro.py
index 40c52ac96..db9d069ed 100755
--- a/spark_json_to_avro.py
+++ b/spark_json_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
@@ -109,7 +110,6 @@ def run(self):
die("Spark version couldn't be determined. " + support_msg('pytools'))
# pylint: disable=invalid-name
- df = None
if isMinVersion(spark_version, 1.4):
df = sqlContext.read.json(json_file)
else:
diff --git a/spark_json_to_parquet.py b/spark_json_to_parquet.py
index 41ab27e0d..6976ce153 100755
--- a/spark_json_to_parquet.py
+++ b/spark_json_to_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -43,6 +43,7 @@
print("Alternatively perhaps you tried to copy this program out without it's adjacent libraries?", file=sys.stderr)
sys.exit(4)
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/spark_parquet_to_avro.py b/spark_parquet_to_avro.py
index d6e49f060..030816963 100755
--- a/spark_parquet_to_avro.py
+++ b/spark_parquet_to_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-11-03 21:38:52 +0000 (Tue, 03 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -52,6 +52,7 @@
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-avro_2.10:2.0.1 %s' \
% os.getenv('PYSPARK_SUBMIT_ARGS', '')
pyspark_path()
+# pylint: disable=import-outside-toplevel
from pyspark import SparkContext # pylint: disable=wrong-import-position,import-error
from pyspark import SparkConf # pylint: disable=wrong-import-position,import-error
from pyspark.sql import SQLContext # pylint: disable=wrong-import-position,import-error
diff --git a/sql b/sql
index 53c1d53b0..30d80bbd3 160000
--- a/sql
+++ b/sql
@@ -1 +1 @@
-Subproject commit 53c1d53b0ff15f8d9d0b7191b522a1b0b1d0bf20
+Subproject commit 30d80bbd33f0e6c26c230f26575cb671c41c3e7d
diff --git a/strip_ansi_escape_codes.py b/strip_ansi_escape_codes.py
index 6fc517dc6..24738194a 100755
--- a/strip_ansi_escape_codes.py
+++ b/strip_ansi_escape_codes.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2018-09-09 23:06:06 +0100 (Sun, 09 Sep 2018)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/.teamcity.vcs.json b/teamcity/.teamcity.vcs.json
similarity index 100%
rename from .teamcity.vcs.json
rename to teamcity/.teamcity.vcs.json
diff --git a/.teamcity.vcs.oauth.json b/teamcity/.teamcity.vcs.oauth.json
similarity index 100%
rename from .teamcity.vcs.oauth.json
rename to teamcity/.teamcity.vcs.oauth.json
diff --git a/.teamcity.vcs.ssh.json b/teamcity/.teamcity.vcs.ssh.json
similarity index 100%
rename from .teamcity.vcs.ssh.json
rename to teamcity/.teamcity.vcs.ssh.json
diff --git a/templates b/templates
index 9021150df..f59532dd6 160000
--- a/templates
+++ b/templates
@@ -1 +1 @@
-Subproject commit 9021150df83050e91ff5c72894de46305d0d54ce
+Subproject commit f59532dd67a583be3522814a15024a561b12c5ec
diff --git a/tests/all.sh b/tests/all.sh
index b1e1957dc..dbac7b8fb 100755
--- a/tests/all.sh
+++ b/tests/all.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -29,8 +29,9 @@ section "Running PyTools ALL"
# runs against . by default
cd "$srcdir/..";
+# shellcheck disable=SC1090
# has to be included so that isExcluded function is inherited
-. bash-tools/check_all.sh
+. "$srcdir/../bash-tools/checks/check_all.sh"
#tests/test_yamllint.sh
@@ -40,4 +41,4 @@ exit 0
# pyspark not found
tests/help.sh
-bash-tools/run_tests.sh
+bash-tools/checks/run_tests.sh
diff --git a/tests/check.sh b/tests/check.sh
index 07e1bd9c2..b3dba3c12 100755
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:47:43 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/compile.sh b/tests/compile.sh
index e235cfd70..2890b07a3 100755
--- a/tests/compile.sh
+++ b/tests/compile.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
@@ -19,17 +19,18 @@ srcdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$srcdir/..";
-. ./tests/utils.sh
+# shellcheck disable=SC1090
+. "$srcdir/utils.sh"
hr
echo "Compiling all Python files"
hr
echo
-for x in $(find . -iname '*.py' -o -iname '*.jy'); do
- isExcluded "$x" && continue
- echo "compiling $x"
- python -m py_compile $x
-done
+while read -r filename; do
+ isExcluded "$filename" && continue
+ echo "compiling $filename"
+ python -m py_compile "$filename"
+done < <(find . -iname '*.py' -o -iname '*.jy')
echo
echo
diff --git a/tests/data/add_ou.ldif b/tests/data/add_ou.ldif
index c175b4bba..c9bc193b2 100644
--- a/tests/data/add_ou.ldif
+++ b/tests/data/add_ou.ldif
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-08-03 17:44:12 +0200 (Thu, 03 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
dn: OU=Hadoop-Cluster-1,OU=Hadoop,DC=harisekhon,DC=co,DC=uk
diff --git a/tests/data/ldap_download.sh b/tests/data/ldap_download.sh
index 1eb3dcd6f..8c7f4a158 100755
--- a/tests/data/ldap_download.sh
+++ b/tests/data/ldap_download.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/data/ldap_upload.sh b/tests/data/ldap_upload.sh
index 0ff49ca02..60cdeb5ab 100755
--- a/tests/data/ldap_upload.sh
+++ b/tests/data/ldap_upload.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-04-11 18:57:53 +0100 (Thu, 11 Apr 2019)
#
-# https://github.com/harisekhon/pytools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/data/uk_marriage_rates_2022.xslx b/tests/data/uk_marriage_rates_2022.xslx
new file mode 100644
index 000000000..54efc3321
Binary files /dev/null and b/tests/data/uk_marriage_rates_2022.xslx differ
diff --git a/tests/docker/apache-drill-docker-compose.yml b/tests/docker/apache-drill-docker-compose.yml
index 50c1de358..d5ede5b8b 100644
--- a/tests/docker/apache-drill-docker-compose.yml
+++ b/tests/docker/apache-drill-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 16:18:36 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/common.yml b/tests/docker/common.yml
index 4c4236d1c..0b2a2d133 100644
--- a/tests/docker/common.yml
+++ b/tests/docker/common.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 15:16:43 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-docker-compose.yml b/tests/docker/elasticsearch-docker-compose.yml
index bd6d9c190..4838ee391 100644
--- a/tests/docker/elasticsearch-docker-compose.yml
+++ b/tests/docker/elasticsearch-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/elasticsearch-elastic.co-docker-compose.yml b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
index 992448ba6..6b7095ab6 100644
--- a/tests/docker/elasticsearch-elastic.co-docker-compose.yml
+++ b/tests/docker/elasticsearch-elastic.co-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 18:41:13 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/hadoop-docker-compose.yml b/tests/docker/hadoop-docker-compose.yml
index bef1fe74b..157d45c63 100644
--- a/tests/docker/hadoop-docker-compose.yml
+++ b/tests/docker/hadoop-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/hbase-docker-compose.yml b/tests/docker/hbase-docker-compose.yml
index 04029f2bb..8ab70732b 100644
--- a/tests/docker/hbase-docker-compose.yml
+++ b/tests/docker/hbase-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-09 22:13:19 +0000 (Fri, 09 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/presto-dev-docker-compose.yml b/tests/docker/presto-dev-docker-compose.yml
index 835b126f1..aaf4872f4 100644
--- a/tests/docker/presto-dev-docker-compose.yml
+++ b/tests/docker/presto-dev-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/presto-docker-compose.yml b/tests/docker/presto-docker-compose.yml
index bb09ee3ed..2b9cc99ed 100644
--- a/tests/docker/presto-docker-compose.yml
+++ b/tests/docker/presto-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-13 14:47:23 +0200 (Wed, 13 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.1'
diff --git a/tests/docker/registry-docker-compose.yml b/tests/docker/registry-docker-compose.yml
index 7a66c2e3f..5fae25ab1 100644
--- a/tests/docker/registry-docker-compose.yml
+++ b/tests/docker/registry-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2017-09-12 17:27:50 +0200 (Tue, 12 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/docker/solrcloud-docker-compose.yml b/tests/docker/solrcloud-docker-compose.yml
index 7799417a7..9cf4885fa 100644
--- a/tests/docker/solrcloud-docker-compose.yml
+++ b/tests/docker/solrcloud-docker-compose.yml
@@ -3,13 +3,13 @@
# Author: Hari Sekhon
# Date: 2016-12-13 14:01:04 +0000 (Tue, 13 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
version: '2.2'
diff --git a/tests/excluded.sh b/tests/excluded.sh
index a85efb228..7535d3be6 100755
--- a/tests/excluded.sh
+++ b/tests/excluded.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# intended only to be sourced by utils.sh
@@ -22,6 +22,7 @@ set -eu
isExcluded(){
local prog="$1"
+ # shellcheck disable=SC2049
[[ "$prog" =~ ^\* ]] && return 0
[[ "$prog" =~ spark_.*\.py ]] && return 0
[[ "$prog" =~ \.jy ]] && return 0
diff --git a/tests/help.sh b/tests/help.sh
index 8666b576c..09b48dfe0 100755
--- a/tests/help.sh
+++ b/tests/help.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/python3.sh b/tests/python3.sh
index 9cd102ac1..b231f8df4 100755
--- a/tests/python3.sh
+++ b/tests/python3.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/syntax.sh b/tests/syntax.sh
index ea6b33a6a..5299ce323 100755
--- a/tests/syntax.sh
+++ b/tests/syntax.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_anonymize.sh b/tests/test_anonymize.sh
index 713b844ea..9aebe794f 100755
--- a/tests/test_anonymize.sh
+++ b/tests/test_anonymize.sh
@@ -4,14 +4,14 @@
# Author: Hari Sekhon
# Date: 2015-07-28 18:47:41 +0100 (Tue, 28 Jul 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -485,7 +485,9 @@ src[140]='ssh -i myKey -N -L 8888:ec2-1-2-3-4.eu-west-1.compute.amazonaws.com:88
#dest[140]='ssh -i myKey -N -L 8888::8888 @'
dest[140]='ssh -i myKey -N -L 8888::8888 @'
+# shellcheck disable=SC1117
src[141]="Failed to open HDFS file hdfs://nameservice1/user/hive/warehouse/area_2/my_database_2.db/my_table_2/part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory"
+# shellcheck disable=SC1117
dest[141]="Failed to open HDFS file hdfs:///user//warehouse/.db//part-r-00030-6a789012-3bc4-56d7-e890-123fa456b7c8.snappy.parquet\nError(2): No such file or directory"
src[142]="ERROR: AnalysisException: Failed to load metadata for table: 'myCustomerTable2'"
diff --git a/tests/test_apache-drill.sh b/tests/test_apache-drill.sh
index fd336dc2c..af4b663c6 100755
--- a/tests/test_apache-drill.sh
+++ b/tests/test_apache-drill.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-26 23:36:03 +0000 (Tue, 26 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_center.sh b/tests/test_center.sh
index 0dfd434f8..adb080442 100755
--- a/tests/test_center.sh
+++ b/tests/test_center.sh
@@ -10,7 +10,7 @@
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
# Quick tests, need to replace with testcmd.exp
diff --git a/tests/test_docker.sh b/tests/test_docker.sh
index 8d00d3be3..37dbb90d5 100755
--- a/tests/test_docker.sh
+++ b/tests/test_docker.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-12-08 14:38:37 +0000 (Thu, 08 Dec 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_docker_registry_show_tags.sh b/tests/test_docker_registry_show_tags.sh
index bc0612399..e327a3135 100755
--- a/tests/test_docker_registry_show_tags.sh
+++ b/tests/test_docker_registry_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-09-12 17:49:29 +0200 (Tue, 12 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_dockerfiles_check_git_branches.sh b/tests/test_dockerfiles_check_git_branches.sh
index 2529a2c9c..1081c3b98 100755
--- a/tests/test_dockerfiles_check_git_branches.sh
+++ b/tests/test_dockerfiles_check_git_branches.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -29,7 +29,7 @@ section "Testing Dockerfiles Check Git branches"
if type -P git &>dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_dockerfiles_check_git_tags.sh b/tests/test_dockerfiles_check_git_tags.sh
index ada872f3c..a6e0ba12c 100755
--- a/tests/test_dockerfiles_check_git_tags.sh
+++ b/tests/test_dockerfiles_check_git_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -29,7 +29,7 @@ section "Testing Dockerfiles Check Git Tags"
if type -P git &>dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_dockerhub_search.sh b/tests/test_dockerhub_search.sh
index 450ee7343..2a4f9c48e 100755
--- a/tests/test_dockerhub_search.sh
+++ b/tests/test_dockerhub_search.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -36,7 +36,7 @@ check './dockerhub_search.py hadoop-dev | grep harisekhon/hadoop-dev' "DockerHub
# causes IOError: [Errno 32] Broken pipe
#unset PYTHONUNBUFFERED
# shellcheck disable=SC2016
-check '[ $(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep "^harisekhon/[A-Za-z0-9_-]*$" | wc -l) = 40 ]' "DockerHub Search quiet mode for shell scripting"
+check '[ "$(./dockerhub_search.py -q harisekhon | head -n 40 | tee /dev/stderr | grep -c "^harisekhon/[A-Za-z0-9_-]*$")" = 40 ]' "DockerHub Search quiet mode for shell scripting"
echo
echo
diff --git a/tests/test_dockerhub_show_tags.sh b/tests/test_dockerhub_show_tags.sh
index 8d052f174..dba64e816 100755
--- a/tests/test_dockerhub_show_tags.sh
+++ b/tests/test_dockerhub_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_elasticsearch.sh b/tests/test_elasticsearch.sh
index 5abd6e53f..17cd5260d 100755
--- a/tests/test_elasticsearch.sh
+++ b/tests/test_elasticsearch.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_find_active_server.sh b/tests/test_find_active_server.sh
index 03e8dee75..615f806d4 100755
--- a/tests/test_find_active_server.sh
+++ b/tests/test_find_active_server.sh
@@ -10,7 +10,7 @@
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_find_duplicate_files.sh b/tests/test_find_duplicate_files.sh
index 9a4e951c9..ade1eeb00 100755
--- a/tests/test_find_duplicate_files.sh
+++ b/tests/test_find_duplicate_files.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-14 20:42:01 +0100 (Sun, 14 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -110,10 +110,10 @@ for testdir in "$testdir1" "$testdir2"; do
echo "now check the file basename matches on 'est'":
run_fail 4 ./find_duplicate_files.py --regex 'est' "$testdir" "$testdir1" --quiet
- echo "now check the file basename matches with specified capture subset '(est)\d'":
+ echo "now check the file basename matches with specified capture subset '(est)\\d'":
run_fail 4 ./find_duplicate_files.py --regex '(est)\d' "$testdir" "$testdir1"
- echo "now check the file basename doesn't match when the capture includes differing numbers 'est\d'":
+ echo "now check the file basename doesn't match when the capture includes differing numbers 'est\\d'":
run ./find_duplicate_files.py --regex 'est\d' "$testdir" "$testdir1"
rm "$testdir/test2.txt"
diff --git a/tests/test_find_python_library_path.sh b/tests/test_find_python_library_path.sh
index 6a13f5779..7cdc1f43f 100755
--- a/tests/test_find_python_library_path.sh
+++ b/tests/test_find_python_library_path.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-09-27
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_getent.sh b/tests/test_getent.sh
index 26a7aa2b5..6afc48810 100755
--- a/tests/test_getent.sh
+++ b/tests/test_getent.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-11-20 15:35:37 +0000 (Sun, 20 Nov 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_git_check_branches_upstream.sh b/tests/test_git_check_branches_upstream.sh
index 35a818eb8..8459249bc 100755
--- a/tests/test_git_check_branches_upstream.sh
+++ b/tests/test_git_check_branches_upstream.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -29,7 +29,7 @@ section "Testing Git check branches upstream"
if type -P git &>/dev/null; then
if ! [ -d Dockerfiles ]; then
- git clone https://github.com/harisekhon/Dockerfiles
+ git clone https://github.com/HariSekhon/Dockerfiles
else
pushd Dockerfiles
git pull
diff --git a/tests/test_hadoop.sh b/tests/test_hadoop.sh
index c2ebedfe9..479789e1f 100755
--- a/tests/test_hadoop.sh
+++ b/tests/test_hadoop.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_hbase.sh b/tests/test_hbase.sh
index 28186c242..4727347a6 100755
--- a/tests/test_hbase.sh
+++ b/tests/test_hbase.sh
@@ -4,14 +4,14 @@
# Author: Hari Sekhon
# Date: 2016-05-06 12:12:15 +0100 (Fri, 06 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_headtail.sh b/tests/test_headtail.sh
index 8f7ceb05b..2dcfb599c 100755
--- a/tests/test_headtail.sh
+++ b/tests/test_headtail.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_json.sh b/tests/test_json.sh
index 9e86b0984..34ab409d9 100755
--- a/tests/test_json.sh
+++ b/tests/test_json.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_json_docs_to_bulk_multiline.sh b/tests/test_json_docs_to_bulk_multiline.sh
index 2e81b392a..53d08f35f 100755
--- a/tests/test_json_docs_to_bulk_multiline.sh
+++ b/tests/test_json_docs_to_bulk_multiline.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-07-30 14:30:00 +0200 (Sun, 30 Jul 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_json_to_xml.sh b/tests/test_json_to_xml.sh
index 0b20f8609..8b66faa03 100755
--- a/tests/test_json_to_xml.sh
+++ b/tests/test_json_to_xml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_json_to_yaml.sh b/tests/test_json_to_yaml.sh
index fc2b58868..4aa156043 100755
--- a/tests/test_json_to_yaml.sh
+++ b/tests/test_json_to_yaml.sh
@@ -4,31 +4,29 @@
# Author: Hari Sekhon
# Date: 2019-12-19 18:04:15 +0000 (Thu, 19 Dec 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "$0")" && pwd)"
-cd "$srcdir";
+cd "$srcdir/..";
-# shellcheck disable=SC1091
-. utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
-# shellcheck disable=SC1091
-. ../bash-tools/lib/utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
section "JSON => YAML"
-cd ..
-
tmpfile="$(mktemp json_to_yaml_test.XXXXX.yml)"
#echo "tmpfile is $tmpfile"
diff --git a/tests/test_opentsdb.sh b/tests/test_opentsdb.sh
index ac98e609a..bbc59a11f 100755
--- a/tests/test_opentsdb.sh
+++ b/tests/test_opentsdb.sh
@@ -4,14 +4,14 @@
# Author: Hari Sekhon
# Date: 2016-10-10 11:54:19 +0100 (Mon, 10 Oct 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_presto.sh b/tests/test_presto.sh
index ba9add032..c031d5697 100755
--- a/tests/test_presto.sh
+++ b/tests/test_presto.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-09-22 17:01:38 +0200 (Fri, 22 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_quay_show_tags.sh b/tests/test_quay_show_tags.sh
index 6765bc9e9..96f451c23 100755
--- a/tests/test_quay_show_tags.sh
+++ b/tests/test_quay_show_tags.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_serf_event_handler.sh b/tests/test_serf_event_handler.sh
index eee228571..d16c8e2f7 100755
--- a/tests/test_serf_event_handler.sh
+++ b/tests/test_serf_event_handler.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-16 16:35:51 +0000 (Sat, 16 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_solrcloud.sh b/tests/test_solrcloud.sh
index 31df34f43..2f0f69196 100755
--- a/tests/test_solrcloud.sh
+++ b/tests/test_solrcloud.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-01-22 21:13:49 +0000 (Fri, 22 Jan 2016)
#
-# https://github.com/harisekhon/nagios-plugins
+# https://github.com/HariSekhon/Nagios-Plugins
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_spark_csv_to_avro.sh b/tests/test_spark_csv_to_avro.sh
index dc43c31d8..5765ba160 100755
--- a/tests/test_spark_csv_to_avro.sh
+++ b/tests/test_spark_csv_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_spark_csv_to_parquet.sh b/tests/test_spark_csv_to_parquet.sh
index 7f95341e1..aa2b8d2dd 100755
--- a/tests/test_spark_csv_to_parquet.sh
+++ b/tests/test_spark_csv_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_spark_json_to_avro.sh b/tests/test_spark_json_to_avro.sh
index d6506a50c..bb281026f 100755
--- a/tests/test_spark_json_to_avro.sh
+++ b/tests/test_spark_json_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_spark_json_to_parquet.sh b/tests/test_spark_json_to_parquet.sh
index 9a4a8bc37..dc3e2fef4 100755
--- a/tests/test_spark_json_to_parquet.sh
+++ b/tests/test_spark_json_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_spark_z_avro_to_parquet.sh b/tests/test_spark_z_avro_to_parquet.sh
index 02fc1ea26..b6c3ff7cb 100755
--- a/tests/test_spark_z_avro_to_parquet.sh
+++ b/tests/test_spark_z_avro_to_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_spark_z_parquet_to_avro.sh b/tests/test_spark_z_parquet_to_avro.sh
index 6e88f0cfc..9e02af7ed 100755
--- a/tests/test_spark_z_parquet_to_avro.sh
+++ b/tests/test_spark_z_parquet_to_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_timeout.sh b/tests/test_timeout.sh
index 26d38d589..5d46e4bd6 100755
--- a/tests/test_timeout.sh
+++ b/tests/test_timeout.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-02-14 16:16:00 +0000 (Sun, 14 Feb 2016)
#
-# https://github.com/harisekhon/bash-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_avro.sh b/tests/test_validate_avro.sh
index d8dab1d88..b8bf5fbad 100755
--- a/tests/test_validate_avro.sh
+++ b/tests/test_validate_avro.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_cson.sh b/tests/test_validate_cson.sh
index 20b9d332b..966a6d201 100755
--- a/tests/test_validate_cson.sh
+++ b/tests/test_validate_cson.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_csv.sh b/tests/test_validate_csv.sh
index 680b63390..1747bebcc 100755
--- a/tests/test_validate_csv.sh
+++ b/tests/test_validate_csv.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_ini.sh b/tests/test_validate_ini.sh
index f2282be6a..72349b24d 100755
--- a/tests/test_validate_ini.sh
+++ b/tests/test_validate_ini.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_json.sh b/tests/test_validate_json.sh
index 39b24306b..b92d16c2c 100755
--- a/tests/test_validate_json.sh
+++ b/tests/test_validate_json.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_ldap_ldif.sh b/tests/test_validate_ldap_ldif.sh
index 53af23074..267269b32 100755
--- a/tests/test_validate_ldap_ldif.sh
+++ b/tests/test_validate_ldap_ldif.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2017-08-20 12:56:43 +0100 (Sun, 20 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_multimedia.sh b/tests/test_validate_multimedia.sh
index 7302ca55d..3493bae6d 100755
--- a/tests/test_validate_multimedia.sh
+++ b/tests/test_validate_multimedia.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2016-05-01 20:46:42 +0100 (Sun, 01 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
@@ -32,7 +32,7 @@ if ! type -P ffmpeg &>/dev/null; then
echo "WARNING: ffmpeg not installed, skipping validate_multimedia.py tests"
exit 0
if type -P apt-get &>/dev/null; then
- sudo apt-get install -y ffmpeg
+ sudo apt-get install -o DPkg::Lock::Timeout=1200 -y ffmpeg
elif type -P yum &>/dev/null; then
echo "WARNING: cannot auto-install ffmpeg on RHEL/CentOS, the 3rd party repos and deps are seriously broken"
fi
diff --git a/tests/test_validate_parquet.sh b/tests/test_validate_parquet.sh
index d75942e84..44b0c8f7d 100755
--- a/tests/test_validate_parquet.sh
+++ b/tests/test_validate_parquet.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_toml.sh b/tests/test_validate_toml.sh
index 3dd37ed60..c20a38cf6 100755
--- a/tests/test_validate_toml.sh
+++ b/tests/test_validate_toml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_xml.sh b/tests/test_validate_xml.sh
index 37b1ee106..7fe631f3b 100755
--- a/tests/test_validate_xml.sh
+++ b/tests/test_validate_xml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_validate_yaml.sh b/tests/test_validate_yaml.sh
index 098cf827d..abd4343ca 100755
--- a/tests/test_validate_yaml.sh
+++ b/tests/test_validate_yaml.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-12-22 23:39:33 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/test_welcome.sh b/tests/test_welcome.sh
index 37e3d59af..0a13136a7 100755
--- a/tests/test_welcome.sh
+++ b/tests/test_welcome.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2015-11-05 23:29:15 +0000 (Thu, 05 Nov 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/tests/test_xml_to_json.sh b/tests/test_xml_to_json.sh
index 8d01e3750..6bd9cf731 100755
--- a/tests/test_xml_to_json.sh
+++ b/tests/test_xml_to_json.sh
@@ -4,30 +4,30 @@
# Author: Hari Sekhon
# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "$0")" && pwd)"
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$srcdir";
+cd "$srcdir"
-# shellcheck disable=SC1091
-. utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
-# shellcheck disable=SC1091
-. ../bash-tools/lib/utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
section "XML => JSON"
-cd ..
+cd "$srcdir/.."
testdata="tests/data/simple.xml"
diff --git a/tests/test_xml_to_yaml.sh b/tests/test_xml_to_yaml.sh
index 6f93a6ddf..4b3d72da0 100755
--- a/tests/test_xml_to_yaml.sh
+++ b/tests/test_xml_to_yaml.sh
@@ -4,30 +4,30 @@
# Author: Hari Sekhon
# Date: 2016-08-29 18:18:39 +0100 (Mon, 29 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
-srcdir="$(cd "$(dirname "$0")" && pwd)"
+srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$srcdir";
+cd "$srcdir"
-# shellcheck disable=SC1091
-. utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/utils.sh"
-# shellcheck disable=SC1091
-. ../bash-tools/lib/utils.sh
+# shellcheck disable=SC1091,SC1090
+. "$srcdir/../bash-tools/lib/utils.sh"
section "XML => YAML"
-cd ..
+cd "$srcdir/.."
testdata="tests/data/simple.xml"
diff --git a/tests/test_yamllint.sh b/tests/test_yamllint.sh
index 31d808ff6..90c983ce4 100755
--- a/tests/test_yamllint.sh
+++ b/tests/test_yamllint.sh
@@ -4,13 +4,13 @@
# Author: Hari Sekhon
# Date: 2019-02-26 14:40:53 +0000 (Tue, 26 Feb 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
diff --git a/tests/utils.sh b/tests/utils.sh
index 57c32ca46..eddb33d01 100755
--- a/tests/utils.sh
+++ b/tests/utils.sh
@@ -5,13 +5,13 @@
# Author: Hari Sekhon
# Date: 2015-05-25 01:38:24 +0100 (Mon, 25 May 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
set -eu
diff --git a/timeout.py b/timeout.py
index b274872a3..831d2e99f 100755
--- a/timeout.py
+++ b/timeout.py
@@ -1,16 +1,16 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-02-14 15:46:37 +0000 (Sun, 14 Feb 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # pylint: disable=line-too-long
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/travis_debug_session.py b/travis_debug_session.py
index 805fecdd0..aafebf35e 100755
--- a/travis_debug_session.py
+++ b/travis_debug_session.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/travis_last_log.py b/travis_last_log.py
index c703ef300..82b7493e5 100755
--- a/travis_last_log.py
+++ b/travis_last_log.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-08-10 18:18:03 +0100 (Wed, 10 Aug 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/urldecode.py b/urldecode.py
index 4dd2242b7..87319d3be 100755
--- a/urldecode.py
+++ b/urldecode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/urlencode.py b/urlencode.py
index 90d6ab5e6..6ca907f15 100755
--- a/urlencode.py
+++ b/urlencode.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-03-03 17:34:06 +0000 (Tue, 03 Mar 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_avro.py b/validate_avro.py
index f81c3afb7..37eaab848 100755
--- a/validate_avro.py
+++ b/validate_avro.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_cson.py b/validate_cson.py
index 4287f6d93..6e017ab97 100755
--- a/validate_cson.py
+++ b/validate_cson.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_csv.py b/validate_csv.py
index c53d16eb1..bac1d583e 100755
--- a/validate_csv.py
+++ b/validate_csv.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_ini.py b/validate_ini.py
index 84411046d..b38b3ab6a 100755
--- a/validate_ini.py
+++ b/validate_ini.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
@@ -58,7 +58,7 @@
sys.exit(4)
__author__ = 'Hari Sekhon'
-__version__ = '0.12.2'
+__version__ = '0.12.3'
class IniValidatorTool(CLI):
@@ -72,7 +72,7 @@ def __init__(self):
self.re_suffix = re.compile(r'.*\.(?:ini|properties)$', re.I)
# In Windows ini key cannot contain equals sign = or semicolon ;
# key=val or [section]
- self.re_ini_section = re.compile(r'^\s*\[([\w=\:\.-]+)\]\s*$')
+ self.re_ini_section = re.compile(r'^\s*\[([\w\s=\:\.-]+)\]\s*$')
self.re_ini_key = re.compile(r'^\s*(?:[^\[;=]+)s*$')
# INI value can be anything .* so not regex'ing it
self.valid_ini_msg = ' => INI OK'
diff --git a/validate_ini2.py b/validate_ini2.py
index aaf17f87f..5c41436a4 100755
--- a/validate_ini2.py
+++ b/validate_ini2.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-09-15 15:29:39 +0200 (Fri, 15 Sep 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_json.py b/validate_json.py
index 1d83b8b04..dbca2c431 100755
--- a/validate_json.py
+++ b/validate_json.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_ldap_ldif.py b/validate_ldap_ldif.py
index 02e3e4bf7..963b725c8 100755
--- a/validate_ldap_ldif.py
+++ b/validate_ldap_ldif.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# coding=utf-8
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2017-08-04 00:53:10 +0200 (Fri, 04 Aug 2017)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn
# and optionally send me feedback to help steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_multimedia.py b/validate_multimedia.py
index 54def6deb..0362bb30d 100755
--- a/validate_multimedia.py
+++ b/validate_multimedia.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-05-01 20:46:56 +0100 (Sun, 01 May 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_parquet.py b/validate_parquet.py
index 71894645f..bf6690d06 100755
--- a/validate_parquet.py
+++ b/validate_parquet.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_toml.py b/validate_toml.py
index b75c2dce0..25628d222 100755
--- a/validate_toml.py
+++ b/validate_toml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-10-07 10:23:31 +0100 (Mon, 07 Oct 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_xml.py b/validate_xml.py
index 5620a81e5..c954bac42 100755
--- a/validate_xml.py
+++ b/validate_xml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/validate_yaml.py b/validate_yaml.py
index 819943294..75ee4034c 100755
--- a/validate_yaml.py
+++ b/validate_yaml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2015-12-22 23:25:25 +0000 (Tue, 22 Dec 2015)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/welcome.py b/welcome.py
index 60d6533af..d9c1b2546 100755
--- a/welcome.py
+++ b/welcome.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2009-12-09 19:58:14 +0000 (Wed, 09 Dec 2009)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# https://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/wercker.yml b/wercker.yml
deleted file mode 100644
index f57ab989f..000000000
--- a/wercker.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Author: Hari Sekhon
-# Date: 2020-02-24 15:41:04 +0000 (Mon, 24 Feb 2020)
-#
-# vim:ts=2:sts=2:sw=2:et
-#
-# https://github.com/harisekhon/devops-python-tools
-#
-# License: see accompanying Hari Sekhon LICENSE file
-#
-# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
-#
-# https://www.linkedin.com/in/harisekhon
-#
-
-# https://devcenter.wercker.com/reference/wercker-yml/
-
-box: debian
-
-build:
- steps:
- - script:
- name: ci bootstrap
- code: setup/ci_bootstrap.sh
- - script:
- name: init
- code: make init
- - script:
- name: build
- code: make ci
- - script:
- name: test
- code: make test
diff --git a/xml_to_json.py b/xml_to_json.py
index a82ea8adf..14d4cde62 100755
--- a/xml_to_json.py
+++ b/xml_to_json.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2016-01-15 00:07:09 +0000 (Fri, 15 Jan 2016)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/xml_to_yaml.py b/xml_to_yaml.py
index 6643a970a..47a098243 100755
--- a/xml_to_yaml.py
+++ b/xml_to_yaml.py
@@ -1,17 +1,17 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2019-12-19 18:19:34 +0000 (Thu, 19 Dec 2019)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""
diff --git a/yaml_to_json.py b/yaml_to_json.py
index 2085e6928..90a5db9b7 100755
--- a/yaml_to_json.py
+++ b/yaml_to_json.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# vim:ts=4:sts=4:sw=4:et
# args: ../bash-tools/.gitlab-ci.yml
#
# Author: Hari Sekhon
# Date: 2020-08-18 01:17:56 +0100 (Tue, 18 Aug 2020)
#
-# https://github.com/harisekhon/devops-python-tools
+# https://github.com/HariSekhon/DevOps-Python-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
-# http://www.linkedin.com/in/harisekhon
+# https://www.linkedin.com/in/HariSekhon
#
"""