From fc0e757fa5e27970325af08625d7298d2a6194e7 Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 27 Dec 2020 16:07:08 -0500 Subject: [PATCH 1/8] updated build files --- script-python/build-lambda.sh | 106 ++++++++++++++++++++++++++++++++++ script-python/build.sh | 65 +-------------------- 2 files changed, 108 insertions(+), 63 deletions(-) create mode 100755 script-python/build-lambda.sh diff --git a/script-python/build-lambda.sh b/script-python/build-lambda.sh new file mode 100755 index 0000000..a8064ef --- /dev/null +++ b/script-python/build-lambda.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +CWD=$(cd "$(dirname "$0")" && pwd) +. $CWD/build.sh + +MODULE_NAME="" +MODULE_VERSION="" + +CORE_MODULE=${MODULE_NAME}-${MODULE_VERSION} + +LOCATION_DEV_ENV=$HOME/dev/env/ +LOCATION_AWS=$CWD/deploy/aws/ + +VIRTUAL_ENV_PACKAGES=lib/python2.7/site-packages + +case "$1" in + 'clean') + rm -rf $CWD/{build,deploy,dist,${MODULE_NAME}.egg-info} + ;; + + 'install') + python setup.py install + ;; + + 'build.src') + python setup.py sdist + ;; + + 'build.wheel') + python setup.py bdist_wheel + ;; + + 'deploy.local') + $0 build.wheel + pip install $CWD/dist/${CORE_MODULE}-py2-none-any.whl + ;; + + 'add.lambda') + cd $CWD/lambda + zip -g $2 lambda_config.py + zip -g $2 lambda_module.py + + cd $CWD + ;; + + 'build.aws') + # goto path/to/ + VIRTUAL_ENV_DIR=$2 + cd $VIRTUAL_ENV_DIR + + # then goto /lib/python2.7/site-packages + # cd lib/python2.7/site-packages + cd $VIRTUAL_ENV_PACKAGES + + # zip everything recursively to $ZIP_FILE + ZIP_FILE=$3 + + echo -e "# creating $ZIP_FILE" + zip -r9 $ZIP_FILE * + cd $CWD + + # add lambda-related modules into $ZIP_FILE + echo -e "# adding lambda modules to $ZIP_FILE" + $0 add.lambda $ZIP_FILE + cd $CWD + ;; + + 'deploy.aws') + if [ ! -d "$LOCATION_AWS" ]; then + mkdir -p $LOCATION_AWS + fi + + $0 deploy.local ${CORE_MODULE} + + OUTPUT_FILE=${CORE_MODULE}.zip + if [ ! -z $2 ]; then + OUTPUT_FILE=${CORE_MODULE}_$2.zip + fi + + $0 build.aws $LOCATION_DEV_ENV $LOCATION_AWS/${OUTPUT_FILE} + ;; + + 'update.aws') + ZIP_FILE=$LOCATION_AWS/${CORE_MODULE}_$2.zip + $0 add.lambda $ZIP_FILE + ;; + + *) + echo "Usage: $0 options" + echo "where options are" + echo "- clean # clean directories" + echo "- install # install to virtual environment" + echo + echo "# for local deployment" + echo "- build.src # build project" + echo "- build.wheel # build project in wheel file for deployment" + echo "- deploy.local - (i.e test-0.0.1)" + echo + echo "# for aws deployment" + echo "- add.lambda path/to/zipfile" + echo "- build.aws path/to/virtualenv path/to/zipfile" + echo "- deploy.aws yyyymmdd" + echo + +esac +exit 0 diff --git a/script-python/build.sh b/script-python/build.sh index 5463e9f..0a7552a 100755 --- a/script-python/build.sh +++ b/script-python/build.sh @@ -4,13 +4,7 @@ CWD=$(cd "$(dirname "$0")" && pwd) MODULE_NAME="" MODULE_VERSION="" - -CORE_MODULE=${MODULE_NAME}-${MODULE_VERSION} - -LOCATION_DEV_ENV=$HOME/dev/env/ -LOCATION_AWS=$CWD/deploy/aws/ - -VIRTUAL_ENV_PACKAGES=lib/python2.7/site-packages +MODULE_WHEEL_FILE=${MODULE_NAME}-${MODULE_VERSION}-py2-none-any.whl case "$1" in 'clean') @@ -31,57 +25,7 @@ case "$1" in 'deploy.local') $0 build.wheel - pip install $CWD/dist/${CORE_MODULE}-py2-none-any.whl - ;; - - 'add.lambda') - cd $CWD/lambda - zip -g $2 lambda_config.py - zip -g $2 lambda_module.py - - cd $CWD - ;; - - 'build.aws') - # goto path/to/ - VIRTUAL_ENV_DIR=$2 - cd $VIRTUAL_ENV_DIR - - # then goto /lib/python2.7/site-packages - # cd lib/python2.7/site-packages - cd $VIRTUAL_ENV_PACKAGES - - # zip everything recursively to $ZIP_FILE - ZIP_FILE=$3 - - echo -e "# creating $ZIP_FILE" - zip -r9 $ZIP_FILE * - cd $CWD - - # add lambda-related modules into $ZIP_FILE - echo -e "# adding lambda modules to $ZIP_FILE" - $0 add.lambda $ZIP_FILE - cd $CWD - ;; - - 'deploy.aws') - if [ ! -d "$LOCATION_AWS" ]; then - mkdir -p $LOCATION_AWS - fi - - $0 deploy.local ${CORE_MODULE} - - OUTPUT_FILE=${CORE_MODULE}.zip - if [ ! -z $2 ]; then - OUTPUT_FILE=${CORE_MODULE}_$2.zip - fi - - $0 build.aws $LOCATION_DEV_ENV $LOCATION_AWS/${OUTPUT_FILE} - ;; - - 'update.aws') - ZIP_FILE=$LOCATION_AWS/${CORE_MODULE}_$2.zip - $0 add.lambda $ZIP_FILE + pip install $CWD/dist/${MODULE_WHEEL_FILE} ;; *) @@ -95,11 +39,6 @@ case "$1" in echo "- build.wheel # build project in wheel file for deployment" echo "- deploy.local - (i.e test-0.0.1)" echo - echo "# for aws deployment" - echo "- add.lambda path/to/zipfile" - echo "- build.aws path/to/virtualenv path/to/zipfile" - echo "- deploy.aws yyyymmdd" - echo esac exit 0 From 4342b6882162922d5983e12f0b204895a3451273 Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 14 Mar 2021 08:09:11 -0400 Subject: [PATCH 2/8] updated script-hadoop --- README.md | 2 ++ script-hadoop/README.md | 14 ++++++++++++-- script-hadoop/hadoop2/bin/sys/setup-yarn.sh | 3 +-- script-hadoop/hadoop2/conf/hdfs-site.xml | 9 --------- script-hadoop/hadoop2/conf/yarn-site.xml | 3 --- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 935bd82..33d298e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ - script-db - script-elasticsearch - script-hadoop +- script-java - script-os +- script-python - script-solr - script-spark - script-zookeeper diff --git a/script-hadoop/README.md b/script-hadoop/README.md index d39dbef..bc3ea6c 100644 --- a/script-hadoop/README.md +++ b/script-hadoop/README.md @@ -1,9 +1,13 @@ This project contains script and configuration files for setting up a stand-alone or single node Hadoop Cluster for development and testing. # References -- Hadoop r1.2.1 [Documentation] (http://hadoop.apache.org/docs/r1.2.1/) | [Single-Node Setup] (http://hadoop.apache.org/docs/r1.2.1/single_node_setup.html) | [Cluster Setup] (http://hadoop.apache.org/docs/r1.2.1/cluster_setup.html) +- All versions [Documentation](https://hadoop.apache.org/docs/stable/index.html) -- Hadoop r2.x [Documentation] (http://hadoop.apache.org/docs/current/) | [Single-Node Setup] (http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup] (http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ClusterSetup.html) +- Hadoop r3.x "current" [Documentation](http://hadoop.apache.org/docs/current/) | [Single-Node Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ClusterSetup.html) + +- Hadoop r2.10.1 [Documentation] http://hadoop.apache.org/docs/r2.10.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/ClusterSetup.html) + +- Hadoop r1.2.1 [Documentation](http://hadoop.apache.org/docs/r1.2.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r1.2.1/single_node_setup.html) | [Cluster Setup](http://hadoop.apache.org/docs/r1.2.1/cluster_setup.html) - Michael G. Knoll [Running Hadoop on Ubuntu Linux (Single-Node Cluster)] (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-single-node-cluster/) @@ -11,6 +15,12 @@ This project contains script and configuration files for setting up a stand-alon - Michael G. Knoll [Running Hadoop on Ubuntu Linux (Multi-Node Cluster)] (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-multi-node-cluster/) +- 20 Notable Difference Between Hadoop 2.x vs Hadoop 3.x +(https://techvidvan.com/tutorials/hadoop-2-x-vs-hadoop-3-x/) + +- Difference Between Hadoop 2.x vs Hadoop 3.x +(https://www.geeksforgeeks.org/difference-between-hadoop-2-x-vs-hadoop-3-x/) + - 10 Big Differences between Hadoop 1 & Hadoop 2 (YARN) (https://acadgild.com/blog/10-big-differences-between-hadoop1-and-hadoop2/) diff --git a/script-hadoop/hadoop2/bin/sys/setup-yarn.sh b/script-hadoop/hadoop2/bin/sys/setup-yarn.sh index 1d053ad..abd9522 100755 --- a/script-hadoop/hadoop2/bin/sys/setup-yarn.sh +++ b/script-hadoop/hadoop2/bin/sys/setup-yarn.sh @@ -20,8 +20,7 @@ if [ -d "$HADOOP_ORG" ]; then cp -R $CWD/../../conf/* $HADOOP_CFG # setup data location - # HADOOP_DATA=$APP_DATA/yarn - HADOOP_DATA=$HADOOP_DATA + HADOOP_DATA=$APP_DATA/yarn mkdir -p $HADOOP_DATA/{logs,pids,local} mkdir -p $HADOOP_DATA/mapred/{local,temp,system} diff --git a/script-hadoop/hadoop2/conf/hdfs-site.xml b/script-hadoop/hadoop2/conf/hdfs-site.xml index 9798969..c2a8257 100755 --- a/script-hadoop/hadoop2/conf/hdfs-site.xml +++ b/script-hadoop/hadoop2/conf/hdfs-site.xml @@ -37,8 +37,6 @@ default: 128MB (use 256MB for large file systems) -<<<<<<< HEAD -======= dfs.permissions.superusergroup hadoop @@ -49,7 +47,6 @@ 1 ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 dfs.namenode.checkpoint.dir @@ -69,14 +66,8 @@ -<<<<<<< HEAD - dfs.permissions.superusergroup - hadoop -======= dfs.datanode.handler.count 10 default: 10 ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 - diff --git a/script-hadoop/hadoop2/conf/yarn-site.xml b/script-hadoop/hadoop2/conf/yarn-site.xml index c3440e5..8697f63 100755 --- a/script-hadoop/hadoop2/conf/yarn-site.xml +++ b/script-hadoop/hadoop2/conf/yarn-site.xml @@ -5,7 +5,6 @@ yarn.resourcemanager.hostname localhost -<<<<<<< HEAD yarn.resourcemanager.webapp.address @@ -17,8 +16,6 @@ true default: false -======= ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 From 7a1e29a21c35954247b0fca7bbc52b59d795cb79 Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 14 Mar 2021 08:13:41 -0400 Subject: [PATCH 3/8] updated README in script-hadoop --- script-hadoop/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script-hadoop/README.md b/script-hadoop/README.md index bc3ea6c..9040758 100644 --- a/script-hadoop/README.md +++ b/script-hadoop/README.md @@ -1,18 +1,18 @@ This project contains script and configuration files for setting up a stand-alone or single node Hadoop Cluster for development and testing. # References -- All versions [Documentation](https://hadoop.apache.org/docs/stable/index.html) +- All [Versions](http://hadoop.apache.org/docs/) - Hadoop r3.x "current" [Documentation](http://hadoop.apache.org/docs/current/) | [Single-Node Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ClusterSetup.html) -- Hadoop r2.10.1 [Documentation] http://hadoop.apache.org/docs/r2.10.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/ClusterSetup.html) +- Hadoop r2.10.1 [Documentation](http://hadoop.apache.org/docs/r2.10.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/ClusterSetup.html) - Hadoop r1.2.1 [Documentation](http://hadoop.apache.org/docs/r1.2.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r1.2.1/single_node_setup.html) | [Cluster Setup](http://hadoop.apache.org/docs/r1.2.1/cluster_setup.html) -- Michael G. Knoll [Running Hadoop on Ubuntu Linux (Single-Node Cluster)] +- Michael G. Knoll - Running Hadoop on Ubuntu Linux (Single-Node Cluster) (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-single-node-cluster/) -- Michael G. Knoll [Running Hadoop on Ubuntu Linux (Multi-Node Cluster)] +- Michael G. Knoll - Running Hadoop on Ubuntu Linux (Multi-Node Cluster) (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-multi-node-cluster/) - 20 Notable Difference Between Hadoop 2.x vs Hadoop 3.x From 9e8460bf60ee9e7d858f149b3577b49658d7a8d8 Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 14 Mar 2021 16:14:37 -0400 Subject: [PATCH 4/8] updated scripte-java --- script-java/bin/common.sh | 7 +++---- script-java/bin/service.sh | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/script-java/bin/common.sh b/script-java/bin/common.sh index ad43fb5..d8cdb70 100755 --- a/script-java/bin/common.sh +++ b/script-java/bin/common.sh @@ -7,7 +7,7 @@ start() { case "$1" in start) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then start @@ -30,14 +30,13 @@ case "$1" in ;; stop) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then echo "${SERVICE_NAME} is not running [${PID}]" rm -f $PID_FILE exit 1 else - #PID=`cat $PID_FILE` kill -term $PID echo "${SERVICE_NAME} is stopped [${PID}]" rm -f $PID_FILE @@ -50,7 +49,7 @@ case "$1" in ;; status) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then echo "${SERVICE_NAME} is not running [${PID}]" diff --git a/script-java/bin/service.sh b/script-java/bin/service.sh index dbe867f..59f4906 100755 --- a/script-java/bin/service.sh +++ b/script-java/bin/service.sh @@ -16,7 +16,7 @@ APP_JARS=$LIB_JARS CLASSPATH=$APP_JARS:$APP_CONFIG export CLASSPATH -SERVICE_NAME="a" +SERVICE_NAME="service-a" PID_FILE="${APP_HOME}/${SERVICE_NAME}.pid" APP_OPTS="-Dlog4j.configurationFile=$APP_CONFIG/${SERVICE_NAME}.xml" APP_OPTS="$APP_OPTS -Dconfig.file=$APP_CONFIG/${SERVICE_NAME}.yml" From 0c03d4536d8bd8dfea1e4439e44aeeaa7b639fae Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 14 Mar 2021 16:40:07 -0400 Subject: [PATCH 5/8] updated script-os --- script-aws/bin/s3api.sh | 66 ++++---- script-db/citusdb/README.md | 5 + script-db/citusdb/bin/README.md | 8 + script-db/citusdb/bin/citus.sh | 146 ++++++++++++++++++ script-db/citusdb/bin/citus_multi-pg.sh | 115 ++++++++++++++ script-db/citusdb/bin/citus_multi.sh | 117 ++++++++++++++ script-db/citusdb/dbtest/README.md | 10 ++ script-db/citusdb/dbtest/ddl_create.sql | 5 + script-db/citusdb/dbtest/ddl_drop.sql | 2 + script-db/citusdb/dbtest/setup.sh | 79 ++++++++++ .../dbtest/setup_distributed-table.sql | 2 + .../citusdb/dbtest/setup_distributed.sql | 2 + script-db/citusdb/dbtest/setup_sharding.sql | 2 + script-db/citusdb/dbtest/users_create.sql | 7 + script-db/citusdb/dbtest/users_drop.sql | 2 + script-os/linux/dev/es.sh | 92 +++++++++++ script-os/linux/dev/kibana.sh | 92 +++++++++++ script-os/linux/sys/commons.sh | 11 ++ script-os/linux/sys/remote-cpi.sh | 3 +- script-os/linux/sys/remote-exi.sh | 3 +- script-os/linux/tools/mu.sh | 4 - 21 files changed, 732 insertions(+), 41 deletions(-) create mode 100644 script-db/citusdb/README.md create mode 100644 script-db/citusdb/bin/README.md create mode 100755 script-db/citusdb/bin/citus.sh create mode 100755 script-db/citusdb/bin/citus_multi-pg.sh create mode 100755 script-db/citusdb/bin/citus_multi.sh create mode 100644 script-db/citusdb/dbtest/README.md create mode 100755 script-db/citusdb/dbtest/ddl_create.sql create mode 100755 script-db/citusdb/dbtest/ddl_drop.sql create mode 100644 script-db/citusdb/dbtest/setup.sh create mode 100755 script-db/citusdb/dbtest/setup_distributed-table.sql create mode 100644 script-db/citusdb/dbtest/setup_distributed.sql create mode 100755 script-db/citusdb/dbtest/setup_sharding.sql create mode 100755 script-db/citusdb/dbtest/users_create.sql create mode 100755 script-db/citusdb/dbtest/users_drop.sql create mode 100755 script-os/linux/dev/es.sh create mode 100755 script-os/linux/dev/kibana.sh create mode 100644 script-os/linux/sys/commons.sh delete mode 100755 script-os/linux/tools/mu.sh diff --git a/script-aws/bin/s3api.sh b/script-aws/bin/s3api.sh index 8aa8dca..48fca1f 100755 --- a/script-aws/bin/s3api.sh +++ b/script-aws/bin/s3api.sh @@ -12,52 +12,52 @@ function s3_list() { } function s3_delete_regions() { - STAGE=$1 - PARTY_ID=$2 - TYPE=$3 - shift - shift - shift - REGIONS=("$@") - - for REGION in ${REGIONS[@]}; do - s3_delete ${STAGE} ${PARTY_ID} ${TYPE} ${REGION} - done + STAGE=$1 + PARTY_ID=$2 + TYPE=$3 + shift + shift + shift + REGIONS=("$@") + + for REGION in ${REGIONS[@]}; do + s3_delete ${STAGE} ${PARTY_ID} ${TYPE} ${REGION} + done } function s3_delete() { - PREFIX=$1 - REGION=$2 + PREFIX=$1 + REGION=$2 - BUCKET_NAME="${PREFIX}-${REGION}" - echo "bucket: ${BUCKET_NAME}" - echo + BUCKET_NAME="${PREFIX}-${REGION}" + echo "bucket: ${BUCKET_NAME}" + echo - aws s3api delete-bucket --bucket ${BUCKET_NAME} \ - --region ${REGION} + aws s3api delete-bucket --bucket ${BUCKET_NAME} \ + --region ${REGION} } function s3_create_regions() { - PREFIX=$1 - shift - REGIONS=("$@") - - for REGION in ${REGIONS[@]}; do - echo "# creating: ${PREFIX}-${REGION}" - s3_create ${PREFIX} ${REGION} - done + PREFIX=$1 + shift + REGIONS=("$@") + + for REGION in ${REGIONS[@]}; do + echo "# creating: ${PREFIX}-${REGION}" + s3_create ${PREFIX} ${REGION} + done } function s3_create() { - PREFIX=$1 - REGION=$2 + PREFIX=$1 + REGION=$2 - BUCKET_NAME="${PREFIX}-${REGION}" - echo "- bucket: ${BUCKET_NAME}" + BUCKET_NAME="${PREFIX}-${REGION}" + echo "- bucket: ${BUCKET_NAME}" - aws s3api create-bucket --bucket ${BUCKET_NAME} \ - --region ${REGION} \ - --create-bucket-configuration LocationConstraint=${REGION} + aws s3api create-bucket --bucket ${BUCKET_NAME} \ + --region ${REGION} \ + --create-bucket-configuration LocationConstraint=${REGION} } case "$1" in diff --git a/script-db/citusdb/README.md b/script-db/citusdb/README.md new file mode 100644 index 0000000..be98ef4 --- /dev/null +++ b/script-db/citusdb/README.md @@ -0,0 +1,5 @@ +CitusDB +======= + +# References +- CitusDB v5.1 [Documentation] (https://docs.citusdata.com/en/v5.1/) | [Single-Machine Cluster] (https://docs.citusdata.com/en/v5.1/installation/development.html) | [Multi-Machine Cluster] (https://docs.citusdata.com/en/v5.1/installation/production.html) diff --git a/script-db/citusdb/bin/README.md b/script-db/citusdb/bin/README.md new file mode 100644 index 0000000..8d8dd3b --- /dev/null +++ b/script-db/citusdb/bin/README.md @@ -0,0 +1,8 @@ +CitusDB Setup scripts +===================== + +- citus.sh # to setup single-machine cluster + +- citus_multi.sh # to setup custom multi-machine cluster + +- citus_multi-pg.sh # to setup multi-machine cluster using default installation \ No newline at end of file diff --git a/script-db/citusdb/bin/citus.sh b/script-db/citusdb/bin/citus.sh new file mode 100755 index 0000000..086b14a --- /dev/null +++ b/script-db/citusdb/bin/citus.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +APP_DATA="/data/citus" +APP_LOG="$APP_DATA/log" + +function init_db() { + echo -e "\n# initializing the installation" + + mkdir -p $APP_DATA/{master,worker1,worker2} + mkdir -p $APP_LOG + + # create three normal postgres instances + initdb -D $APP_DATA/master + initdb -D $APP_DATA/worker1 + initdb -D $APP_DATA/worker2 + + # set workers in master + echo "localhost 9701" >> $APP_DATA/master/pg_worker_list.conf + echo "localhost 9702" >> $APP_DATA/master/pg_worker_list.conf + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/master/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/worker1/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/worker2/postgresql.conf +} + +function start_db() { + echo -e "\n# starting master instance" + pg_ctl -D $APP_DATA/master -o "-p 9700" -l $APP_LOG/master.log start + + echo -e "\n# starting worker1 instance" + pg_ctl -D $APP_DATA/worker1 -o "-p 9701" -l $APP_LOG/worker1.log start + + echo -e "\n# starting worker2 instance" + pg_ctl -D $APP_DATA/worker2 -o "-p 9702" -l $APP_LOG/worker2.log start +} + +function stop_db() { + echo -e "\n# stopping worker1 instance" + pg_ctl -D $APP_DATA/worker1 -m fast stop + + echo -e "\n# stopping worker2 instance" + pg_ctl -D $APP_DATA/worker2 -m fast stop + + echo -e "\n# stopping master instance" + pg_ctl -D $APP_DATA/master -m fast stop +} + +function status_db() { + echo -e "\n# worker1 instance's status" + pg_ctl -D $APP_DATA/worker1 status + + echo -e "\n# worker2 instance's status" + pg_ctl -D $APP_DATA/worker2 status + + echo -e "\n# master instance's status" + pg_ctl -D $APP_DATA/master status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + createdb -p 9700 $DB_NAME + createdb -p 9701 $DB_NAME + createdb -p 9702 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + dropdb -p 9700 $DB_NAME + dropdb -p 9701 $DB_NAME + dropdb -p 9702 $DB_NAME +} + +function load_ext() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" + psql -p 9701 -d $DB_NAME -c "CREATE EXTENSION citus;" + psql -p 9702 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_ext "$2" + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 (init | {create | drop | verify } | start | stop | status | clear)" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/bin/citus_multi-pg.sh b/script-db/citusdb/bin/citus_multi-pg.sh new file mode 100755 index 0000000..8b1956d --- /dev/null +++ b/script-db/citusdb/bin/citus_multi-pg.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +BASE_DIR="/var/lib/pgsql/9.5" +APP_DATA="$BASE_DIR/data" +APP_LOGS="$BASE_DIR/logs" + +PG_HOME="/usr/pgsql-" +PG_BIN="$PG_HOME/bin" + +function init_db() { + echo -e "\n# initializing the installation" + mkdir -p $APP_DATA + mkdir -p $APP_LOGS + + # create normal postgres instance + sudo service postgresql-9.5 initdb || sudo $PG_BIN/postgresql95-setup initdb + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" | sudo tee -a $APP_DATA/postgresql.conf +} + +function start_db() { + echo -e "\n# starting postgresql instance" + sudo service postgresql-9.5 restart +} + +function stop_db() { + echo -e "\n# stopping postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -m fast stop +} + +function status_db() { + echo -e "\n# postgresql instance's status" + $PG_BIN/pg_ctl -D $APP_DATA status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + $PG_BIN/createdb -p 9700 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + $PG_BIN/dropdb -p 9700 $DB_NAME +} + +function load_db() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* + rm -rf $APP_LOGS/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_db "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 {init | {create | drop } | start | stop | status | clear}" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/bin/citus_multi.sh b/script-db/citusdb/bin/citus_multi.sh new file mode 100755 index 0000000..3a44cca --- /dev/null +++ b/script-db/citusdb/bin/citus_multi.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +BASE_DIR="/data/citus" +APP_DATA="$BASE_DIR/data" +APP_LOGS="$BASE_DIR/logs" + +PG_HOME="/usr/pgsql-" +PG_BIN="$PG_HOME/bin" + +function init_db() { + echo -e "\n# initializing the installation" + mkdir -p $APP_DATA + mkdir -p $APP_LOGS + + # create normal postgres instance + $PG_BIN/initdb -D $APP_DATA + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/postgresql.conf +} + +function start_db() { + echo -e "\n# starting postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -o "-p 9700" -l $APP_LOGS/citus.log start +} + +function stop_db() { + echo -e "\n# stopping postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -m fast stop +} + +function status_db() { + echo -e "\n# postgresql instance's status" + $PG_BIN/pg_ctl -D $APP_DATA status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + $PG_BIN/createdb -p 9700 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + $PG_BIN/dropdb -p 9700 $DB_NAME +} + +function load_ext() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* + rm -rf $APP_LOGS/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_ext "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 {init | {create | drop | verify } | start | stop | status | clear}" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/dbtest/README.md b/script-db/citusdb/dbtest/README.md new file mode 100644 index 0000000..9b5f88a --- /dev/null +++ b/script-db/citusdb/dbtest/README.md @@ -0,0 +1,10 @@ +Schema Scripts +============== + +Contains scripts to create a simple schema and a user in dbtest database + +# Instructions + +- run "setup.sh" then use appropriate parameters + + diff --git a/script-db/citusdb/dbtest/ddl_create.sql b/script-db/citusdb/dbtest/ddl_create.sql new file mode 100755 index 0000000..d6d7012 --- /dev/null +++ b/script-db/citusdb/dbtest/ddl_create.sql @@ -0,0 +1,5 @@ +create table mytable ( + id char(36) not null, + name char(64) not null, + insert_time timestamp not null +); diff --git a/script-db/citusdb/dbtest/ddl_drop.sql b/script-db/citusdb/dbtest/ddl_drop.sql new file mode 100755 index 0000000..c7fd3a8 --- /dev/null +++ b/script-db/citusdb/dbtest/ddl_drop.sql @@ -0,0 +1,2 @@ +drop table if exists mytable cascade; + diff --git a/script-db/citusdb/dbtest/setup.sh b/script-db/citusdb/dbtest/setup.sh new file mode 100644 index 0000000..550aa2e --- /dev/null +++ b/script-db/citusdb/dbtest/setup.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/9.5.3 +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +function execute() { + DB_NAME="$1" + DB_SCRIPT="$2" + + psql -p 9700 -d $DB_NAME < $DB_SCRIPT +} + +function execute_users() { + DB_NAME="$1" + DB_SCRIPT="$2" + + psql -p 9700 -d $DB_NAME < $DB_SCRIPT + psql -p 9701 -d $DB_NAME < $DB_SCRIPT + psql -p 9702 -d $DB_NAME < $DB_SCRIPT +} + +case "$1" in + 'db.create') + execute "$2" ddl_create.sql + ;; + + 'db.drop') + execute "$2" ddl_drop.sql + ;; + + 'db.reset') + $0 db.drop "$2" + $0 db.create "$2" + ;; + + 'db.dist') + execute "$2" setup_distributed-table.sql + ;; + + 'db.shards') + execute "$2" setup_sharding.sql + ;; + + 'db.new') + $0 db.drop "$2" + $0 users.drop "$2" + + $0 db.create "$2" + $0 db.dist "$2" + $0 db.shards "$2" + + $0 users.create "$2" + ;; + + 'users.create') + execute_users "$2" users_create.sql + ;; + + 'users.drop') + execute_users "$2" users_drop.sql + ;; + + 'users.reset') + $0 users.drop "$2" + $0 users.create "$2" + ;; + + *) + echo + echo "Usage: $0 options" + echo "where option are" + echo "- {db.create | db.drop | db.reset | db.dist | db.shards} " + echo "- {users.create | users.drop | users.reset} " + echo + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/dbtest/setup_distributed-table.sql b/script-db/citusdb/dbtest/setup_distributed-table.sql new file mode 100755 index 0000000..46ad105 --- /dev/null +++ b/script-db/citusdb/dbtest/setup_distributed-table.sql @@ -0,0 +1,2 @@ +--select master_create_distributed_table('mytable', 'id', 'hash'); +select master_create_distributed_table('mytable', 'insert_time', 'hash'); diff --git a/script-db/citusdb/dbtest/setup_distributed.sql b/script-db/citusdb/dbtest/setup_distributed.sql new file mode 100644 index 0000000..46ad105 --- /dev/null +++ b/script-db/citusdb/dbtest/setup_distributed.sql @@ -0,0 +1,2 @@ +--select master_create_distributed_table('mytable', 'id', 'hash'); +select master_create_distributed_table('mytable', 'insert_time', 'hash'); diff --git a/script-db/citusdb/dbtest/setup_sharding.sql b/script-db/citusdb/dbtest/setup_sharding.sql new file mode 100755 index 0000000..6abe2ac --- /dev/null +++ b/script-db/citusdb/dbtest/setup_sharding.sql @@ -0,0 +1,2 @@ +set citus.shard_replication_factor = 1; +select master_create_worker_shards('mytable', 3, 1); \ No newline at end of file diff --git a/script-db/citusdb/dbtest/users_create.sql b/script-db/citusdb/dbtest/users_create.sql new file mode 100755 index 0000000..3b5f055 --- /dev/null +++ b/script-db/citusdb/dbtest/users_create.sql @@ -0,0 +1,7 @@ +create user testuser with password 'testpwd'; + +grant all privileges on all tables in schema public to testuser; + +grant usage, select on all sequences in schema public to testuser; + +grant all privileges on database dbtest to testuser; diff --git a/script-db/citusdb/dbtest/users_drop.sql b/script-db/citusdb/dbtest/users_drop.sql new file mode 100755 index 0000000..df70941 --- /dev/null +++ b/script-db/citusdb/dbtest/users_drop.sql @@ -0,0 +1,2 @@ +drop owned by testuser; +drop user if exists testuser; \ No newline at end of file diff --git a/script-os/linux/dev/es.sh b/script-os/linux/dev/es.sh new file mode 100755 index 0000000..25a8286 --- /dev/null +++ b/script-os/linux/dev/es.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +APP_PROG=es +APP_HOME="/opt/$APP_PROG" +APP_NODE=$(hostname) + +APP_PID="$APP_HOME/$APP_PROG.pid" + +APP_DATA="/data/$APP_PROG" + +function set_process() { + NODE_NAME=$1 + PID_FILE=$2 + + if [ $? -eq 0 ]; + then + echo -n "$APP_PROG: $NODE_NAME ... " + sleep 5 + + if [ -f "$PID_FILE" ]; + then + echo "STARTED" + else + echo "failed to write to $PID_FILE" + exit 1 + fi + + else + echo "$APP_PROG is not running..." + exit 1 + fi +} + +function start_process() { + NODE_NAME=$1 + PID_FILE=$2 + + $APP_HOME/bin/elasticsearch -d -p $PID_FILE + set_process $NODE_NAME $PID_FILE +} + +function stop_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ]; + then + echo "no $APP_PROG to stop (could not find file $PID_FILE)" + else + kill $(cat "$PID_FILE") + sleep 1 + echo "STOPPED" + + if [ -f "$PID_FILE" ]; + then + /bin/rm "$PID_FILE" + fi + fi +} + +function check_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ]; + then + echo "NOT RUNNING" + else + echo "RUNNING" + fi +} + +case "$1" in + 'start') + start_process $APP_NODE $APP_PID + ;; + + 'stop') + stop_process $APP_NODE $APP_PID + ;; + + 'status') + check_process $APP_NODE $APP_PID + ;; + + *) + echo "Usage: $0 { start | stop | status }" + +esac +exit 0 \ No newline at end of file diff --git a/script-os/linux/dev/kibana.sh b/script-os/linux/dev/kibana.sh new file mode 100755 index 0000000..9c78f34 --- /dev/null +++ b/script-os/linux/dev/kibana.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +APP_PROG=kibana + +APP_HOME="/opt/$APP_PROG" +APP_NODE=$(hostname) + +APP_PID="$APP_HOME/$APP_PROG.pid" + +APP_DATA="/data/$APP_PROG" +APP_LOG="$APP_DATA/logs/$APP_PROG.log" + +function set_process() { + NODE_NAME=$1 + PID_FILE=$2 + + if [ $? -eq 0 ] + then + echo -n "$APP_PROG: $NODE_NAME ... " + if /bin/echo -n $! > $PID_FILE + then + sleep 1 + echo "STARTED" + else + echo "failed to write to $PID_FILE" + exit 1 + fi + else + echo "$APP_PROG is not running..." + exit 1 + fi +} + +function start_process() { + NODE_NAME=$1 + PID_FILE=$2 + + nohup $APP_HOME/bin/kibana -l $APP_LOG > /dev/null 2>&1 & + set_process $NODE_NAME $PID_FILE +} + +function stop_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ] + then + echo "no $APP_PROG to stop (could not find file $PID_FILE)" + else + kill $(cat "$PID_FILE") + sleep 1 + echo "STOPPED" + + if [ -f "$PID_FILE" ] + then + rm "$PID_FILE" + fi + fi +} + +function check_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ] + then + echo "NOT RUNNING" + else + echo "RUNNING" + fi +} + +case "$1" in + 'start') + start_process $APP_NODE $APP_PID + ;; + + 'stop') + stop_process $APP_NODE $APP_PID + ;; + + 'status') + check_process $APP_NODE $APP_PID + ;; + + *) + echo "Usage: $0 { start | stop | status }" + +esac +exit 0 diff --git a/script-os/linux/sys/commons.sh b/script-os/linux/sys/commons.sh new file mode 100644 index 0000000..412eadd --- /dev/null +++ b/script-os/linux/sys/commons.sh @@ -0,0 +1,11 @@ +function random() { + MAX=$1 + : ${MAX:=15} +} + +function loop() { + for index in $(seq 1 $COUNT); do + echo "index: ${index}" + done +} + diff --git a/script-os/linux/sys/remote-cpi.sh b/script-os/linux/sys/remote-cpi.sh index 99a2d06..631b28a 100755 --- a/script-os/linux/sys/remote-cpi.sh +++ b/script-os/linux/sys/remote-cpi.sh @@ -1,7 +1,6 @@ #!/bin/bash -function copy_ex() -{ +function copy_ex() { HOSTS="$1" PEM_FILE="$2" USER_NAME="$3" diff --git a/script-os/linux/sys/remote-exi.sh b/script-os/linux/sys/remote-exi.sh index 206e0ac..1794fbe 100755 --- a/script-os/linux/sys/remote-exi.sh +++ b/script-os/linux/sys/remote-exi.sh @@ -1,7 +1,6 @@ #!/bin/bash -function remote_ex() -{ +function remote_ex() { HOSTS="$1" PEM_FILE="$2" USER_NAME="$3" diff --git a/script-os/linux/tools/mu.sh b/script-os/linux/tools/mu.sh deleted file mode 100755 index 9123b10..0000000 --- a/script-os/linux/tools/mu.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -java -jar $HOME/tools/mucommander.jar & - From 37e5d47eb270c9b1fc978a4d17b0a3bf51469dd1 Mon Sep 17 00:00:00 2001 From: "Tri H. Nguyen" Date: Sun, 14 Mar 2021 16:51:04 -0400 Subject: [PATCH 6/8] updated script-db --- script-db/mysql/{dbmysql.cmd => db-mysql.cmd} | 0 script-db/mysql/dbtest/populate.sql | 20 +-- script-db/mysql/howto-windows.txt | 12 +- script-db/postgresql/bin/db-psql.sh | 44 ++++++ script-os/linux/dev/kafka.sh | 136 ++++++++++++++++++ 5 files changed, 196 insertions(+), 16 deletions(-) rename script-db/mysql/{dbmysql.cmd => db-mysql.cmd} (100%) create mode 100755 script-db/postgresql/bin/db-psql.sh create mode 100755 script-os/linux/dev/kafka.sh diff --git a/script-db/mysql/dbmysql.cmd b/script-db/mysql/db-mysql.cmd similarity index 100% rename from script-db/mysql/dbmysql.cmd rename to script-db/mysql/db-mysql.cmd diff --git a/script-db/mysql/dbtest/populate.sql b/script-db/mysql/dbtest/populate.sql index e96f9a7..2bb682e 100644 --- a/script-db/mysql/dbtest/populate.sql +++ b/script-db/mysql/dbtest/populate.sql @@ -5,34 +5,34 @@ delete from Person; -- Person insert into Person (PersonName, Email, Phone, Status) - values ('John Smith', 'john.smith@smith.com', '703-555-1212', 1); + values ('John Smith', 'john.smith@smith.com', '703-555-1212', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Jones Smith', 'jones.smith@smith.com', '703-555-1213', 1); + values ('Jones Smith', 'jones.smith@smith.com', '703-555-1213', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Abu Okolada', 'abu.okolada@abu.com', '707-555-1214', 1); + values ('Abu Okolada', 'abu.okolada@abu.com', '707-555-1214', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Noah Lock', 'noah.lock@lock.com', '890-555-1212', 1); + values ('Noah Lock', 'noah.lock@lock.com', '890-555-1212', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Zico Tacoram Alcoduma', 'zico.alco@alco.com', '450-555-1212', 1); + values ('Zico Tacoram Alcoduma', 'zico.alco@alco.com', '450-555-1212', 1); -- Address insert into Address (PersonID, Street, City, Country, Status) - values (1, '239 Luthkia Blvd', 'Buckingham', 'Hungary', 1); + values (1, '239 Luthkia Blvd', 'Buckingham', 'Hungary', 1); insert into Address (PersonID, Street, City, Country, Status) - values (2, '8 Listco Ave', 'Colide, UT', 'US', 1); + values (2, '8 Listco Ave', 'Colide, UT', 'US', 1); insert into Address (PersonID, Street, City, Country, Status) - values (1, '3901 Cocacola Dr', 'Albama', 'Norway', 0); + values (1, '3901 Cocacola Dr', 'Albama', 'Norway', 0); insert into Address (PersonID, Street, City, Country, Status) - values (3, '32 Bolacia Blve', 'Knox, TX', 'US', 1); + values (3, '32 Bolacia Blve', 'Knox, TX', 'US', 1); insert into Address (PersonID, Street, City, Country, Status) - values (4, '54/2 Tran Van Khon', 'Bac Lieu, Q4', 'Vietnam', 1); + values (4, '54/2 Tran Van Khon', 'Bac Lieu, Q4', 'Vietnam', 1); commit; diff --git a/script-db/mysql/howto-windows.txt b/script-db/mysql/howto-windows.txt index 6de3bce..fbffda4 100644 --- a/script-db/mysql/howto-windows.txt +++ b/script-db/mysql/howto-windows.txt @@ -4,26 +4,26 @@ # assumptions - MYSQL database server is installed at c:\database\mysql - If not, adjust the database path in dbmysql.cmd + If not, adjust the database path in db-mysql.cmd # start Launch the DOS windows or console window, at the prompt: # set up environmental variables and system path for mysql -dbmysql.cmd +db-mysql.cmd # install mysql as a windows service # for windows 7, launch the console as admin first -dbmysql.cmd service.install +db-mysql.cmd service.install # uninstall mysql windows service -dbmysql.cmd service.uninstall +db-mysql.cmd service.uninstall # start mysql windows service -dbmysql.cmd service.start +db-mysql.cmd service.start # stop mysql windows service -dbmysql.cmd service.stop +db-mysql.cmd service.stop #------------------------------------------------------------------------------- # local admin account diff --git a/script-db/postgresql/bin/db-psql.sh b/script-db/postgresql/bin/db-psql.sh new file mode 100755 index 0000000..f35e4a4 --- /dev/null +++ b/script-db/postgresql/bin/db-psql.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +DB_HOST="localhost" +DB_PORT=5432 +DB_USER="postgres" +DB_PWD="" +DB_NAME="" + +case "$1" in + 'platforms.ssh') + DB_HOST="localhost" + DB_PORT=19700 + DB_NAME="platforms" + DB_USER="postgres_admin" + + psql --dbname="${DB_NAME}" --host="${DB_HOST}" --port=${DB_PORT} --username="${DB_USER}" --password + ;; + + 'platforms.ssh.import') + DB_HOST="localhost" + DB_PORT=19700 + DB_NAME="platforms" + DB_USER="postgres_admin" + + psql --dbname="${DB_NAME}" --host="${DB_HOST}" --port=${DB_PORT} --username="${DB_USER}" --password < $2 + ;; + + 'psql.local.platforms') + DB_NAME="platforms" + + psql --dbname="${DB_NAME}" + ;; + + 'psql.local') + DB_NAME="postgres" + + psql --dbname="${DB_NAME}" + ;; + + *) + echo "Usage: $0 { psql.local | psql.local.platforms | platforms | platforms.ssh | platforms.ssh.import }" + +esac +exit 0 diff --git a/script-os/linux/dev/kafka.sh b/script-os/linux/dev/kafka.sh new file mode 100755 index 0000000..bfc37db --- /dev/null +++ b/script-os/linux/dev/kafka.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +CWD=$(cd "$(dirname "$0")" && pwd) + +APP_HOME=/opt/kafka +APP_BIN=$APP_HOME/bin +APP_CONFIG=$APP_HOME/config + +APP_DATA=/data/kafka + +ZK_CONNECT=flax:2181 +KAFKA_SERVER=flax:9092 + +case "$1" in + 'zk.start') + $APP_BIN/zookeeper-server-start.sh $APP_CONFIG/zookeeper.properties & + ;; + + 'zk.stop') + $APP_BIN/zookeeper-server-stop.sh $APP_CONFIG/zookeeper.properties & + ;; + + 'k.start') + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server.properties & + ;; + + 'k.stop') + $APP_BIN/kafka-server-stop.sh $APP_CONFIG/server.properties & + ;; + + 'k.clear') + rm -rf $APP_DATA/klogs/* + rm -rf $APP_DATA/zk/{data,logs}/* + ;; + + 'k.producer') + $APP_BIN/kafka-console-producer.sh --broker-list $KAFKA_SERVER --topic $2 + ;; + + 'k.consumer') + $APP_BIN/kafka-console-consumer.sh --zookeeper $ZK_CONNECT --from-beginning --topic $2 + ;; + + 'k.configs.topics') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --describe --entity-type topics + ;; + + 'k.configs.clients') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --describe --entity-type clients + ;; + + 'kb.start') + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-0.properties & + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-1.properties & + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-2.properties & + ;; + + 'kb.clear') + rm -rf $APP_DATA/{klogs-0,klogs-1,klogs-2}/* + rm -rf $APP_DATA/zk/{data,logs}/* + ;; + + 'kb.topic.create') + PARTITIONS=$3 + if [ -z "$3" ]; then + PARTITIONS=1 + fi + + REPLICATION_FACTOR=$4 + if [ -z "$4" ]; then + REPLICATION_FACTOR=3 + fi + $APP_BIN/kafka-topics.sh --create --zookeeper $ZK_CONNECT --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $2 + ;; + + 'topic.create') + PARTITIONS=$3 + if [ -z "$3" ]; then + PARTITIONS=1 + fi + + REPLICATION_FACTOR=$4 + if [ -z "$4" ]; then + REPLICATION_FACTOR=1 + fi + + $APP_BIN/kafka-topics.sh --create --zookeeper $ZK_CONNECT --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $2 + ;; + + 'topic.delete') + $APP_BIN/kafka-topics.sh --delete --zookeeper $ZK_CONNECT --topic $2 + ;; + + 'topic.list') + $APP_BIN/kafka-topics.sh --list --zookeeper $ZK_CONNECT + ;; + + 'topic.purge') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --alter --entity-type topics --entity-name $2 --add-config retention.ms=$3 + ;; + + 'topic.purge.reset') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --alter --entity-type topics --entity-name $2 --delete-config retention.ms + ;; + + 'topic.describe') + $APP_BIN/kafka-topics.sh --describe --zookeeper $ZK_CONNECT --topic $2 + ;; + + *) + echo "" + echo "Usage: $0 options" + echo "where options are:" + echo "- zk.start - stop zookeeper" + echo "- zk.stop - stop zookeeper" + echo "" + echo "- k.start - start kafka server" + echo "- k.stop - stop kafka server" + echo "- k.clear - reset kafka server" + echo "- k.producer - launch kafka producer" + echo "- k.consumer - launch kafka consumer" + echo "" + echo "- kb.start - start multi-broker kafka servers" + echo "- kb.clear - reset multi-broker kafka servers" + echo "- kb.topic.create [number of partitions (1)] [replication factor (3)]- create a topic in multi-broker environment" + echo "" + echo "- topic.describe - create a topic" + echo "- topic.delete - delete a topic" + echo "- topic.create [number of partitions (1)] [replication factor (1)] - create a topic" + echo "- topic.purge