diff --git a/README.md b/README.md index 935bd82..33d298e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ - script-db - script-elasticsearch - script-hadoop +- script-java - script-os +- script-python - script-solr - script-spark - script-zookeeper diff --git a/script-aws/bin/s3api.sh b/script-aws/bin/s3api.sh index 8aa8dca..48fca1f 100755 --- a/script-aws/bin/s3api.sh +++ b/script-aws/bin/s3api.sh @@ -12,52 +12,52 @@ function s3_list() { } function s3_delete_regions() { - STAGE=$1 - PARTY_ID=$2 - TYPE=$3 - shift - shift - shift - REGIONS=("$@") - - for REGION in ${REGIONS[@]}; do - s3_delete ${STAGE} ${PARTY_ID} ${TYPE} ${REGION} - done + STAGE=$1 + PARTY_ID=$2 + TYPE=$3 + shift + shift + shift + REGIONS=("$@") + + for REGION in ${REGIONS[@]}; do + s3_delete ${STAGE} ${PARTY_ID} ${TYPE} ${REGION} + done } function s3_delete() { - PREFIX=$1 - REGION=$2 + PREFIX=$1 + REGION=$2 - BUCKET_NAME="${PREFIX}-${REGION}" - echo "bucket: ${BUCKET_NAME}" - echo + BUCKET_NAME="${PREFIX}-${REGION}" + echo "bucket: ${BUCKET_NAME}" + echo - aws s3api delete-bucket --bucket ${BUCKET_NAME} \ - --region ${REGION} + aws s3api delete-bucket --bucket ${BUCKET_NAME} \ + --region ${REGION} } function s3_create_regions() { - PREFIX=$1 - shift - REGIONS=("$@") - - for REGION in ${REGIONS[@]}; do - echo "# creating: ${PREFIX}-${REGION}" - s3_create ${PREFIX} ${REGION} - done + PREFIX=$1 + shift + REGIONS=("$@") + + for REGION in ${REGIONS[@]}; do + echo "# creating: ${PREFIX}-${REGION}" + s3_create ${PREFIX} ${REGION} + done } function s3_create() { - PREFIX=$1 - REGION=$2 + PREFIX=$1 + REGION=$2 - BUCKET_NAME="${PREFIX}-${REGION}" - echo "- bucket: ${BUCKET_NAME}" + BUCKET_NAME="${PREFIX}-${REGION}" + echo "- bucket: ${BUCKET_NAME}" - aws s3api create-bucket --bucket ${BUCKET_NAME} \ - --region ${REGION} \ - --create-bucket-configuration LocationConstraint=${REGION} + aws s3api create-bucket --bucket ${BUCKET_NAME} \ + --region ${REGION} \ + --create-bucket-configuration LocationConstraint=${REGION} } case "$1" in diff --git a/script-db/citusdb/README.md b/script-db/citusdb/README.md new file mode 100644 index 0000000..be98ef4 --- /dev/null +++ b/script-db/citusdb/README.md @@ -0,0 +1,5 @@ +CitusDB +======= + +# References +- CitusDB v5.1 [Documentation] (https://docs.citusdata.com/en/v5.1/) | [Single-Machine Cluster] (https://docs.citusdata.com/en/v5.1/installation/development.html) | [Multi-Machine Cluster] (https://docs.citusdata.com/en/v5.1/installation/production.html) diff --git a/script-db/citusdb/bin/README.md b/script-db/citusdb/bin/README.md new file mode 100644 index 0000000..8d8dd3b --- /dev/null +++ b/script-db/citusdb/bin/README.md @@ -0,0 +1,8 @@ +CitusDB Setup scripts +===================== + +- citus.sh # to setup single-machine cluster + +- citus_multi.sh # to setup custom multi-machine cluster + +- citus_multi-pg.sh # to setup multi-machine cluster using default installation \ No newline at end of file diff --git a/script-db/citusdb/bin/citus.sh b/script-db/citusdb/bin/citus.sh new file mode 100755 index 0000000..086b14a --- /dev/null +++ b/script-db/citusdb/bin/citus.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +APP_DATA="/data/citus" +APP_LOG="$APP_DATA/log" + +function init_db() { + echo -e "\n# initializing the installation" + + mkdir -p $APP_DATA/{master,worker1,worker2} + mkdir -p $APP_LOG + + # create three normal postgres instances + initdb -D $APP_DATA/master + initdb -D $APP_DATA/worker1 + initdb -D $APP_DATA/worker2 + + # set workers in master + echo "localhost 9701" >> $APP_DATA/master/pg_worker_list.conf + echo "localhost 9702" >> $APP_DATA/master/pg_worker_list.conf + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/master/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/worker1/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/worker2/postgresql.conf +} + +function start_db() { + echo -e "\n# starting master instance" + pg_ctl -D $APP_DATA/master -o "-p 9700" -l $APP_LOG/master.log start + + echo -e "\n# starting worker1 instance" + pg_ctl -D $APP_DATA/worker1 -o "-p 9701" -l $APP_LOG/worker1.log start + + echo -e "\n# starting worker2 instance" + pg_ctl -D $APP_DATA/worker2 -o "-p 9702" -l $APP_LOG/worker2.log start +} + +function stop_db() { + echo -e "\n# stopping worker1 instance" + pg_ctl -D $APP_DATA/worker1 -m fast stop + + echo -e "\n# stopping worker2 instance" + pg_ctl -D $APP_DATA/worker2 -m fast stop + + echo -e "\n# stopping master instance" + pg_ctl -D $APP_DATA/master -m fast stop +} + +function status_db() { + echo -e "\n# worker1 instance's status" + pg_ctl -D $APP_DATA/worker1 status + + echo -e "\n# worker2 instance's status" + pg_ctl -D $APP_DATA/worker2 status + + echo -e "\n# master instance's status" + pg_ctl -D $APP_DATA/master status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + createdb -p 9700 $DB_NAME + createdb -p 9701 $DB_NAME + createdb -p 9702 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + dropdb -p 9700 $DB_NAME + dropdb -p 9701 $DB_NAME + dropdb -p 9702 $DB_NAME +} + +function load_ext() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" + psql -p 9701 -d $DB_NAME -c "CREATE EXTENSION citus;" + psql -p 9702 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_ext "$2" + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 (init | {create | drop | verify } | start | stop | status | clear)" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/bin/citus_multi-pg.sh b/script-db/citusdb/bin/citus_multi-pg.sh new file mode 100755 index 0000000..8b1956d --- /dev/null +++ b/script-db/citusdb/bin/citus_multi-pg.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +BASE_DIR="/var/lib/pgsql/9.5" +APP_DATA="$BASE_DIR/data" +APP_LOGS="$BASE_DIR/logs" + +PG_HOME="/usr/pgsql-" +PG_BIN="$PG_HOME/bin" + +function init_db() { + echo -e "\n# initializing the installation" + mkdir -p $APP_DATA + mkdir -p $APP_LOGS + + # create normal postgres instance + sudo service postgresql-9.5 initdb || sudo $PG_BIN/postgresql95-setup initdb + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" | sudo tee -a $APP_DATA/postgresql.conf +} + +function start_db() { + echo -e "\n# starting postgresql instance" + sudo service postgresql-9.5 restart +} + +function stop_db() { + echo -e "\n# stopping postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -m fast stop +} + +function status_db() { + echo -e "\n# postgresql instance's status" + $PG_BIN/pg_ctl -D $APP_DATA status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + $PG_BIN/createdb -p 9700 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + $PG_BIN/dropdb -p 9700 $DB_NAME +} + +function load_db() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* + rm -rf $APP_LOGS/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_db "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 {init | {create | drop } | start | stop | status | clear}" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/bin/citus_multi.sh b/script-db/citusdb/bin/citus_multi.sh new file mode 100755 index 0000000..3a44cca --- /dev/null +++ b/script-db/citusdb/bin/citus_multi.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/ +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +BASE_DIR="/data/citus" +APP_DATA="$BASE_DIR/data" +APP_LOGS="$BASE_DIR/logs" + +PG_HOME="/usr/pgsql-" +PG_BIN="$PG_HOME/bin" + +function init_db() { + echo -e "\n# initializing the installation" + mkdir -p $APP_DATA + mkdir -p $APP_LOGS + + # create normal postgres instance + $PG_BIN/initdb -D $APP_DATA + + # set a configuration variable + echo "shared_preload_libraries = 'citus'" >> $APP_DATA/postgresql.conf +} + +function start_db() { + echo -e "\n# starting postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -o "-p 9700" -l $APP_LOGS/citus.log start +} + +function stop_db() { + echo -e "\n# stopping postgresql instance" + $PG_BIN/pg_ctl -D $APP_DATA -m fast stop +} + +function status_db() { + echo -e "\n# postgresql instance's status" + $PG_BIN/pg_ctl -D $APP_DATA status +} + +function create_db() { + DB_NAME="$1" + echo -e "\n# creating database: $DB_NAME" + + $PG_BIN/createdb -p 9700 $DB_NAME +} + +function drop_db() { + DB_NAME="$1" + echo -e "\n# dropping database: $DB_NAME" + + $PG_BIN/dropdb -p 9700 $DB_NAME +} + +function load_ext() { + DB_NAME="$1" + echo -e "\n# loading the extension" + + psql -p 9700 -d $DB_NAME -c "CREATE EXTENSION citus;" +} + +function verify_db() { + DB_NAME="$1" + echo -e "\n# verifying the number of active worker nodes" + + psql -p 9700 -d $DB_NAME -c "select * from master_get_active_worker_nodes();" +} + +function clear_db() { + stop_db + + echo -e "\n# clearing the installation" + rm -rf $APP_DATA/* + rm -rf $APP_LOGS/* +} + +case "$1" in + 'init') + init_db + start_db + ;; + + 'create') + create_db "$2" + load_ext "$2" + ;; + + 'verify') + verify_db "$2" + ;; + + 'drop') + drop_db "$2" + ;; + + 'start') + start_db + ;; + + 'stop') + stop_db + ;; + + 'status') + status_db + ;; + + 'clear') + clear_db + ;; + + *) + echo "Usage: $0 {init | {create | drop | verify } | start | stop | status | clear}" + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/dbtest/README.md b/script-db/citusdb/dbtest/README.md new file mode 100644 index 0000000..9b5f88a --- /dev/null +++ b/script-db/citusdb/dbtest/README.md @@ -0,0 +1,10 @@ +Schema Scripts +============== + +Contains scripts to create a simple schema and a user in dbtest database + +# Instructions + +- run "setup.sh" then use appropriate parameters + + diff --git a/script-db/citusdb/dbtest/ddl_create.sql b/script-db/citusdb/dbtest/ddl_create.sql new file mode 100755 index 0000000..d6d7012 --- /dev/null +++ b/script-db/citusdb/dbtest/ddl_create.sql @@ -0,0 +1,5 @@ +create table mytable ( + id char(36) not null, + name char(64) not null, + insert_time timestamp not null +); diff --git a/script-db/citusdb/dbtest/ddl_drop.sql b/script-db/citusdb/dbtest/ddl_drop.sql new file mode 100755 index 0000000..c7fd3a8 --- /dev/null +++ b/script-db/citusdb/dbtest/ddl_drop.sql @@ -0,0 +1,2 @@ +drop table if exists mytable cascade; + diff --git a/script-db/citusdb/dbtest/setup.sh b/script-db/citusdb/dbtest/setup.sh new file mode 100644 index 0000000..550aa2e --- /dev/null +++ b/script-db/citusdb/dbtest/setup.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# CITUSDB_HOME=/usr/local/Cellar/postgresql/9.5.3 +# CITUSDB_BIN=$CITUSDB_HOME/bin + +CWD=$(cd "$(dirname "$0")" && pwd) + +function execute() { + DB_NAME="$1" + DB_SCRIPT="$2" + + psql -p 9700 -d $DB_NAME < $DB_SCRIPT +} + +function execute_users() { + DB_NAME="$1" + DB_SCRIPT="$2" + + psql -p 9700 -d $DB_NAME < $DB_SCRIPT + psql -p 9701 -d $DB_NAME < $DB_SCRIPT + psql -p 9702 -d $DB_NAME < $DB_SCRIPT +} + +case "$1" in + 'db.create') + execute "$2" ddl_create.sql + ;; + + 'db.drop') + execute "$2" ddl_drop.sql + ;; + + 'db.reset') + $0 db.drop "$2" + $0 db.create "$2" + ;; + + 'db.dist') + execute "$2" setup_distributed-table.sql + ;; + + 'db.shards') + execute "$2" setup_sharding.sql + ;; + + 'db.new') + $0 db.drop "$2" + $0 users.drop "$2" + + $0 db.create "$2" + $0 db.dist "$2" + $0 db.shards "$2" + + $0 users.create "$2" + ;; + + 'users.create') + execute_users "$2" users_create.sql + ;; + + 'users.drop') + execute_users "$2" users_drop.sql + ;; + + 'users.reset') + $0 users.drop "$2" + $0 users.create "$2" + ;; + + *) + echo + echo "Usage: $0 options" + echo "where option are" + echo "- {db.create | db.drop | db.reset | db.dist | db.shards} " + echo "- {users.create | users.drop | users.reset} " + echo + +esac +exit 0 \ No newline at end of file diff --git a/script-db/citusdb/dbtest/setup_distributed-table.sql b/script-db/citusdb/dbtest/setup_distributed-table.sql new file mode 100755 index 0000000..46ad105 --- /dev/null +++ b/script-db/citusdb/dbtest/setup_distributed-table.sql @@ -0,0 +1,2 @@ +--select master_create_distributed_table('mytable', 'id', 'hash'); +select master_create_distributed_table('mytable', 'insert_time', 'hash'); diff --git a/script-db/citusdb/dbtest/setup_distributed.sql b/script-db/citusdb/dbtest/setup_distributed.sql new file mode 100644 index 0000000..46ad105 --- /dev/null +++ b/script-db/citusdb/dbtest/setup_distributed.sql @@ -0,0 +1,2 @@ +--select master_create_distributed_table('mytable', 'id', 'hash'); +select master_create_distributed_table('mytable', 'insert_time', 'hash'); diff --git a/script-db/citusdb/dbtest/setup_sharding.sql b/script-db/citusdb/dbtest/setup_sharding.sql new file mode 100755 index 0000000..6abe2ac --- /dev/null +++ b/script-db/citusdb/dbtest/setup_sharding.sql @@ -0,0 +1,2 @@ +set citus.shard_replication_factor = 1; +select master_create_worker_shards('mytable', 3, 1); \ No newline at end of file diff --git a/script-db/citusdb/dbtest/users_create.sql b/script-db/citusdb/dbtest/users_create.sql new file mode 100755 index 0000000..3b5f055 --- /dev/null +++ b/script-db/citusdb/dbtest/users_create.sql @@ -0,0 +1,7 @@ +create user testuser with password 'testpwd'; + +grant all privileges on all tables in schema public to testuser; + +grant usage, select on all sequences in schema public to testuser; + +grant all privileges on database dbtest to testuser; diff --git a/script-db/citusdb/dbtest/users_drop.sql b/script-db/citusdb/dbtest/users_drop.sql new file mode 100755 index 0000000..df70941 --- /dev/null +++ b/script-db/citusdb/dbtest/users_drop.sql @@ -0,0 +1,2 @@ +drop owned by testuser; +drop user if exists testuser; \ No newline at end of file diff --git a/script-db/mysql/dbmysql.cmd b/script-db/mysql/db-mysql.cmd similarity index 100% rename from script-db/mysql/dbmysql.cmd rename to script-db/mysql/db-mysql.cmd diff --git a/script-db/mysql/dbtest/populate.sql b/script-db/mysql/dbtest/populate.sql index e96f9a7..2bb682e 100644 --- a/script-db/mysql/dbtest/populate.sql +++ b/script-db/mysql/dbtest/populate.sql @@ -5,34 +5,34 @@ delete from Person; -- Person insert into Person (PersonName, Email, Phone, Status) - values ('John Smith', 'john.smith@smith.com', '703-555-1212', 1); + values ('John Smith', 'john.smith@smith.com', '703-555-1212', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Jones Smith', 'jones.smith@smith.com', '703-555-1213', 1); + values ('Jones Smith', 'jones.smith@smith.com', '703-555-1213', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Abu Okolada', 'abu.okolada@abu.com', '707-555-1214', 1); + values ('Abu Okolada', 'abu.okolada@abu.com', '707-555-1214', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Noah Lock', 'noah.lock@lock.com', '890-555-1212', 1); + values ('Noah Lock', 'noah.lock@lock.com', '890-555-1212', 1); insert into Person (PersonName, Email, Phone, Status) - values ('Zico Tacoram Alcoduma', 'zico.alco@alco.com', '450-555-1212', 1); + values ('Zico Tacoram Alcoduma', 'zico.alco@alco.com', '450-555-1212', 1); -- Address insert into Address (PersonID, Street, City, Country, Status) - values (1, '239 Luthkia Blvd', 'Buckingham', 'Hungary', 1); + values (1, '239 Luthkia Blvd', 'Buckingham', 'Hungary', 1); insert into Address (PersonID, Street, City, Country, Status) - values (2, '8 Listco Ave', 'Colide, UT', 'US', 1); + values (2, '8 Listco Ave', 'Colide, UT', 'US', 1); insert into Address (PersonID, Street, City, Country, Status) - values (1, '3901 Cocacola Dr', 'Albama', 'Norway', 0); + values (1, '3901 Cocacola Dr', 'Albama', 'Norway', 0); insert into Address (PersonID, Street, City, Country, Status) - values (3, '32 Bolacia Blve', 'Knox, TX', 'US', 1); + values (3, '32 Bolacia Blve', 'Knox, TX', 'US', 1); insert into Address (PersonID, Street, City, Country, Status) - values (4, '54/2 Tran Van Khon', 'Bac Lieu, Q4', 'Vietnam', 1); + values (4, '54/2 Tran Van Khon', 'Bac Lieu, Q4', 'Vietnam', 1); commit; diff --git a/script-db/mysql/howto-windows.txt b/script-db/mysql/howto-windows.txt index 6de3bce..fbffda4 100644 --- a/script-db/mysql/howto-windows.txt +++ b/script-db/mysql/howto-windows.txt @@ -4,26 +4,26 @@ # assumptions - MYSQL database server is installed at c:\database\mysql - If not, adjust the database path in dbmysql.cmd + If not, adjust the database path in db-mysql.cmd # start Launch the DOS windows or console window, at the prompt: # set up environmental variables and system path for mysql -dbmysql.cmd +db-mysql.cmd # install mysql as a windows service # for windows 7, launch the console as admin first -dbmysql.cmd service.install +db-mysql.cmd service.install # uninstall mysql windows service -dbmysql.cmd service.uninstall +db-mysql.cmd service.uninstall # start mysql windows service -dbmysql.cmd service.start +db-mysql.cmd service.start # stop mysql windows service -dbmysql.cmd service.stop +db-mysql.cmd service.stop #------------------------------------------------------------------------------- # local admin account diff --git a/script-db/postgresql/bin/db-psql.sh b/script-db/postgresql/bin/db-psql.sh new file mode 100755 index 0000000..f35e4a4 --- /dev/null +++ b/script-db/postgresql/bin/db-psql.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +DB_HOST="localhost" +DB_PORT=5432 +DB_USER="postgres" +DB_PWD="" +DB_NAME="" + +case "$1" in + 'platforms.ssh') + DB_HOST="localhost" + DB_PORT=19700 + DB_NAME="platforms" + DB_USER="postgres_admin" + + psql --dbname="${DB_NAME}" --host="${DB_HOST}" --port=${DB_PORT} --username="${DB_USER}" --password + ;; + + 'platforms.ssh.import') + DB_HOST="localhost" + DB_PORT=19700 + DB_NAME="platforms" + DB_USER="postgres_admin" + + psql --dbname="${DB_NAME}" --host="${DB_HOST}" --port=${DB_PORT} --username="${DB_USER}" --password < $2 + ;; + + 'psql.local.platforms') + DB_NAME="platforms" + + psql --dbname="${DB_NAME}" + ;; + + 'psql.local') + DB_NAME="postgres" + + psql --dbname="${DB_NAME}" + ;; + + *) + echo "Usage: $0 { psql.local | psql.local.platforms | platforms | platforms.ssh | platforms.ssh.import }" + +esac +exit 0 diff --git a/script-hadoop/README.md b/script-hadoop/README.md index d39dbef..9040758 100644 --- a/script-hadoop/README.md +++ b/script-hadoop/README.md @@ -1,16 +1,26 @@ This project contains script and configuration files for setting up a stand-alone or single node Hadoop Cluster for development and testing. # References -- Hadoop r1.2.1 [Documentation] (http://hadoop.apache.org/docs/r1.2.1/) | [Single-Node Setup] (http://hadoop.apache.org/docs/r1.2.1/single_node_setup.html) | [Cluster Setup] (http://hadoop.apache.org/docs/r1.2.1/cluster_setup.html) +- All [Versions](http://hadoop.apache.org/docs/) -- Hadoop r2.x [Documentation] (http://hadoop.apache.org/docs/current/) | [Single-Node Setup] (http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup] (http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ClusterSetup.html) +- Hadoop r3.x "current" [Documentation](http://hadoop.apache.org/docs/current/) | [Single-Node Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/ClusterSetup.html) -- Michael G. Knoll [Running Hadoop on Ubuntu Linux (Single-Node Cluster)] +- Hadoop r2.10.1 [Documentation](http://hadoop.apache.org/docs/r2.10.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/SingleCluster.html) | [Cluster Setup](http://hadoop.apache.org/docs/r2.10.1/hadoop-project-dist/hadoop-common/ClusterSetup.html) + +- Hadoop r1.2.1 [Documentation](http://hadoop.apache.org/docs/r1.2.1/) | [Single-Node Setup](http://hadoop.apache.org/docs/r1.2.1/single_node_setup.html) | [Cluster Setup](http://hadoop.apache.org/docs/r1.2.1/cluster_setup.html) + +- Michael G. Knoll - Running Hadoop on Ubuntu Linux (Single-Node Cluster) (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-single-node-cluster/) -- Michael G. Knoll [Running Hadoop on Ubuntu Linux (Multi-Node Cluster)] +- Michael G. Knoll - Running Hadoop on Ubuntu Linux (Multi-Node Cluster) (http://www.michael-noll.com/tutorials/running-hadoop-on-ubuntu-linux-multi-node-cluster/) +- 20 Notable Difference Between Hadoop 2.x vs Hadoop 3.x +(https://techvidvan.com/tutorials/hadoop-2-x-vs-hadoop-3-x/) + +- Difference Between Hadoop 2.x vs Hadoop 3.x +(https://www.geeksforgeeks.org/difference-between-hadoop-2-x-vs-hadoop-3-x/) + - 10 Big Differences between Hadoop 1 & Hadoop 2 (YARN) (https://acadgild.com/blog/10-big-differences-between-hadoop1-and-hadoop2/) diff --git a/script-hadoop/hadoop2/bin/sys/setup-yarn.sh b/script-hadoop/hadoop2/bin/sys/setup-yarn.sh index 1d053ad..abd9522 100755 --- a/script-hadoop/hadoop2/bin/sys/setup-yarn.sh +++ b/script-hadoop/hadoop2/bin/sys/setup-yarn.sh @@ -20,8 +20,7 @@ if [ -d "$HADOOP_ORG" ]; then cp -R $CWD/../../conf/* $HADOOP_CFG # setup data location - # HADOOP_DATA=$APP_DATA/yarn - HADOOP_DATA=$HADOOP_DATA + HADOOP_DATA=$APP_DATA/yarn mkdir -p $HADOOP_DATA/{logs,pids,local} mkdir -p $HADOOP_DATA/mapred/{local,temp,system} diff --git a/script-hadoop/hadoop2/conf/hdfs-site.xml b/script-hadoop/hadoop2/conf/hdfs-site.xml index 9798969..c2a8257 100755 --- a/script-hadoop/hadoop2/conf/hdfs-site.xml +++ b/script-hadoop/hadoop2/conf/hdfs-site.xml @@ -37,8 +37,6 @@ default: 128MB (use 256MB for large file systems) -<<<<<<< HEAD -======= dfs.permissions.superusergroup hadoop @@ -49,7 +47,6 @@ 1 ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 dfs.namenode.checkpoint.dir @@ -69,14 +66,8 @@ -<<<<<<< HEAD - dfs.permissions.superusergroup - hadoop -======= dfs.datanode.handler.count 10 default: 10 ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 - diff --git a/script-hadoop/hadoop2/conf/yarn-site.xml b/script-hadoop/hadoop2/conf/yarn-site.xml index c3440e5..8697f63 100755 --- a/script-hadoop/hadoop2/conf/yarn-site.xml +++ b/script-hadoop/hadoop2/conf/yarn-site.xml @@ -5,7 +5,6 @@ yarn.resourcemanager.hostname localhost -<<<<<<< HEAD yarn.resourcemanager.webapp.address @@ -17,8 +16,6 @@ true default: false -======= ->>>>>>> 30f044ed7464b29ceed9f8766dc164e14d2ef885 diff --git a/script-java/bin/common.sh b/script-java/bin/common.sh index ad43fb5..d8cdb70 100755 --- a/script-java/bin/common.sh +++ b/script-java/bin/common.sh @@ -7,7 +7,7 @@ start() { case "$1" in start) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then start @@ -30,14 +30,13 @@ case "$1" in ;; stop) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then echo "${SERVICE_NAME} is not running [${PID}]" rm -f $PID_FILE exit 1 else - #PID=`cat $PID_FILE` kill -term $PID echo "${SERVICE_NAME} is stopped [${PID}]" rm -f $PID_FILE @@ -50,7 +49,7 @@ case "$1" in ;; status) - if [! $PID_FILE ]; then + if [ -f $PID_FILE ]; then PID=`cat $PID_FILE` if [ -z "`ps -ef | grep ${PID} | grep -v grep`" ]; then echo "${SERVICE_NAME} is not running [${PID}]" diff --git a/script-java/bin/service.sh b/script-java/bin/service.sh index dbe867f..59f4906 100755 --- a/script-java/bin/service.sh +++ b/script-java/bin/service.sh @@ -16,7 +16,7 @@ APP_JARS=$LIB_JARS CLASSPATH=$APP_JARS:$APP_CONFIG export CLASSPATH -SERVICE_NAME="a" +SERVICE_NAME="service-a" PID_FILE="${APP_HOME}/${SERVICE_NAME}.pid" APP_OPTS="-Dlog4j.configurationFile=$APP_CONFIG/${SERVICE_NAME}.xml" APP_OPTS="$APP_OPTS -Dconfig.file=$APP_CONFIG/${SERVICE_NAME}.yml" diff --git a/script-os/linux/dev/es.sh b/script-os/linux/dev/es.sh new file mode 100755 index 0000000..25a8286 --- /dev/null +++ b/script-os/linux/dev/es.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +APP_PROG=es +APP_HOME="/opt/$APP_PROG" +APP_NODE=$(hostname) + +APP_PID="$APP_HOME/$APP_PROG.pid" + +APP_DATA="/data/$APP_PROG" + +function set_process() { + NODE_NAME=$1 + PID_FILE=$2 + + if [ $? -eq 0 ]; + then + echo -n "$APP_PROG: $NODE_NAME ... " + sleep 5 + + if [ -f "$PID_FILE" ]; + then + echo "STARTED" + else + echo "failed to write to $PID_FILE" + exit 1 + fi + + else + echo "$APP_PROG is not running..." + exit 1 + fi +} + +function start_process() { + NODE_NAME=$1 + PID_FILE=$2 + + $APP_HOME/bin/elasticsearch -d -p $PID_FILE + set_process $NODE_NAME $PID_FILE +} + +function stop_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ]; + then + echo "no $APP_PROG to stop (could not find file $PID_FILE)" + else + kill $(cat "$PID_FILE") + sleep 1 + echo "STOPPED" + + if [ -f "$PID_FILE" ]; + then + /bin/rm "$PID_FILE" + fi + fi +} + +function check_process() { + NODE_NAME=$1 + PID_FILE=$2 + + echo -n "$APP_PROG: $NODE_NAME ... " + if [ ! -f "$PID_FILE" ]; + then + echo "NOT RUNNING" + else + echo "RUNNING" + fi +} + +case "$1" in + 'start') + start_process $APP_NODE $APP_PID + ;; + + 'stop') + stop_process $APP_NODE $APP_PID + ;; + + 'status') + check_process $APP_NODE $APP_PID + ;; + + *) + echo "Usage: $0 { start | stop | status }" + +esac +exit 0 \ No newline at end of file diff --git a/script-os/linux/dev/kafka.sh b/script-os/linux/dev/kafka.sh new file mode 100755 index 0000000..bfc37db --- /dev/null +++ b/script-os/linux/dev/kafka.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +CWD=$(cd "$(dirname "$0")" && pwd) + +APP_HOME=/opt/kafka +APP_BIN=$APP_HOME/bin +APP_CONFIG=$APP_HOME/config + +APP_DATA=/data/kafka + +ZK_CONNECT=flax:2181 +KAFKA_SERVER=flax:9092 + +case "$1" in + 'zk.start') + $APP_BIN/zookeeper-server-start.sh $APP_CONFIG/zookeeper.properties & + ;; + + 'zk.stop') + $APP_BIN/zookeeper-server-stop.sh $APP_CONFIG/zookeeper.properties & + ;; + + 'k.start') + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server.properties & + ;; + + 'k.stop') + $APP_BIN/kafka-server-stop.sh $APP_CONFIG/server.properties & + ;; + + 'k.clear') + rm -rf $APP_DATA/klogs/* + rm -rf $APP_DATA/zk/{data,logs}/* + ;; + + 'k.producer') + $APP_BIN/kafka-console-producer.sh --broker-list $KAFKA_SERVER --topic $2 + ;; + + 'k.consumer') + $APP_BIN/kafka-console-consumer.sh --zookeeper $ZK_CONNECT --from-beginning --topic $2 + ;; + + 'k.configs.topics') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --describe --entity-type topics + ;; + + 'k.configs.clients') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --describe --entity-type clients + ;; + + 'kb.start') + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-0.properties & + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-1.properties & + $APP_BIN/kafka-server-start.sh $APP_CONFIG/server-2.properties & + ;; + + 'kb.clear') + rm -rf $APP_DATA/{klogs-0,klogs-1,klogs-2}/* + rm -rf $APP_DATA/zk/{data,logs}/* + ;; + + 'kb.topic.create') + PARTITIONS=$3 + if [ -z "$3" ]; then + PARTITIONS=1 + fi + + REPLICATION_FACTOR=$4 + if [ -z "$4" ]; then + REPLICATION_FACTOR=3 + fi + $APP_BIN/kafka-topics.sh --create --zookeeper $ZK_CONNECT --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $2 + ;; + + 'topic.create') + PARTITIONS=$3 + if [ -z "$3" ]; then + PARTITIONS=1 + fi + + REPLICATION_FACTOR=$4 + if [ -z "$4" ]; then + REPLICATION_FACTOR=1 + fi + + $APP_BIN/kafka-topics.sh --create --zookeeper $ZK_CONNECT --replication-factor $REPLICATION_FACTOR --partitions $PARTITIONS --topic $2 + ;; + + 'topic.delete') + $APP_BIN/kafka-topics.sh --delete --zookeeper $ZK_CONNECT --topic $2 + ;; + + 'topic.list') + $APP_BIN/kafka-topics.sh --list --zookeeper $ZK_CONNECT + ;; + + 'topic.purge') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --alter --entity-type topics --entity-name $2 --add-config retention.ms=$3 + ;; + + 'topic.purge.reset') + $APP_BIN/kafka-configs.sh --zookeeper $ZK_CONNECT --alter --entity-type topics --entity-name $2 --delete-config retention.ms + ;; + + 'topic.describe') + $APP_BIN/kafka-topics.sh --describe --zookeeper $ZK_CONNECT --topic $2 + ;; + + *) + echo "" + echo "Usage: $0 options" + echo "where options are:" + echo "- zk.start - stop zookeeper" + echo "- zk.stop - stop zookeeper" + echo "" + echo "- k.start - start kafka server" + echo "- k.stop - stop kafka server" + echo "- k.clear - reset kafka server" + echo "- k.producer - launch kafka producer" + echo "- k.consumer - launch kafka consumer" + echo "" + echo "- kb.start - start multi-broker kafka servers" + echo "- kb.clear - reset multi-broker kafka servers" + echo "- kb.topic.create [number of partitions (1)] [replication factor (3)]- create a topic in multi-broker environment" + echo "" + echo "- topic.describe - create a topic" + echo "- topic.delete - delete a topic" + echo "- topic.create [number of partitions (1)] [replication factor (1)] - create a topic" + echo "- topic.purge