From 8c0c64bff049ee2079d2d5e7bb3461469fa51381 Mon Sep 17 00:00:00 2001 From: Raymond Colebaugh Date: Mon, 22 May 2017 15:36:09 -0700 Subject: [PATCH] Added initial Vagrant environment, local fixes to x2util --- .gitignore | 4 ++ README.md | 42 +++++++++++++-- Vagrantfile | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++ x2util | 39 +++++++++----- 4 files changed, 216 insertions(+), 18 deletions(-) create mode 100644 Vagrantfile diff --git a/.gitignore b/.gitignore index 0d20b64..e92bb85 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ *.pyc +*.sw[op] +.vagrant +/X2CRM +/workdir diff --git a/README.md b/README.md index 76b8865..78f6618 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#X2CRM Developer Utilities +# X2CRM Developer Utilities ## Installation & Configuration Clone the Git repository and either add the directory to your path or copy both x2util and .x2util_rsync_exclude to a directory within @@ -31,7 +31,41 @@ Simply run "x2util [command]" where command is one of the options documented in the help text for x2util. Run "x2util -h" for more information on the command line options -## Future Plans --Add support for ignoring server-specific unit/functional testing configuration +## Developing in the Vagrant Environment +To simplify setup and configuration, a Vagrantfile has been provided. You will need to have [Vagrant](https://www.vagrantup.com/) installed with your chosen virtualization platform. VirtualBox is supported by default, and is both easy to use and available for Windows, OS X, and Linux. + +Once you have Vagrant installed, fork and clone the X2CRM repo from [GitHub](https://github.com/X2Engine/X2CRM) into the developer tools directory. Then, create a new branch for your feature or fix, following a naming convention of "{name}\_{feature}". You can then bring up the Vagrant environment, perform your first setup, and begin developing! + +```bash +local$ vagrant up +# Lots of output the first time this machine is brought up +local$ vagrant ssh +vagrant$ x2util setup +``` + +A new `workdir` directory will be created, which is synced to the virtual machines webroot at /var/www/html. You can edit the source code in your IDE from this directory to have the changes automatically propagated to your web server in the virtual machine. When you have finished developing your feature or fix, sync your changes back into the Git repo, commit and push the changes, then submit a pull request to the X2CRM repo. --Rebuild additional existing scripts in python as we discover we need them +```bash +vagrant$ x2util rsync +vagrant$ exit +local$ vagrant halt +local$ cd X2CRM +local$ git add -p +local$ git commit +local$ git push +``` + +A helper utility is provided within the Vagrant environment to simplify test execution. While writing your unit tests, you can execute them using the `x2test` command. Tab completion is provided for the files within the protected/tests directory. If no test is specified, the entire unit test suite will be executed. + +```bash +vagrant$ x2test +# Executes the complete unit testing suite. This will take 30+ minutes, depending on hardware. +# Don't worry if you see the translation test fail, this is likely caused by the performance +# hit due to the synced directory. +vagrant$ x2test unit/components/X2IPAddressTest.php +# Executes a specific test +``` + +## Future Plans +* Add support for ignoring server-specific unit/functional testing configuration +* Rebuild additional existing scripts in python as we discover we need them diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..30b1130 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,149 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "debian/jessie64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + if not Dir.exists?("X2CRM") + puts "Please clone the X2CRM repo before proceeding!" + exit + end + Dir.mkdir("workdir") if not Dir.exists?("workdir") + config.vm.synced_folder "workdir", "/var/www/html", owner: 'www-data' + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "1024" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell", inline: <<-SHELL + # Install dependencies and set up database + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y git rsync apache2 mysql-server php5 php5-mysql php5-gd php5-curl php5-mcrypt phpunit phpunit-selenium xvfb openjdk-7-jre chromium + sudo ln -s /usr/bin/chromium /usr/local/bin/google-chrome # Fake it for the default Selenium config + mysqladmin -u root password x2crm + mysqladmin -u root -px2crm create x2crm + mysql -u root -px2crm -e 'create user x2crm@localhost identified by "x2crm";' + mysql -u root -px2crm -e 'grant all on x2crm.* to x2crm@localhost;' + mysqladmin -u root -px2crm flush-privileges + + # Configure virtual host + grep X2 /etc/apache2/sites-enabled/000-default.conf + if [ $? -ne 0 ]; then + sed -i '/DocumentRoot/ s/html/html\\/X2CRM/' /etc/apache2/sites-enabled/000-default.conf + sudo chown www-data /var/www + sudo service apache2 restart + fi + + # Set up environment and aliases + grep x2 /home/vagrant/.bashrc + if [ $? -ne 0 ]; then + cat >> /home/vagrant/.bashrc <<-BASHCONFIG +export DISPLAY=:22 +export GITDIR="/vagrant" +export WORKINGDIR="/var/www/html" +export INSTALLEDDIR="/var/www/html" +export MYSQLUSER="x2crm" +export MYSQLPASS="x2crm" +export MYSQLDATABASE="x2crm" +export INSTALLREMOTE=0 +export REMOTEUSER="username" +export REMOTESERVER="remoteserver.com" +export REMOTEWEBROOT="public_html" +alias x2util=/vagrant/x2util +alias x2test=/home/vagrant/x2test.sh +_x2testing() { + local cur=\\${COMP_WORDS[COMP_CWORD]} + COMPREPLY=( \\$(compgen -W "\\$(find /var/www/html/X2CRM/x2engine/protected/tests/ | sed 's|/var/www/html/X2CRM/x2engine/protected/tests/||')" -- \\$cur) ) +} +complete -F _x2testing x2test +BASHCONFIG + fi + + # Create testing script + if [ ! -f /home/vagrant/x2test.sh ]; then + cat >> /home/vagrant/x2test.sh <<-TESTSCRIPT +#!/bin/bash +if [ -n "\\$1" ]; then + TEST="x2engine/protected/tests/\\${1}" +else + TEST="x2engine/protected/tests/unit" +fi + +cd /var/www/html/X2CRM +sudo -u www-data phpunit --config x2engine/protected/tests/phpunit.xml \\$TEST +cd +TESTSCRIPT + chmod +x /home/vagrant/x2test.sh + fi + + # Configure Selenium for functional testing + if [ ! -f selenium-server-standalone-2.48.2.jar ]; then + wget https://selenium-release.storage.googleapis.com/2.48/selenium-server-standalone-2.48.2.jar + fi + if [ ! -f /etc/init.d/selenium ]; then + cat >> /etc/init.d/selenium <<-SELENIUMSCRIPT +#!/bin/sh +export DISPLAY=:22 +su vagrant -c 'Xvfb -fp /usr/share/fonts/X11/misc/ :22 -screen 0 1024x768x16 2>&1 &' +su vagrant -c 'java -jar /home/vagrant/selenium-server-standalone-2.48.2.jar -browserSessionReuse -singlewindow &' +SELENIUMSCRIPT + chmod +x /etc/init.d/selenium + ln -s /etc/init.d/selenium /etc/rc2.d/S99selenium + /etc/init.d/selenium + fi + SHELL +end diff --git a/x2util b/x2util index 0298bac..24e1ee1 100755 --- a/x2util +++ b/x2util @@ -28,7 +28,7 @@ def parse_arguments(): parser.add_argument('command', choices = ['setup', 'rsync', 'reinstall', 'testing'], help = "Command to be run, corresponding to a series of actions related to installing or manipulating X2CRM installations") - parser.add_argument('-g', '--gitdir', default = os.environ["GITDIR"], help = 'Direcory of your X2CRM Git Repository') + parser.add_argument('-g', '--gitdir', default = os.environ["GITDIR"], help = 'Directory of your X2CRM Git Repository') parser.add_argument('-d', '--directory', default = os.environ["WORKINGDIR"], help = 'Directory of your working fileset for X2CRM') parser.add_argument('-u', '--mysqluser', default = os.environ["MYSQLUSER"], help = 'User with the ability to drop/create their own database') parser.add_argument('-p', '--mysqlpass', default = os.environ["MYSQLPASS"], help = "Password corresponding to the provided MySQL user") @@ -37,7 +37,7 @@ def parse_arguments(): parser.add_argument('-a', '--dummydata', type=int, default=1, help = 'Toggles including dummy data with the installation process') parser.add_argument('-i', '--installremote', type=int, default=os.environ["INSTALLREMOTE"], help = "Toggles installation to a remote server") parser.add_argument('-l', '--remoteuser', help = "Username to connect to the remote server with, only required with -i" ) - parser.add_argument('-s', '--remoteserver', help = "Remote hostname to connect to, only required iwith -i") + parser.add_argument('-s', '--remoteserver', help = "Remote hostname to connect to, only required with -i") parser.add_argument('-w', '--remotewebroot', help = "Remote webroot where X2CRM is being deployed to") args = parser.parse_args() @@ -60,11 +60,13 @@ def refresh_database(options): but does mean that users will need permissions to drop and create their own database. """ - cmd = ['mysql', '-u', options.mysqluser, '-p"'+options.mysqlpass+'"', '-e', - '"drop database if exists '+options.database+'; create database '+options.database+'"'] - if options.installremote == 1: + cmd = ['mysql', '-u', options.mysqluser, '-p"'+options.mysqlpass+'"', '-e', + 'drop database if exists '+options.database+'; create database '+options.database+''] cmd = ['ssh', options.remoteuser+'@'+options.remoteserver] + cmd + else: + cmd = ['mysql', '-u', options.mysqluser, '-p'+options.mysqlpass+'', '-e', + 'drop database if exists '+options.database+'; create database '+options.database+''] subprocess.check_call(cmd) @@ -79,11 +81,11 @@ def install_gitdir(options): rmcmd = subprocess.check_call(['xargs', '-r', 'rm', '-r'], stdin=tailcmd.stdout) print('deleting directory '+options.directory) - cmd = ['sudo', 'rsync', '-avc', '--delete', options.gitdir+'/', options.directory] + cmd = ['sudo', 'rsync', '-avc', '--delete', options.gitdir+'/X2CRM', options.directory] subprocess.check_call(cmd) if options.installremote != 1: - cmd = ['sudo', 'chown', '-R', '33', options.directory+'/X2CRM/x2engine'] + cmd = ['sudo', 'chown', '-R', '33', options.directory] subprocess.check_call(cmd) else: cmd = ['ssh', options.remoteuser+'@'+options.remoteserver, 'sudo', 'chown', '-R', options.remoteuser+':'+options.remoteuser, options.remotewebroot] @@ -113,7 +115,7 @@ def refresh_install_files(options): ] for file in files: - cmd = ['cp', options.gitdir+'/X2CRM/x2engine'+file, options.directory] + cmd = ['sudo', 'cp', options.gitdir+'/X2CRM/x2engine'+file, options.directory+'/X2CRM/x2engine/'] subprocess.check_call(cmd) if options.installremote != 1: @@ -127,8 +129,8 @@ def refresh_install_files(options): cmd = ['rsync', '-avzcO', options.directory+'/X2CRM/x2engine'+file, options.remoteuser+'@'+options.remoteserver+':'+options.remotewebroot] subprocess.check_call(cmd) - cmd = ['ssh', options.remoteuser+'@'+options.remoteserver, 'sudo', 'chown', '-R', '33', options.remotewebroot] - subprocess.check_call(cmd) + cmd = ['ssh', options.remoteuser+'@'+options.remoteserver, 'sudo', 'chown', '-R', '33', options.remotewebroot] + subprocess.check_call(cmd) def prep_installation(options, testing=0): @@ -150,6 +152,15 @@ def prep_installation(options, testing=0): ] if testing == 1: sedlist.append("s/test_db = 0/test_db = 1/") + configfiles = ['phpunit_example.xml', 'WebTestConfig_example.php'] + if options.installremote == 1: + basecmd = [] + else: + basecmd = ['sudo'] + for srcfile in configfiles: + srcfile = options.directory + '/X2CRM/x2engine/protected/tests/' + srcfile + dstfile = re.sub('_example', '', srcfile) + subprocess.check_call(basecmd + ['cp', srcfile, dstfile]) sedstr = ';'.join(sedlist) @@ -158,7 +169,7 @@ def prep_installation(options, testing=0): sedstr = '"' + sedstr + '"' filepath = options.remotewebroot+'/installConfig.php' else: - basecmd = [] + basecmd = ['sudo'] filepath = options.directory+'/X2CRM/x2engine/installConfig.php' subprocess.check_call(basecmd + ['sed', '-i', sedstr, filepath]) @@ -223,7 +234,7 @@ def chset(options, flags = {}): base_cmd = ['ssh', options.remoteuser+'@'+options.remoteserver] else: install_path = options.directory+'/X2CRM/x2engine' - base_cmd = [] + base_cmd = ['sudo'] for flag in flags: sedstr = flag_data[flag]['sedstr'].replace('{value}',str(flags[flag])) if options.installremote == 1: @@ -310,8 +321,8 @@ def initialize(options): base_cmd = ['ssh', options.remoteuser+'@'+options.remoteserver] file_path = options.remotewebroot else: - base_cmd = [] - file_path = options.directory + base_cmd = ['sudo', '-u', '#33'] + file_path = options.directory + '/X2CRM/x2engine' cmd = base_cmd + ['php', file_path+'/initialize.php', 'silent'] subprocess.check_call(cmd)