diff --git a/demo/sprint-20150630/demo_notes.sv.txt b/demo/sprint-20150630/demo_notes.sv.txt new file mode 100644 index 0000000000000000000000000000000000000000..64d7374eb1e5a82b4286ddb93422450f805bd1ce --- /dev/null +++ b/demo/sprint-20150630/demo_notes.sv.txt @@ -0,0 +1,238 @@ +# Demo environment consists of two linux VMs: +# - data acquisition (DAQ) and data storage (DS) nodes +# - CentOS 6.6, 64-bit +# - no shared storage +# - DS node runs database server, Web Portal and DS Web Service +# - DAQ node runs DAQ Web Service + +# Machine Preparation +# =================== + +# install dependencies (both machines) +yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel + +# Download globus RPM repo and install gridftp (both machines) +# http://toolkit.globus.org/ftppub/gt6/installers/repo/globus-toolkit-repo-latest.noarch.rpm +yum install globus-gridftp + +# Disable requiredtty in /etc/sudoers + +# Prepare gridftp server to use sshd (dmstorage machine) +globus-gridftp-server-enable-sshftp + +# create system (dm) account on both machines, configure ssh-keys and +# authorized_keys files + +# create several user accounts (dmstorage machine): dmuser1, dmuser2, dmuser3 + +# build and install epics base and SDDS/SDDSepics extensions under +# /opt/epics (dmstorage machine) + +# configure /opt/DM area for software installation +mkdir -p /opt/DM +chown -R dm.dm /opt/DM +chmod 755 /opt/DM + +# configure (or disable) firewall (both machines) +/etc/init.d/iptables stop + +# DM Deployment: DS Machine +# ========================= + +# Log into dmstorage node and create local DM deployment directory +# in dm user home area +cd /opt/DM +ls -l + +# Checkout code as release 0.2 +svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.2 + +# Build support area +cd dm-0.2 +make support + +# Source setup +source setup.sh + +# Create db +make db + +# Configure Web Portal +# Note: +# - this needs to be done only during the first portal deployment, +# or after portal has been unconfigured explicitly +# - this step configures DB access +# - adds initial DM system user to the DB +make configure-web-portal + +# The above step used two new utilities that go directly to the db: +dm-add-user -h +dm-add-user-system-role -h + +# Add few users +dm-add-user --username dmuser1 --first-name Test --last-name User1 +dm-add-user --username dmuser2 --first-name Test --last-name User2 +dm-add-user --username dmuser3 --first-name Test --last-name User3 + +# Deploy Web Portal +# Note: +# - deploys portal war file into glassfish +# - after this step, users can access portal at +# https://dmstorage.svdev.net:8181/dm +make deploy-web-portal + +# Deploy DS Web Service +# Note: +# - generates SSL certificates and configuration files +# - after this step, DS web service is accessible at port 22236 +# - log files are under DM/var/log +# - configuration files are under DM/etc +# - user setup file is DM/etc/dm.setup.sh +# - service control script is under DM/dm-0.2/etc/init.d +make deploy-ds-web-service + +# Show sudo functionality for DM account that enables group/permission +# management +sudo -l + +# Check functionality. Open second terminal and log into dmstorage node +# as user sveseli +# Source setup file to get access to DM commands +source /opt/DM/etc/dm.setup.sh + +# Get user list as administrator (dm) account +dm-get-users + +# DM Deployment: DAQ Machine +# ========================== + +# Log into dmdaq node and create local DM deployment directory +# in dm user home area +cd /opt/DM +ls -l + +# Checkout code as release 0.2 +svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150630 dm-0.2 + +# Build support area +# Note the following: +# - since demo machines are identical, we could simply copy support/dm code +# from the storage node; this is not necessarily the case in general +# - support area and DM code distribution can be shared between DAQ and DS +# nodes +# - support area on the daq node is much lighter (i.e., no need +# for glassfish, etc.) +cd dm-0.2 +make support-daq + +# Source setup +source setup.sh + +# Deploy DAQ Web Service +# Note: +# - requires storage node to be installed +# - generates SSL certificates and configuration files +# - after this step, DAQ web service is accessible at port 33336 +# - log files are under DM/var/log +# - configuration files are under DM/etc +# - user setup file is DM/etc/dm.setup.sh +make deploy-daq-web-service + +# DM Functionality: DAQ +# ===================== + +# add new experiment and couple of users (sveseli@dmstorage) +dm-add-experiment --name exp1 --type-id 1 --description test +dm-add-user-experiment-role --username dmuser1 --experiment exp1 --role=User +dm-add-user-experiment-role --username dmuser2 --experiment exp1 --role=User + +# Note that dmuser1 and 2 are on the list of experiment users +dm-get-experiments +dm-get-experiment --name exp1 --display-keys=__all__ + +# check directory content on the storage node (dm@dmstorage) +ls -l /opt/DM/data + +# Show that unix account corresponding to dmuser1 has no special groups +# associated with it +id dmuser1 + +# start experiment (sveseli@dmstorage) +dm-start-experiment --name exp1 + +# check directory content on the storage node (dm@dmstorage) +# note that experiment directory permissions are restricted +ls -l /opt/DM/data/ESAF +ls -l /opt/DM/data/ESAF/exp1/ + +# Check experiment user groups: only 1 and 2 should have new group assigned +# to them +id dmuser1 +id dmuser2 +id dmuser3 + +# in the first terminal on the daq node, tail log file (dm@dmdaq) +tail -f /opt/DM/var/log/dm.daq-web-service.log + +# open second terminal for daq node, login as system (dm) user +# source setup file (dm@dmdaq) +source /opt/DM/etc/dm.setup.sh + +# prepare DAQ directory for this experiment (dm@dmdaq) +mkdir -p /tmp/data/exp1 + +# create test file in the DAQ directory (daq node) +# observe log file entries, point out file transfer +touch /tmp/data/exp1/file1 +echo "Hello there, data management is here" > /tmp/data/exp1/file1 + +# check directory content on the storage node (dm@dmstorage) +# file1 should be transferred +ls -l /opt/DM/data/ESAF/exp1/ + +# upload data (dm@dmdaq) +dm-upload --experiment exp1 --data-directory /tmp/data/exp1 + +# check directory content on the storage node (dm@dmstorage) +# file1 should be transferred +# note permissions +ls -l /opt/DM/data/ESAF/exp1/ + +# as root@dmstorage, su into dmuser1 account and try to read data +# should work +cat /opt/DM/data/ESAF/exp1/file1 + +# as root@dmstorage, su into dmuser3 account and try to read data +# should fail +cat /opt/DM/data/ESAF/exp1/file1 + +# Demonstrate retries: show config file +vi /opt/DM/etc/dm.daq-web-service.conf + +# As root@dmdaq, temporarily move rsync +mv /usr/bin/rsync /usr/bin/rsync.orig + +# upload new data (dm@dmdaq), observe how transfer fails +echo "Hello there, data management is here again" > /tmp/data/exp1/file1 +dm-upload --experiment exp1 --data-directory /tmp/data/exp1 + +# As root@dmdaq, restore rsync, observe how transfer succeeds +mv /usr/bin/rsync.orig /usr/bin/rsync + +# Demonstrate gridftp plugin +# Edit config file as dm@dmdaq, comment out rsync plugin, uncomment gridftp +# plugin; restart service +vi /opt/DM/etc/dm.daq-web-service.conf +./etc/init.d/dm-daq-web-service restart +tail -f /opt/DM/var/log/dm.daq-web-service.log + +# upload new data (dm@dmdaq), observe how transfer succeeds +echo "Hello there, data management is here yet again" > /tmp/data/exp1/file1 +dm-upload --experiment exp1 --data-directory /tmp/data/exp1 + +# stop experiment (sveseli@dmstorage) +dm-stop-experiment --name exp1 + + + +