sprint demo notes

demo/sprint-20150630/demo_notes.sv.txt

+# Demo environment consists of two linux VMs: 
+#     - data acquisition (DAQ) and data storage (DS) nodes
+#     - CentOS 6.6, 64-bit
+#     - no shared storage
+#     - DS node runs database server, Web Portal and DS Web Service
+#     - DAQ node runs DAQ Web Service
+
+# Machine Preparation
+# ===================
+
+# install dependencies (both machines)
+yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel
+
+# Download globus RPM repo and install gridftp (both machines)
+# http://toolkit.globus.org/ftppub/gt6/installers/repo/globus-toolkit-repo-latest.noarch.rpm
+yum install globus-gridftp
+
+# Disable requiredtty in /etc/sudoers
+
+# Prepare gridftp server to use sshd (dmstorage machine)
+globus-gridftp-server-enable-sshftp
+
+# create system (dm) account on both machines, configure ssh-keys and 
+# authorized_keys files
+
+# create several user accounts (dmstorage machine): dmuser1, dmuser2, dmuser3
+
+# build and install epics base and SDDS/SDDSepics extensions under 
+# /opt/epics (dmstorage machine)
+
+# configure /opt/DM area for software installation
+mkdir -p /opt/DM
+chown -R dm.dm /opt/DM
+chmod 755 /opt/DM
+
+# configure (or disable) firewall (both machines)
+/etc/init.d/iptables stop
+
+# DM Deployment: DS Machine
+# =========================
+
+# Log into dmstorage node and create local DM deployment directory 
+# in dm user home area
+cd /opt/DM
+ls -l
+
+# Checkout code as release 0.2
+svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.2
+
+# Build support area
+cd dm-0.2
+make support
+
+# Source setup 
+source setup.sh
+
+# Create db
+make db
+
+# Configure Web Portal
+# Note:
+#   - this needs to be done only during the first portal deployment,
+#     or after portal has been unconfigured explicitly
+#   - this step configures DB access
+#   - adds initial DM system user to the DB
+make configure-web-portal
+
+# The above step used two new utilities that go directly to the db:
+dm-add-user -h
+dm-add-user-system-role -h
+
+# Add few users
+dm-add-user --username dmuser1 --first-name Test --last-name User1
+dm-add-user --username dmuser2 --first-name Test --last-name User2
+dm-add-user --username dmuser3 --first-name Test --last-name User3
+
+# Deploy Web Portal
+# Note:
+#   - deploys portal war file into glassfish
+#   - after this step, users can access portal at
+#     https://dmstorage.svdev.net:8181/dm
+make deploy-web-portal
+
+# Deploy DS Web Service
+# Note:
+#   - generates SSL certificates and configuration files 
+#   - after this step, DS web service is accessible at port 22236 
+#   - log files are under DM/var/log
+#   - configuration files are under DM/etc
+#   - user setup file is DM/etc/dm.setup.sh
+#   - service control script is under DM/dm-0.2/etc/init.d 
+make deploy-ds-web-service
+
+# Show sudo functionality for DM account that enables group/permission
+# management
+sudo -l
+
+# Check functionality. Open second terminal and log into dmstorage node
+# as user sveseli
+# Source setup file to get access to DM commands
+source /opt/DM/etc/dm.setup.sh
+
+# Get user list as administrator (dm) account 
+dm-get-users
+
+# DM Deployment: DAQ Machine
+# ==========================
+
+# Log into dmdaq node and create local DM deployment directory 
+# in dm user home area
+cd /opt/DM
+ls -l
+
+# Checkout code as release 0.2
+svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150630 dm-0.2
+
+# Build support area 
+# Note the following:
+#   - since demo machines are identical, we could simply copy support/dm code
+#     from the storage node; this is not necessarily the case in general
+#   - support area and DM code distribution can be shared between DAQ and DS
+#     nodes
+#   - support area on the daq node is much lighter (i.e., no need
+#     for glassfish, etc.)
+cd dm-0.2
+make support-daq
+
+# Source setup 
+source setup.sh
+
+# Deploy DAQ Web Service
+# Note:
+#   - requires storage node to be installed 
+#   - generates SSL certificates and configuration files 
+#   - after this step, DAQ web service is accessible at port 33336 
+#   - log files are under DM/var/log
+#   - configuration files are under DM/etc
+#   - user setup file is DM/etc/dm.setup.sh
+make deploy-daq-web-service
+
+# DM Functionality: DAQ
+# =====================
+
+# add new experiment and couple of users (sveseli@dmstorage)
+dm-add-experiment --name exp1 --type-id 1 --description test
+dm-add-user-experiment-role --username dmuser1 --experiment exp1 --role=User
+dm-add-user-experiment-role --username dmuser2 --experiment exp1 --role=User
+
+# Note that dmuser1 and 2 are on the list of experiment users
+dm-get-experiments
+dm-get-experiment --name exp1 --display-keys=__all__
+
+# check directory content on the storage node (dm@dmstorage)
+ls -l /opt/DM/data
+
+# Show that unix account corresponding to dmuser1 has no special groups
+# associated with it
+id dmuser1
+
+# start experiment (sveseli@dmstorage)
+dm-start-experiment --name exp1
+
+# check directory content on the storage node (dm@dmstorage)
+# note that experiment directory permissions are restricted
+ls -l /opt/DM/data/ESAF
+ls -l /opt/DM/data/ESAF/exp1/
+
+# Check experiment user groups: only 1 and 2 should have new group assigned
+# to them
+id dmuser1
+id dmuser2
+id dmuser3
+
+# in the first terminal on the daq node, tail log file (dm@dmdaq)
+tail -f /opt/DM/var/log/dm.daq-web-service.log
+
+# open second terminal for daq node, login as system (dm) user
+# source setup file (dm@dmdaq)
+source /opt/DM/etc/dm.setup.sh
+
+# prepare DAQ directory for this experiment (dm@dmdaq)
+mkdir -p /tmp/data/exp1
+
+# create test file in the DAQ directory (daq node)
+# observe log file entries, point out file transfer
+touch /tmp/data/exp1/file1
+echo "Hello there, data management is here" > /tmp/data/exp1/file1
+
+# check directory content on the storage node (dm@dmstorage)
+# file1 should be transferred
+ls -l /opt/DM/data/ESAF/exp1/
+
+# upload data (dm@dmdaq)
+dm-upload --experiment exp1 --data-directory /tmp/data/exp1
+
+# check directory content on the storage node (dm@dmstorage)
+# file1 should be transferred
+# note permissions
+ls -l /opt/DM/data/ESAF/exp1/
+
+# as root@dmstorage, su into dmuser1 account and try to read data
+# should work
+cat /opt/DM/data/ESAF/exp1/file1
+
+# as root@dmstorage, su into dmuser3 account and try to read data
+# should fail
+cat /opt/DM/data/ESAF/exp1/file1
+
+# Demonstrate retries: show config file
+vi /opt/DM/etc/dm.daq-web-service.conf
+
+# As root@dmdaq, temporarily move rsync
+mv /usr/bin/rsync /usr/bin/rsync.orig
+
+# upload new data (dm@dmdaq), observe how transfer fails
+echo "Hello there, data management is here again" > /tmp/data/exp1/file1
+dm-upload --experiment exp1 --data-directory /tmp/data/exp1
+
+# As root@dmdaq, restore rsync, observe how transfer succeeds
+mv /usr/bin/rsync.orig /usr/bin/rsync
+
+# Demonstrate gridftp plugin
+# Edit config file as dm@dmdaq, comment out rsync plugin, uncomment gridftp
+# plugin; restart service
+vi /opt/DM/etc/dm.daq-web-service.conf
+./etc/init.d/dm-daq-web-service restart
+tail -f /opt/DM/var/log/dm.daq-web-service.log
+
+# upload new data (dm@dmdaq), observe how transfer succeeds
+echo "Hello there, data management is here yet again" > /tmp/data/exp1/file1
+dm-upload --experiment exp1 --data-directory /tmp/data/exp1
+
+# stop experiment (sveseli@dmstorage)
+dm-stop-experiment --name exp1
+
+
+
+