demo_notes.sv.txt

# Demo environment consists of two linux VMs: 
#     - data acquisition (DAQ) and data storage (DS) nodes
#     - CentOS 6.6, 64-bit
#     - no shared storage
#     - DS node runs database server, Web Portal and DS Web Service
#     - DAQ node runs DAQ Web Service

# Machine Preparation
# ===================

# install dependencies (both machines)
yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel

# create system (dm) account on both machines, configure ssh-keys and 
# authorized_keys files

# configure /opt/DM area for software installation
mkdir -p /opt/DM
chown -R dm.dm /opt/DM
chmod 755 /opt/DM

# configure (or disable) firewall (both machines)
/etc/init.d/iptables stop

# DM Deployment: DS Machine
# =========================

# Log into dmstorage node and create local DM deployment directory 
# in dm user home area
cd /opt/DM
ls -l

# Checkout code as release 0.1
svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.1

# Build support area
cd dm-0.1
make support

# Source setup 
source setup.sh

# Create db
make db

# Configure Web Portal
# Note:
#   - this needs to be done only during the first portal deployment,
#     or after portal has been unconfigured explicitly
#   - this step configures DB access
make configure-web-portal

# Deploy Web Portal
# Note:
#   - deploys portal war file into glassfish
#   - after this step, users can access portal at
#     https://dmstorage.svdev.net:8181/dm
make deploy-web-portal

# Deploy DS Web Service
# Note:
#   - generates SSL certificates and configuration files 
#   - after this step, DS web service is accessible at port 22236 
#   - log files are under DM/var/log
#   - configuration files are under DM/etc
#   - user setup file is DM/etc/dm.setup.sh
#   - service control script is under DM/dm-0.1/etc/init.d 
make deploy-ds-web-service

# Check functionality. Open second terminal and log into dmstorage node
# as user sveseli
# Source setup file to get access to DM commands
source /opt/DM/etc/dm.setup.sh

# Attempt to get list of users as user sveseli, should result 
# in authorization error
# Note:
#   - every command comes with common set of options 
dm-get-users -h
dm-get-users --version
dm-get-users
echo $?

# Repeat command, this time us administrator (dm) account 
dm-get-users

# Repeat command, note that session with DS service has been established, so no
# more password prompts until session expires
cat ~/.dm/.ds.session.cache 
dm-get-users


# DM Deployment: DAQ Machine
# ==========================

# Log into dmdaq node and create local DM deployment directory 
# in dm user home area
cd /opt/DM
ls -l

# Checkout code as release 0.1
svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.1

# Build support area 
# Note the following:
#   - since demo machines are identical, we could simply copy support/dm code
#     from the storage node; this is not necessarily the case in general
#   - support area and DM code distribution can be shared between DAQ and DS
#     nodes
#   - support area on the daq node is much lighter (i.e., no need
#     for glassfish, etc.)
cd dm-0.1
make support-daq

# Source setup 
source setup.sh

# Deploy DAQ Web Service
# Note:
#   - requires storage node to be installed 
#   - generates SSL certificates and configuration files 
#   - after this step, DAQ web service is accessible at port 33336 
#   - log files are under DM/var/log
#   - configuration files are under DM/etc
#   - user setup file is DM/etc/dm.setup.sh
make deploy-daq-web-service

# DM Functionality: DAQ
# =====================

# add new experiment (sveseli@dmstorage)
dm-add-experiment -h
dm-add-experiment --name exp1 --type-id 1 --description test

dm-get-experiments
dm-get-experiment --name exp1 
dm-get-experiment --name exp1 --display-keys=__all__

# check directory content on the storage node (dm@dmstorage)
ls -l /opt/DM/data

# start experiment (sveseli@dmstorage)
dm-start-experiment --name exp1

# check directory content on the storage node (dm@dmstorage)
ls -l /opt/DM/data
ls -l /opt/DM/data/ESAF
ls -l /opt/DM/data/ESAF/exp1/

# at this point we can log into the portal to see experiment that was created
# observe that start time is entered correctly

# in the first terminal on the daq node, tail log file (dm@dmdaq)
tail -f /opt/DM/var/log/dm.daq-web-service.log

# open second terminal for daq node, login as system (dm) user
# source setup file (dm@dmdaq)
cat /opt/DM/etc/dm.setup.sh
source /opt/DM/etc/dm.setup.sh

# prepare DAQ directory for this experiment (dm@dmdaq)
mkdir -p /tmp/data/exp1

# start DAQ (dm@dmdaq)
dm-start-daq -h
dm-start-daq --experiment exp1 --data-directory /tmp/data/exp1

# create test file in the DAQ directory (daq node)
# observe log file entries, point out file transfer
touch /tmp/data/exp1/file1
echo "Hello there, data management is here" > /tmp/data/exp1/file1

# check directory content on the storage node (dm@dmstorage)
# file1 should be transferred
ls -l /opt/DM/data/ESAF/exp1/

# stop DAQ (dm@dmdaq)
dm-stop-daq -h
dm-stop-daq --experiment exp1 

 
# DM Functionality: Upload
# ========================

# prepare data directory we want to upload (dm@dmdaq)
mkdir -p /tmp/data/exp1/2015/04/21
echo "this is file 2" > /tmp/data/exp1/2015/04/21/file2
echo "this is file 3" > /tmp/data/exp1/2015/04/21/file3

# check directory content on the storage node (dm@dmstorage)
ls -l /opt/DM/data/ESAF/exp1/

# upload data (dm@dmdaq)
dm-upload -h
dm-upload --experiment exp1 --data-directory /tmp/data/exp1

# check directory content on the storage node (dm@dmstorage)
ls -l /opt/DM/data/ESAF/exp1/
ls -l /opt/DM/data/ESAF/exp1/2015/04/21/
cat /opt/DM/data/ESAF/exp1/2015/04/21/file3 

# stop experiment (sveseli@dmstorage)
dm-stop-experiment --name exp1

# at this point we can log into the portal to see modified experiment 
# observe that end time is entered correctly