beamline_names.txt

+"1-BM-B,C"
+"1-ID-B,C,E"
+"2-BM-A,B"
+"2-ID-D"
+"2-ID-E"
+"3-ID-B,C,D"
+"4-ID-C"
+"4-ID-D"
+"5-BM-C"
+"5-BM-D"
+"5-ID-B,C,D"
+"6-BM-A,B"
+"6-ID-B,C"
+"6-ID-D"
+"7-BM-B"
+"7-ID-B,C,D"
+"8-BM-B"
+"8-ID-E"
+"8-ID-I"
+"9-BM-B,C"
+"9-ID-B,C"
+"10-BM-A,B"
+"10-ID-B"
+"11-BM-B"
+"11-ID-B"
+"11-ID-C"
+"11-ID-D"
+"12-BM-B"
+"12-ID-B"
+"12-ID-C,D"
+"13-BM-C"
+"13-BM-D"
+"13-ID-C,D"
+"13-ID-E"
+"14-BM-C"
+"14-ID-B"
+"15-ID-B,C,D"
+"16-BM-B"
+"16-BM-D"
+"16-ID-B"
+"16-ID-D"
+"17-BM-B"
+"17-ID-B"
+"18-ID-D"
+"19-BM-D"
+"19-ID-D"
+"20-BM-B"
+"20-ID-B,C"
+"21-ID-D"
+"21-ID-E"
+"21-ID-F"
+"21-ID-G"
+"22-BM-D"
+"22-ID-D"
+"23-BM-B"
+"23-ID-B"
+"23-ID-D"
+"24-ID-C"
+"24-ID-E"
+"26-ID-C"
+"27-ID-B"
+"28-ID-B"
+"28-ID-C"
+"29-ID-C,D"
+"30-ID-B,C"
+"31-ID-D"
+"32-ID-B,C"
+"33-BM-C"
+"33-ID-D,E"
+"34-ID-C"
+"34-ID-E"
+"35-ID-B,C,D,E"

cdb_dm_integration_notes.txt

+- DS service
+  - runs on extrepid
+- DAQ service
+  - APSU station, runs on ctlsdaqsrv2
+- Catalog service
+  - APSU station, runs on ctlsdaqsrv2
+- Processing (Workflow) service
+  - can run workflows for each file, or for set of files, in real-time
+  - RT workflow args can be set via PV
+  - DAQ needs to send burst number (e.g, file 3/10)
+  - can always request workflow processing on a set of files (post processing)
+  - need portal that will be able to submit processing job
+  - need to be able to keep track of processing jobs and their results
+
+- DM/CDB integration plugin
+  - triggered via CDB:<qrId> keyword
+  - adds DM-document property, will have metadata catalog URL, 
+    file download URL, metadata key/value pairs as needed
+  - must be able to associate multiple qrID's
+- CDB Document domain
+- CDB/DM integration plugin: needs to be able to click on the file to see the 
+  metadata, or to download the file
+- DM/SDDS plugin
+  - triggered by ".sdds"
+  - reads parameters, converts them to metadata
+
+- Use Case: StudiesSession-20170422
+  - From CDB perspective StudiesSession-20170422 is an item in the 
+    Documents Domain
+  - From DM perspective, StudiesSession-20170422 is an experiment on an APSU 
+    station
+  - When collecting DAQ data, DM will automatically asign all files to be
+    part of the StudiesSession-20170422 experiment
+  - DM/CDB plugin will add DataFile property to StudiesSession-20170422 for 
+    each file
+  - CDB/DM file property handler will be able to display DM metadata (URL link)
+    and download file; later, it may be be able to submit processing job
+    to DM processing service

demo/sprint-20150421/demo_notes.sv.txt

-# Demo environment consists of two linux VMs: 
-#     - data acquisition (DAQ) and data storage (DS) nodes
-#     - CentOS 6.6, 64-bit
-#     - no shared storage
-#     - DS node runs database server, Web Portal and DS Web Service
-#     - DAQ node runs DAQ Web Service
-
-# Machine Preparation
-# ===================
-
-# install dependencies (both machines)
-yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel
-
-# create system (dm) account on both machines, configure ssh-keys and 
-# authorized_keys files
-
-# configure /opt/DM area for software installation
-mkdir -p /opt/DM
-chown -R dm.dm /opt/DM
-chmod 755 /opt/DM
-
-# configure (or disable) firewall (both machines)
-/etc/init.d/iptables stop
-
-# DM Deployment: DS Machine
-# =========================
-
-# Log into dmstorage node and create local DM deployment directory 
-# in dm user home area
-cd /opt/DM
-ls -l
-
-# Checkout code as release 0.1
-svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.1
-
-# Build support area
-cd dm-0.1
-make support
-
-# Source setup 
-source setup.sh
-
-# Create db
-make db
-
-# Configure Web Portal
-# Note:
-#   - this needs to be done only during the first portal deployment,
-#     or after portal has been unconfigured explicitly
-#   - this step configures DB access
-make configure-web-portal
-
-# Deploy Web Portal
-# Note:
-#   - deploys portal war file into glassfish
-#   - after this step, users can access portal at
-#     https://dmstorage.svdev.net:8181/dm
-make deploy-web-portal
-
-# Deploy DS Web Service
-# Note:
-#   - generates SSL certificates and configuration files 
-#   - after this step, DS web service is accessible at port 22236 
-#   - log files are under DM/var/log
-#   - configuration files are under DM/etc
-#   - user setup file is DM/etc/dm.setup.sh
-#   - service control script is under DM/dm-0.1/etc/init.d 
-make deploy-ds-web-service
-
-# Check functionality. Open second terminal and log into dmstorage node
-# as user sveseli
-# Source setup file to get access to DM commands
-source /opt/DM/etc/dm.setup.sh
-
-# Attempt to get list of users as user sveseli, should result 
-# in authorization error
-# Note:
-#   - every command comes with common set of options 
-dm-get-users -h
-dm-get-users --version
-dm-get-users
-echo $?
-
-# Repeat command, this time us administrator (dm) account 
-dm-get-users
-
-# Repeat command, note that session with DS service has been established, so no
-# more password prompts until session expires
-cat ~/.dm/.ds.session.cache 
-dm-get-users
-
-
-# DM Deployment: DAQ Machine
-# ==========================
-
-# Log into dmdaq node and create local DM deployment directory 
-# in dm user home area
-cd /opt/DM
-ls -l
-
-# Checkout code as release 0.1
-svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150421 dm-0.1
-
-# Build support area 
-# Note the following:
-#   - since demo machines are identical, we could simply copy support/dm code
-#     from the storage node; this is not necessarily the case in general
-#   - support area and DM code distribution can be shared between DAQ and DS
-#     nodes
-#   - support area on the daq node is much lighter (i.e., no need
-#     for glassfish, etc.)
-cd dm-0.1
-make support-daq
-
-# Source setup 
-source setup.sh
-
-# Deploy DAQ Web Service
-# Note:
-#   - requires storage node to be installed 
-#   - generates SSL certificates and configuration files 
-#   - after this step, DAQ web service is accessible at port 33336 
-#   - log files are under DM/var/log
-#   - configuration files are under DM/etc
-#   - user setup file is DM/etc/dm.setup.sh
-make deploy-daq-web-service
-
-# DM Functionality: DAQ
-# =====================
-
-# add new experiment (sveseli@dmstorage)
-dm-add-experiment -h
-dm-add-experiment --name exp1 --type-id 1 --description test
-
-dm-get-experiments
-dm-get-experiment --name exp1 
-dm-get-experiment --name exp1 --display-keys=__all__
-
-# check directory content on the storage node (dm@dmstorage)
-ls -l /opt/DM/data
-
-# start experiment (sveseli@dmstorage)
-dm-start-experiment --name exp1
-
-# check directory content on the storage node (dm@dmstorage)
-ls -l /opt/DM/data
-ls -l /opt/DM/data/ESAF
-ls -l /opt/DM/data/ESAF/exp1/
-
-# at this point we can log into the portal to see experiment that was created
-# observe that start time is entered correctly
-
-# in the first terminal on the daq node, tail log file (dm@dmdaq)
-tail -f /opt/DM/var/log/dm.daq-web-service.log
-
-# open second terminal for daq node, login as system (dm) user
-# source setup file (dm@dmdaq)
-cat /opt/DM/etc/dm.setup.sh
-source /opt/DM/etc/dm.setup.sh
-
-# prepare DAQ directory for this experiment (dm@dmdaq)
-mkdir -p /tmp/data/exp1
-
-# start DAQ (dm@dmdaq)
-dm-start-daq -h
-dm-start-daq --experiment exp1 --data-directory /tmp/data/exp1
-
-# create test file in the DAQ directory (daq node)
-# observe log file entries, point out file transfer
-touch /tmp/data/exp1/file1
-echo "Hello there, data management is here" > /tmp/data/exp1/file1
-
-# check directory content on the storage node (dm@dmstorage)
-# file1 should be transferred
-ls -l /opt/DM/data/ESAF/exp1/
-
-# stop DAQ (dm@dmdaq)
-dm-stop-daq -h
-dm-stop-daq --experiment exp1 
-
- 
-# DM Functionality: Upload
-# ========================
-
-# prepare data directory we want to upload (dm@dmdaq)
-mkdir -p /tmp/data/exp1/2015/04/21
-echo "this is file 2" > /tmp/data/exp1/2015/04/21/file2
-echo "this is file 3" > /tmp/data/exp1/2015/04/21/file3
-
-# check directory content on the storage node (dm@dmstorage)
-ls -l /opt/DM/data/ESAF/exp1/
-
-# upload data (dm@dmdaq)
-dm-upload -h
-dm-upload --experiment exp1 --data-directory /tmp/data/exp1
-
-# check directory content on the storage node (dm@dmstorage)
-ls -l /opt/DM/data/ESAF/exp1/
-ls -l /opt/DM/data/ESAF/exp1/2015/04/21/
-cat /opt/DM/data/ESAF/exp1/2015/04/21/file3 
-
-# stop experiment (sveseli@dmstorage)
-dm-stop-experiment --name exp1
-
-# at this point we can log into the portal to see modified experiment 
-# observe that end time is entered correctly
-
-
-

demo/sprint-20150630/demo_notes.sv.txt

-# Demo environment consists of two linux VMs: 
-#     - data acquisition (DAQ) and data storage (DS) nodes
-#     - CentOS 6.6, 64-bit
-#     - no shared storage
-#     - DS node runs database server, Web Portal and DS Web Service
-#     - DAQ node runs DAQ Web Service
-
-# Machine Preparation
-# ===================
-
-# install dependencies (both machines)
-yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel
-
-# Download globus RPM repo and install gridftp (both machines)
-# http://toolkit.globus.org/ftppub/gt6/installers/repo/globus-toolkit-repo-latest.noarch.rpm
-yum install globus-gridftp
-
-# Disable requiredtty in /etc/sudoers
-
-# Prepare gridftp server to use sshd (dmstorage machine)
-globus-gridftp-server-enable-sshftp
-
-# create system (dm) account on both machines, configure ssh-keys and 
-# authorized_keys files
-
-# create several user accounts (dmstorage machine): dmuser1, dmuser2, dmuser3
-
-# build and install epics base and SDDS/SDDSepics extensions under 
-# /opt/epics (dmstorage machine)
-
-# configure /opt/DM area for software installation
-mkdir -p /opt/DM
-chown -R dm.dm /opt/DM
-chmod 755 /opt/DM
-
-# configure (or disable) firewall (both machines)
-/etc/init.d/iptables stop
-
-# DM Deployment: DS Machine
-# =========================
-
-# Log into dmstorage node and create local DM deployment directory 
-# in dm user home area
-cd /opt/DM
-ls -l
-
-# Checkout code as release 0.2
-svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150630 dm-0.2
-
-# Build support area
-cd dm-0.2
-make support
-
-# Source setup 
-source setup.sh
-
-# Create db
-make db
-
-# Configure Web Portal
-# Note:
-#   - this needs to be done only during the first portal deployment,
-#     or after portal has been unconfigured explicitly
-#   - this step configures DB access
-#   - adds initial DM system user to the DB
-make configure-web-portal
-
-# The above step used two new utilities that go directly to the db:
-dm-add-user -h
-dm-add-user-system-role -h
-
-# Add few users
-dm-add-user --username dmuser1 --first-name Test --last-name User1
-dm-add-user --username dmuser2 --first-name Test --last-name User2
-dm-add-user --username dmuser3 --first-name Test --last-name User3
-
-# Deploy Web Portal
-# Note:
-#   - deploys portal war file into glassfish
-#   - after this step, users can access portal at
-#     https://dmstorage.svdev.net:8181/dm
-make deploy-web-portal
-
-# Show no sudo functionality for DM account 
-sudo -l
-
-# Deploy DS Web Service
-# Note:
-#   - generates SSL certificates and configuration files 
-#   - after this step, DS web service is accessible at port 22236 
-#   - log files are under DM/var/log
-#   - configuration files are under DM/etc
-#   - user setup file is DM/etc/dm.setup.sh
-#   - service control script is under DM/dm-0.2/etc/init.d 
-make deploy-ds-web-service
-
-# Show sudo functionality for DM account that enables group/permission
-# management
-sudo -l
-
-# Check functionality. Open second terminal and log into dmstorage node
-# as user sveseli
-# Source setup file to get access to DM commands
-source /opt/DM/etc/dm.setup.sh
-
-# Get user list as administrator (dm) account 
-dm-get-users
-
-# DM Deployment: DAQ Machine
-# ==========================
-
-# Log into dmdaq node and create local DM deployment directory 
-# in dm user home area
-cd /opt/DM
-ls -l
-
-# Checkout code as release 0.2
-svn co https://subversion.xray.aps.anl.gov/DataManagement/tags/20150630 dm-0.2
-
-# Build support area 
-# Note the following:
-#   - since demo machines are identical, we could simply copy support/dm code
-#     from the storage node; this is not necessarily the case in general
-#   - support area and DM code distribution can be shared between DAQ and DS
-#     nodes
-#   - support area on the daq node is much lighter (i.e., no need
-#     for glassfish, etc.)
-cd dm-0.2
-make support-daq
-
-# Source setup 
-source setup.sh
-
-# Deploy DAQ Web Service
-# Note:
-#   - requires storage node to be installed 
-#   - generates SSL certificates and configuration files 
-#   - after this step, DAQ web service is accessible at port 33336 
-#   - log files are under DM/var/log
-#   - configuration files are under DM/etc
-#   - user setup file is DM/etc/dm.setup.sh
-make deploy-daq-web-service
-
-# DM Functionality: DAQ
-# =====================
-
-# add new experiment and couple of users (sveseli@dmstorage)
-dm-add-experiment --name exp1 --type-id 1 --description test
-dm-add-user-experiment-role --username dmuser1 --experiment exp1 --role=User
-dm-add-user-experiment-role --username dmuser2 --experiment exp1 --role=User
-
-# Note that dmuser1 and 2 are on the list of experiment users
-dm-get-experiments
-dm-get-experiment --name exp1 --display-keys=__all__
-
-# check directory content on the storage node (dm@dmstorage)
-ls -l /opt/DM/data
-
-# Show that unix account corresponding to dmuser1 has no special groups
-# associated with it
-id dmuser1
-
-# Show there is no exp1 unix group
-grep exp1 /etc/group
-
-# start experiment (sveseli@dmstorage)
-dm-start-experiment --name exp1
-
-# Show there is now exp1 unix group
-grep exp1 /etc/group
-
-# check directory content on the storage node (dm@dmstorage)
-# note that experiment directory permissions are restricted
-ls -l /opt/DM/data/ESAF
-ls -l /opt/DM/data/ESAF/exp1/
-
-# Check experiment user groups: only 1 and 2 should have new group assigned
-# to them
-id dmuser1
-id dmuser2
-id dmuser3
-
-# in the first terminal on the storage node, tail log file (dm@dmdstorage)
-tail -f /opt/DM/var/log/dm.ds-web-service.log
-
-# in the first terminal on the daq node, tail log file (dm@dmdaq)
-tail -f /opt/DM/var/log/dm.daq-web-service.log
-
-# open second terminal for daq node, login as system (dm) user
-# source setup file (dm@dmdaq)
-source /opt/DM/etc/dm.setup.sh
-
-# prepare DAQ directory for this experiment (dm@dmdaq)
-mkdir -p /tmp/data/exp1
-
-# create test file in the DAQ directory (daq node)
-# observe log file entries, point out file transfer
-echo "Hello there, data management is here" > /tmp/data/exp1/file1
-
-# check directory content on the storage node (dm@dmstorage)
-# file1 should be transferred
-ls -l /opt/DM/data/ESAF/exp1/
-
-# upload data (dm@dmdaq)
-dm-upload --experiment exp1 --data-directory /tmp/data/exp1
-
-# check directory content on the storage node (dm@dmstorage)
-# file1 should be transferred
-# note permissions
-ls -l /opt/DM/data/ESAF/exp1/
-
-# as root@dmstorage, su into dmuser1 account and try to read data
-# should work
-cat /opt/DM/data/ESAF/exp1/file1
-
-# as root@dmstorage, su into dmuser3 account and try to read data
-# should fail
-cat /opt/DM/data/ESAF/exp1/file1
-
-# Demonstrate retries: show config file
-vi /opt/DM/etc/dm.daq-web-service.conf
-
-# As root@dmdaq, temporarily move rsync
-mv /usr/bin/rsync /usr/bin/rsync.orig
-
-# upload new data (dm@dmdaq), observe how transfer fails
-echo "Hello there, data management is here again" > /tmp/data/exp1/file1
-dm-upload --experiment exp1 --data-directory /tmp/data/exp1
-
-# As root@dmdaq, restore rsync, observe how transfer succeeds
-mv /usr/bin/rsync.orig /usr/bin/rsync
-
-# check directory content on the storage node (dm@dmstorage)
-# file1 should be transferred
-ls -l /opt/DM/data/ESAF/exp1/
-
-# Demonstrate gridftp plugin
-# Edit config file as dm@dmdaq, comment out rsync plugin, uncomment gridftp
-# plugin; restart service
-vi /opt/DM/etc/dm.daq-web-service.conf
-./etc/init.d/dm-daq-web-service restart
-tail -f /opt/DM/var/log/dm.daq-web-service.log
-
-# upload new data (dm@dmdaq), observe how transfer succeeds
-echo "Hello there, data management is here yet again" > /tmp/data/exp1/file1
-dm-upload --experiment exp1 --data-directory /tmp/data/exp1
-
-# stop experiment (sveseli@dmstorage)
-dm-stop-experiment --name exp1
-
-
-
-

getting_started/getting-started.md

+Document can now be found [here](https://git.aps.anl.gov/DM/dm-docs/-/wikis/DM/HowTos/Getting-Started) on the [DM Wiki](https://git.aps.anl.gov/DM/dm-docs/-/wikis/home).
+

getting_started/gettingDataFromGlobus.md

+Document can now be found [here](https://git.aps.anl.gov/DM/dm-docs/-/wikis/DM/HowTos/Getting-Data-From-Globus) on the [DM Wiki](https://git.aps.anl.gov/DM/dm-docs/-/wikis/home).
+

mongo_express_notes.txt

+ cd /local/DmSystem/dm
+ source setup.sh 
+ make deploy-cat-web-service
+
+ cd /local/DmSystem/support
+ ./bin/install_node.sh 
+ cd /local/DmSystem/support/node/linux-x86_64/bin/node_modules/mongo-express/
+ cp config.default.js config.js 
+ vi config.js 
+ export PATH=/local/DmSystem/support/node/linux-x86_64/bin:$PATH
+ ../forever/bin/forever start app.js
+ ../forever/bin/forever list

rhel7_package_list.txt

+RHEL7 Packages
+===============
+
+make
+autoconf
+expect
+gcc
+gcc-c++
+subversion
+zlib-devel
+openssl-devel
+libffi-devel
+openldap-devel
+readline-devel
+ncurses-devel
+qt-x11
+qt-postgresql
+qt-devel
+gtk2-devel
+
+Globus Packages
+===============
+
+globus-openssl-module
+globus-ftp-client
+globus-gsi-proxy
+globus-gsi-openssl
+globus-ftp-control
+globus-gass-copy
+globus-common-16.8
+globus-gass-transfer
+globus-io-11.8
+globus-gss-assist
+globus-gsi-cert
+globus-xio-popen
+globus-xio-gsi
+globus-gsi-credential
+globus-callout-3.15
+globus-gsi-sysconfig
+globus-gsi-callback
+globus-xio-5.14
+globus-gssapi-gsi
+globus-gass-copy
+globus-gsi-proxy
+globus-gssapi-error
+