bin/dm-stop-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/stopUploadCli.py $@
+
+

bin/dm-update-experiment

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/ds_web_service/cli/updateExperimentCli.py $@
+
+

bin/dm-update-experiment-dataset

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-update-experiment-file

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-update-user-from-aps-db

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_user_db/cli/updateUserFromApsDbCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-update-users-from-aps-db

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/dm_command_setup.sh

+#!/bin/sh
+
+# Helper functions for DM commands.
+
+# Fix command line arguments 
+DM_COMMAND_ARGS=""
+while [ $# -ne 0 ]; do
+    arg=$1
+    if [[ $arg == -* ]]; then
+        key=`echo $arg | cut -f1 -d'='`
+        keyHasValue=`echo $arg | grep '='`
+        if [ ! -z "$keyHasValue" ]; then
+            value=`echo $arg | cut -f2- -d'='`
+            DM_COMMAND_ARGS="$DM_COMMAND_ARGS $key=\"$value\""
+        else
+            DM_COMMAND_ARGS="$DM_COMMAND_ARGS $key"
+        fi 
+    else
+        DM_COMMAND_ARGS="$DM_COMMAND_ARGS \"$arg\""
+    fi
+    shift
+done
+
+

doc/Makefile

+
+TOP = ..
+SUBDIRS = sphinx
+
+include $(TOP)/tools/make/RULES_DM
+

doc/RELEASE_NOTES.txt

+Release 1.1 (03/10/2017)
+=============================
+
+- Introduced integration with Beamline Scheduling System:
+  - New commands: 
+    - list-runs
+    - list-proposals
+    - get-proposal
+  - Modified add-experiment command to automatically add users associated
+    with a given beamline proposal
+- Added the following functionality for managing DAQs:
+  - Max. run time specification causes DAQ to be stopped automatically
+  - Destination directory specification causes files to be uploaded into
+    a specific directory relative to experiment root path
+  - Upload data/target directory on exit specify upload of the given
+    directory automatically after DAQ completes
+- Added the following functionality for managing uploads:
+  - Destination directory specification causes files to be uploaded into
+    a specific directory relative to experiment root path
+- Introduced framework for higher level beamline specific tools 
+  - New commands that combine adding experiment and running DAQs or
+    uploads
+- Introduced sphinx as python API documentation framework
+- Resolved possible timeout issue when starting DAQ or directory upload 
+  with a directory containing large number of files
+- Simplified data directory command line option for beamlines that use 
+  gridftp (via DM_DATA_DIRECTORY_MAP environment variable)
+
+Release 1.0 (01/31/2017)
+=============================
+
+- Introduced concept of experiment station and redesigned authorization 
+  mechanisms to allow beamline managers to manage their stations; all
+  APIs and CLIs now conform to the new authorization scheme
+- Modified get-experiments utility to allow retrieving list of experiments
+  for a given station
+- Cleaned up web portal by removing unused views, and enabled station 
+  management functionality
+- GPFS DDN (extrepid) has replaced xstor as the main APS storage
+- CLI changes:
+  - add-experiment command requires station name (can be set from env.
+    variable); experiment type can be specified using type name
+  - get-experiments command requires station name for beamline managers (can 
+    be set from env. variable)
+  - start-experiment command is now optional
+
+Release 0.15 (11/01/2016)
+=============================
+
+- Resolved issue with incorrect accounting of processing errors for DAQs
+- Improved DAQ processing algorithm to avoid resource starvation between
+  simultaneous DAQs and uploads
+- Enhanced monitoring status information for both DAQs and uploads
+
+Release 0.14 (10/14/2016)
+=============================
+
+- Introduced new framework and utilities for synchronizing users with 
+  APS DB
+- Resolved several issues with special characters in file names for
+  gridftp transfer plugin  
+
+Release 0.13 (05/27/2016)
+=============================
+
+- Added SFTP file system observer agent
+- Enhanced MongoDB plugin with file md5 sum calculation
+
+Release 0.12 (05/06/2016)
+=============================
+
+- Developed processing for HDF5 metadata in Mongo cataloging plugin
+- Modified catalog API and service interfaces to use file collections on 
+  a per-experiment basis
+
+Release 0.11 (04/29/2016)
+=============================
+
+- Resolved issue with upload command for directories containing large
+  number of files
+- Implemented enhanced upload processing algorithm to avoid resource 
+  starvation between simultaneous DAQs and uploads
+- Added new polling file system observer agent as option for monitoring 
+  directories
+- Reworked catalog API and corresponding MongoDB interfaces to use unique 
+  experiment file paths, rather than file names
+
+Release 0.10 (03/11/2016)
+=============================
+
+- Added dm-list-daqs and dm-list-uploads commands
+- Resolved issue with newly created directories treated as files for 
+  real-time data acquisitions
+
+Release 0.9 (02/25/2016)
+=============================
+
+- Developed directory processing mode for uploads; in this mode file transfer 
+  plugins transfer entire directories as opposed to individual files
+- Added dm-get-processing-plugins command
+- Resolved working directory issue that may occur with simutaneous uploads
+
+Release 0.8 (01/26/2016)
+=============================
+
+- Enhanced upload/daq performance and functionality (hidden files are not
+  processed; for uploads system can detect files that had been processed 
+  already; improved handling and reporting of processing errors)
+- Source file checksum is calculated for rsync/gridftp plugins by default
+- Added dm-stop-upload command
+- Resolved globus online user authorization delay issue 
+
+Release 0.7 (12/08/2015)
+=============================
+
+- Introduced framework and user interfaces for tracking progress of file 
+  uploads and data acquisitions in DAQ service
+- Added ability to monitor multiple directories for the same experiment 
+  simultaneously (required changes to DAQ service REST interfaces)
+- Enhanced start/stop DAQ and upload commands to use DM_FILE_SERVER_URL
+  environment variable
+- Added user interfaces and utilities that enable experiment data download
+  from machines that have SSH access to the storage host
+
+Release 0.6 (11/6/2015)
+=============================
+
+- Added file system observer agent interface for DAQ service
+- Implemented FTP file system observer for DAQ service
+- Added interfaces for deleting user experiment role in DS service
+- Introduced java REST API framework, and specific experiment DS service API 
+- Web Portal notifies DS service about experiment user modifications 
+
+Release 0.5 (10/08/2015)
+=============================
+
+- Implemented Single Sign-On solution for backend services
+- Enabled user authentication via login file
+- Added file stat (with checksum) interface in DS web service
+- After adding user role to experiment via command line, user is also
+  added to experiment group (if one exists)
+- Added rsync file transfer plugin with checksum and delete
+
+Release 0.4 (09/21/2015) 
+=============================
+
+- Number of minor modifications made in preparation for test deployment at
+  beamlines
+
+Release 0.3 (07/22/2015) 
+=============================
+
+- Developed initial version of Catalogging Web Service based on MongoDB
+- Developed sample processing plugins: file metadata catalog, SDDS processing, 
+  SGE job submission
+
+Release 0.2 (06/30/2015) 
+=============================
+
+- Implemented storage permission management and user group management
+- Developed common file processing service plugin framework
+
+Release 0.1 (04/21/2015) 
+=============================
+
+- Functional web portal (user, experiment, and policy pages)
+- Developed web service and its API/CLI frameworks
+- Developed initial version of Data Storage Web Service
+- Developed initial version of Data Acquisition Web Service;
+- DAQ service can monitor file system on a detector node and subsequently
+  transfer data to storage
+

doc/beamline_names.txt

+"1-BM-B,C"
+"1-ID-B,C,E"
+"2-BM-A,B"
+"2-ID-D"
+"2-ID-E"
+"3-ID-B,C,D"
+"4-ID-C"
+"4-ID-D"
+"5-BM-C"
+"5-BM-D"
+"5-ID-B,C,D"
+"6-BM-A,B"
+"6-ID-B,C"
+"6-ID-D"
+"7-BM-B"
+"7-ID-B,C,D"
+"8-BM-B"
+"8-ID-E"
+"8-ID-I"
+"9-BM-B,C"
+"9-ID-B,C"
+"10-BM-A,B"
+"10-ID-B"
+"11-BM-B"
+"11-ID-B"
+"11-ID-C"
+"11-ID-D"
+"12-BM-B"
+"12-ID-B"
+"12-ID-C,D"
+"13-BM-C"
+"13-BM-D"
+"13-ID-C,D"
+"13-ID-E"
+"14-BM-C"
+"14-ID-B"
+"15-ID-B,C,D"
+"16-BM-B"
+"16-BM-D"
+"16-ID-B"
+"16-ID-D"
+"17-BM-B"
+"17-ID-B"
+"18-ID-D"
+"19-BM-D"
+"19-ID-D"
+"20-BM-B"
+"20-ID-B,C"
+"21-ID-D"
+"21-ID-E"
+"21-ID-F"
+"21-ID-G"
+"22-BM-D"
+"22-ID-D"
+"23-BM-B"
+"23-ID-B"
+"23-ID-D"
+"24-ID-C"
+"24-ID-E"
+"26-ID-C"
+"27-ID-B"
+"29-ID-C,D"
+"30-ID-B,C"
+"31-ID-D"
+"32-ID-B,C"
+"33-BM-C"
+"33-ID-D,E"
+"34-ID-C"
+"34-ID-E"
+"35-ID-B,C,D,E"