.gitignore

+**/*.pyc
+**/build
+**/dist
+**/glassfish-resources.xml
+.idea

LICENSE

+Copyright (c) UChicago Argonne, LLC. All rights reserved.
+
+Copyright UChicago Argonne, LLC. This software was produced 
+under U.S. Government contract DE-AC02-06CH11357 for Argonne National 
+Laboratory (ANL), which is operated by UChicago Argonne, LLC for the 
+U.S. Department of Energy. The U.S. Government has rights to use, 
+reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR 
+UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR 
+ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is 
+modified to produce derivative works, such modified software should 
+be clearly marked, so as not to confuse it with the version available 
+from ANL.
+
+Additionally, redistribution and use in source and binary forms, with 
+or without modification, are permitted provided that the following 
+conditions are met:
+
+    * Redistributions of source code must retain the above copyright 
+      notice, this list of conditions and the following disclaimer. 
+
+    * Redistributions in binary form must reproduce the above copyright 
+      notice, this list of conditions and the following disclaimer in 
+      the documentation and/or other materials provided with the 
+      distribution. 
+
+    * Neither the name of UChicago Argonne, LLC, Argonne National 
+      Laboratory, ANL, the U.S. Government, nor the names of its 
+      contributors may be used to endorse or promote products derived 
+      from this software without specific prior written permission. 
+
+THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS 
+AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago 
+Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.

Makefile

+# The top level makefile. Targets like "all" and "clean"
+# are defined in the RULES file.
+
+TOP = .
+SUBDIRS = src doc
+
+.PHONY: support dev-config
+.PHONY: db backup deploy-web-portal undeploy-web-portal deploy-ds-web-service undeploy-ds-web-service deploy-daq-web-service undeploy-daq-web-service deploy-cat-web-service undeploy-cat-web-service deploy-proc-web-service undeploy-proc-web-service
+.PHONY: db-dev backup-dev deploy-web-portal-dev undeploy-web-portal-dev deploy-ds-web-service-dev undeploy-ds-web-service-dev deploy-daq-web-service-dev undeploy-daq-web-service-dev deploy-proc-web-service-dev undeploy-proc-web-service-dev
+
+default:
+
+prepare-dev-env: support db dev-config
+
+support:
+	$(TOP)/sbin/dm_install_support.sh
+
+support-daq:
+	$(TOP)/sbin/dm_install_support.sh daq
+
+dev-config:
+	$(TOP)/sbin/dm_prepare_dev_config.sh
+
+db:
+	$(TOP)/sbin/dm_create_db.sh
+
+destroy-db:
+	$(TOP)/sbin/dm_destroy_db.sh
+
+ca:
+	$(TOP)/sbin/dm_create_ca.sh
+
+backup:
+	$(TOP)/sbin/dm_backup.sh 
+
+api-doc:
+	export PYTHONPATH=../../src/python ; make -C doc/sphinx html
+	cd doc && rm -f html && ln -s sphinx/build/html .
+
+configure-web-portal: dist
+	$(TOP)/sbin/dm_configure_web_portal.sh
+
+deploy-web-portal: dist
+	$(TOP)/sbin/dm_deploy_web_portal.sh
+
+deploy-ds-web-service: 
+	$(TOP)/sbin/dm_deploy_ds_web_service.sh
+
+deploy-aps-db-web-service: 
+	$(TOP)/sbin/dm_deploy_aps_db_web_service.sh
+
+deploy-daq-web-service: 
+	$(TOP)/sbin/dm_deploy_daq_web_service.sh
+
+deploy-cat-web-service: 
+	$(TOP)/sbin/dm_deploy_cat_web_service.sh
+
+deploy-proc-web-service: 
+	$(TOP)/sbin/dm_deploy_proc_web_service.sh
+
+unconfigure-web-portal: 
+	$(TOP)/sbin/dm_unconfigure_web_portal.sh 
+
+undeploy-web-portal: 
+	$(TOP)/sbin/dm_undeploy_web_portal.sh
+
+undeploy-ds-web-service: 
+	$(TOP)/sbin/dm_undeploy_ds_web_service.sh
+
+undeploy-daq-web-service: 
+	$(TOP)/sbin/dm_undeploy_daq_web_service.sh
+
+undeploy-cat-web-service: 
+	$(TOP)/sbin/dm_undeploy_cat_web_service.sh
+
+undeploy-proc-web-service: 
+	$(TOP)/sbin/dm_undeploy_proc_web_service.sh
+
+db-dev:
+	$(TOP)/sbin/dm_create_db.sh dm_dev
+
+destroy-db-dev:
+	$(TOP)/sbin/dm_destroy_db.sh dm_dev
+
+backup-dev:
+	$(TOP)/sbin/dm_backup_all.sh dm_dev
+
+configure-web-portal-dev: dist
+	$(TOP)/sbin/dm_configure_web_portal.sh dm_dev
+
+deploy-web-portal-dev: dist
+	$(TOP)/sbin/dm_deploy_web_portal.sh dm_dev
+
+deploy-ds-web-service-dev: 
+	$(TOP)/sbin/dm_deploy_ds_web_service.sh dm_dev
+
+deploy-daq-web-service-dev: 
+	$(TOP)/sbin/dm_deploy_daq_web_service.sh dm_dev
+
+deploy-cat-web-service-dev: 
+	$(TOP)/sbin/dm_deploy_cat_web_service.sh dm_dev
+
+deploy-proc-web-service-dev: 
+	$(TOP)/sbin/dm_deploy_proc_web_service.sh dm_dev
+
+unconfigure-web-portal-dev: 
+	$(TOP)/sbin/dm_unconfigure_web_portal.sh dm_dev
+
+undeploy-web-portal-dev: 
+	$(TOP)/sbin/dm_undeploy_web_portal.sh dm_dev
+
+undeploy-ds-web-service-dev: 
+	$(TOP)/sbin/dm_undeploy_ds_web_service.sh dm_dev
+
+undeploy-daq-web-service-dev: 
+	$(TOP)/sbin/dm_undeploy_daq_web_service.sh dm_dev
+
+undeploy-cat-web-service-dev: 
+	$(TOP)/sbin/dm_undeploy_cat_web_service.sh dm_dev
+
+undeploy-proc-web-service-dev: 
+	$(TOP)/sbin/dm_undeploy_proc_web_service.sh dm_dev
+
+include $(TOP)/tools/make/RULES_DM
+

README.md

+APS Data Management Software
+
+# Introduction
+
+
+The APS Data Management system will streamline processing files created during data collection, and will ease the process of electronic access to data for users. The main design goal for the system is to alleviate tedious data management tasks for beamline staff, while ensuring the integrity and security of data. The system provides an infrastructure for organizing the files that comprise these data sets. By  simplifying remote electronic access to data for users, it allows them to transfer or analyze their data shortly after their beam time.
+
+The Data Management system provides easy-to-use automatic data transfers from beamlines to the central APS storage system (currently 250 TB) for short-term  curation. From there, the system can also be used to transfer data to outside facilities, such as to a user’s home institution or to a longer-term storage system such as that maintained by Argonne’s CELS Directorate (currently 1.7 PB) allowing data to be staged for analysis with Argonne’s Leadership Computing Facility resources, such as the Mira supercomputer. A service layer within the Data Management system automates data transfer from acquisition systems to the storage system and policy enforcement, such as automatic deletion/archival. The GlobusOnline service is used to transfer data to remote locations
+
+A web portal, integrated with the APS User database and experiment scheduling and safety systems, is used to control data access settings and policies. The GUI is largely complete and the service layer is under development. Prototype deployment targeted for mid-2015. The status of the project can be found on the Project Status page.
+
+Details about the software and hardware design and infrastructure can be found of the project's Infrastructure page.
+
+This project was started using Argonne LDRD funds and is now being made production ready using operational funding from the APS, contract DE-AC02-06CH11357.
+
+ 
+# Installation
+
+Work is underway to separate general features of the Data Management System to make this useable at facilities outside of the APS.  As these efforts continue, this page will be updated to provide instructions for developers and for production use at the APS and then also to provide instructions for use outside of the APS.
+
+ - [APS Developer Installation/Setup](doc/Installation/APSDeveloperInstallation.md)
\ No newline at end of file

bin/11BM/dm-11bm-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/11BM/dm-11bm-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/11IDB/dm-11idb-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/11IDB/dm-11idb-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/11IDC/dm-11idc-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/11IDC/dm-11idc-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/12IDB/dm-12idb-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/12IDB/dm-12idb-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/17BM/dm-17bm-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/17BM/dm-17bm-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/1BM/dm-1bm-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/1BM/dm-1bm-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/1ID/dm-1id-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/1ID/dm-1id-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/26ID/dm-26id-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/26ID/dm-26id-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+