Merge remote-tracking branch 'upstream/master'

Conflicts:
	src/java/DmWebPortal/nbproject/project.properties

.gitignore

 **/*.pyc
+**/build
+**/dist
+**/glassfish-resources.xml

.svnignore

-var

doc/.svnignore

-html

doc/RELEASE_NOTES.txt

+Release 3.0.0 (07/26/2019)
+=============================
+- Added python web service API for downloading files 
+- Updated DB schema for experiment data archival support
+- Web Portal changes:
+  - Added storage views
+  - Added connection between station and experiment types
+  - Updated experiment views with storage and root path fields
+
 Release 2.6.0 (07/15/2019)
 =============================
 - Moved code repo to gitlab

doc/dm_test_system_deployment_notes.txt

-***************
-
 Prerequisites:
-
-========
-- required OS packages are listed here https://confluence.aps.anl.gov/display/DMGT/DM+Station+System+Requirements
-
+======================
+- required OS packages are listed here:
+  https://confluence.aps.anl.gov/display/DMGT/DM+Station+System+Requirements
+- make sure that user ssh login keys are setup and work for both 127.0.0.1 
+  interface, as well as for the short/full installation machine name 
+- installing DM support software and deploying test system should not
+  require elevated privileges
+- instructions below assume that user's git ssh keys have been setup
 
 Installing DM Support
-=============
+======================
 1) mkdir -p DM_INSTALL_DIR && cd DM_INSTALL_DIR
-2) svn co https://subversion.xray.aps.anl.gov/DataManagement/support
+2) git clone git@git.aps.anl.gov:DM/dm-support support
 3) cd support
 4) ./sbin/install_support_all.sh
-  - you will need to enter two passwords of your choice for glassfish (master and admin password); each password needs to be entered only once, as expect scripts handle repeated requests
-
+  - you will need to enter two passwords of your choice for glassfish 
+    (master and admin password) 
+  - each password needs to be entered only once, as expect scripts handle 
+    repeated requests
 
 Deploying Test System
-==============
+======================
 1) cd DM_INSTALL_DIR
-2) svn co https://subversion.xray.aps.anl.gov/DataManagement/trunk dev
-3) cd dev
+2) git clone git@git.aps.anl.gov:DM/dm
+3) cd dm
 4) ./sbin/dm_deploy_test_system.sh
   - passwords needed:
     * postgres admin password (your choice)
@@ -28,9 +32,11 @@ Deploying Test System
     * dmadmin LDAP password (existing)
     * dmadmin BSS login password (existing)
     * dmadmin ESAF DB password (existing)
-  - scripts also require entry for the data storage directory (e.g, DM_INSTALL_DIR/data), etc; for most of those entries the defaults, if given, are fine
+  - scripts also require entry for the data storage directory 
+    (e.g, DM_INSTALL_DIR/data), etc
+  - for most of the required entries the defaults, if given, are fine
 
 Removing Test System
-==============
+======================
 1) DM_INSTALL_DIR/dev/sbin/dm_remove_test_system.sh 
 

doc/sphinx/.svnignore

-build
-

sbin/dm_remove_catalog_duplicates.sh

-e=Cycle2018_2_Axalta; for f in `cat $e.diff`; do echo ; echo
-"*******************************" ; echo $f; efp=`echo $f | sed
-'s?Cycle2018_2_Axalta/??'`; echo $efp; dm-stat-file --experiment=$e
---relative-path=$efp; echo ; echo LIST; dm-list-experiment-files
---experiment=$e experimentFilePath:$efp; echo ; read -p "Enter file id
-to be deleted: " fileId; dm-delete-file --keep-in-storage
---experiment=$e --file-id=$fileId; done
+#!/bin/bash
 
+# Script for removing duplicate/bad entries from DM Catalog
+
+usage() {
+    echo "Usage:"
+    echo "    $0 <experiment name> [<work dir>]"
+    echo ""
+}
+EXPERIMENT_NAME=$1
+if [ -z "$EXPERIMENT_NAME" ]; then
+    usage
+    exit 1
+fi
+WORK_DIR=${2:-/tmp}
+mkdir -p $WORK_DIR || exit 1
+echo "Using work directory $WORK_DIR for experiment $EXPERIMENT_NAME"
+
+FULL_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.all`
+UNIQUE_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.unique`
+STAT_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.stat`
+GOOD_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.good`
+DELETED_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.deleted`
+BAD_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.bad`
+DUPLICATE_LIST_FILE=`realpath $WORK_DIR/$EXPERIMENT_NAME.duplicate`
+rm -f $GOOD_LIST_FILE $BAD_LIST_FILE $DELETED_LIST_FILE $DUPLICATE_LIST_FILE
+touch $GOOD_LIST_FILE $BAD_LIST_FILE $DELETED_LIST_FILE $DUPLICATE_LIST_FILE
+
+echo "Retrieving list of all catalog entries"
+dm-list-experiment-files --experiment=$EXPERIMENT_NAME --display-keys=id,experimentFilePath,fileSize,md5Sum > $FULL_LIST_FILE || exit 1
+echo "Retrieving list of unique file paths"
+dm-list-experiment-files --experiment=$EXPERIMENT_NAME --display-keys=experimentFilePath | sort -u | sed 's?experimentFilePath=??' | awk '{print $0}' > $UNIQUE_LIST_FILE || exit 1
+
+nUnique=`wc -l $UNIQUE_LIST_FILE | awk '{print $1}'`
+nFiles=`wc -l $FULL_LIST_FILE | awk '{print $1}'`
+echo "Total number of catalog entries: $nFiles"
+echo "Total number of unique files: $nUnique"
+
+storageHost=`dm-get-experiment --experiment=$EXPERIMENT_NAME --display-keys=storageHost | sed 's?storageHost=??'`
+storageDirectory=`dm-get-experiment --experiment=$EXPERIMENT_NAME --display-keys=storageDirectory | sed 's?storageDirectory=??'`
+nFilesInStorage=`ssh $storageHost "find $storageDirectory -type f | wc -l"`
+echo "Total number of files in storage: $nFilesInStorage $storageDirectory"
+echo
+
+OLD_IFS=$IFS
+IFS=
+fCount=0
+while read -r f; do
+    fCount=`expr $fCount + 1`
+    IFS=$OLD_IFS
+    f=`echo $f | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//'`
+    echo "**********"
+    echo "Working on file: \"$f\" ($fCount / $nUnique)"
+    dm-stat-file --experiment=$EXPERIMENT_NAME --relative-path=$f --md5sum > $STAT_FILE || exit 1
+    fileSize=`cat $STAT_FILE | sed -e '1,/STAT INFO/d' | awk '{print $2}' | sed 's?fileSize=??'`
+    md5Sum=`cat $STAT_FILE | sed -e '1,/STAT INFO/d' | awk '{print $3}' | sed 's?md5Sum=??'`
+    echo "File size: $fileSize, md5Sum: $md5Sum"
+    nEntries=`cat $FULL_LIST_FILE | grep "experimentFilePath=$f " | wc -l`
+    echo "There are $nEntries catalog entries"
+    goodId=""
+    idList=`cat $FULL_LIST_FILE | grep "experimentFilePath=$f " | awk '{print $1}'`
+    for id in $idList; do
+        catFileSize=`cat $FULL_LIST_FILE | grep "$id" | awk '{print $3}' | sed 's?fileSize=??' `
+        catMd5Sum=`cat $FULL_LIST_FILE | grep "$id" | awk '{print $4}' | sed 's?md5Sum=??' `
+        if [ "$catFileSize" = "$fileSize" -a "$catMd5Sum" = "$md5Sum" ]; then
+            echo "Catalog info is correct for $f, $id"
+            if [ "x$goodId" = "x" ]; then
+                echo "File $id is marked as good"
+                echo "$f $id" >> $GOOD_LIST_FILE
+                goodId=$id
+            else
+                echo "File $id is marked as duplicate of $goodId"
+                echo "$f $id" >> $DUPLICATE_LIST_FILE
+                echo dm-delete-file --keep-in-storage --experiment=$EXPERIMENT_NAME --file-$id >> $DELETED_LIST_FILE
+            fi
+        else
+            echo "Catalog info is not correct for $f, file size: $catFileSize, md5Sum: $catMd5Sum"
+            echo "$f $id" >> $BAD_LIST_FILE
+            echo dm-delete-file --keep-in-storage --experiment=$EXPERIMENT_NAME --file-$id >> $DELETED_LIST_FILE
+        fi
+    done
+done < "$UNIQUE_LIST_FILE"
+
+echo
+echo "**********"
+echo
+echo "Total number of files in storage  : $nFilesInStorage $storageDirectory"
+echo "Number of all catalog entries     : `wc -l $FULL_LIST_FILE`"
+echo "Number of unique catalog entries  : `wc -l $UNIQUE_LIST_FILE`"
+echo "Number of good catalog entries    : `wc -l $GOOD_LIST_FILE`"
+echo "Number of bad catalog entries     : `wc -l $BAD_LIST_FILE`"
+echo "Number of deleted catalog entries : `wc -l $DELETED_LIST_FILE`"

src/c++/configure/.svnignore

-O.*
-*.local
-*.support

src/c++/iocBoot/iocDmInt/.svnignore

-O.*
-.svnignore
-cdCommands.debug
-cdCommands
-envPaths

src/java/DmApi/.svnignore

-build
-dist

src/java/DmBase/.svnignore

-build
-dist

src/java/DmWebPortal/nbproject/private/.svnignore

-private.properties
-private.xml

src/java/DmWebPortal/nbproject/private/private.properties

-j2ee.server.home=/glassfish
-j2ee.server.middleware=
+deploy.ant.properties.file=/home/oxygen/SVESELI/.netbeans/8.2/config/GlassFishEE6/Properties/gfv3700118484.properties
+j2ee.platform.is.jsr109=true
+j2ee.server.domain=/local/sveseli/DM/git-support/opt/netbeans/glassfish-4.1.1/glassfish/domains/domain1
+j2ee.server.home=/local/sveseli/DM/git-support/opt/netbeans/glassfish-4.1.1/glassfish
+j2ee.server.instance=[/local/sveseli/DM/git-support/opt/netbeans/glassfish-4.1.1/glassfish:/local/sveseli/DM/git-support/opt/netbeans/glassfish-4.1.1/glassfish/domains/domain1]deployer:gfv3ee6wc:localhost:4848
+j2ee.server.middleware=/local/sveseli/DM/git-support/opt/netbeans/glassfish-4.1.1
+user.properties.file=/home/oxygen/SVESELI/.netbeans/8.2/build.properties

src/java/DmWebPortal/nbproject/private/private.xml

@@ -2,9 +2,6 @@
 <project-private xmlns="http://www.netbeans.org/ns/project-private/1">
     <editor-bookmarks xmlns="http://www.netbeans.org/ns/editor-bookmarks/2" lastBookmarkId="0"/>
     <open-files xmlns="http://www.netbeans.org/ns/projectui-open-files/2">
-        <group>
-            <file>file:/local/sveseli/Work/DM/dev/src/java/DmWebPortal/web/templates/menubarTemplate.xhtml</file>
-            <file>file:/local/sveseli/Work/DM/dev/src/java/DmWebPortal/web/views/home.xhtml</file>
-        </group>
+        <group/>
     </open-files>
 </project-private>

src/java/DmWebPortal/setup/.svnignore

-glassfish-resources.xml

src/java/DmWebPortal/src/java/gov/anl/aps/dm/portal/controllers/ExperimentStationController.java

@@ -7,6 +7,7 @@ import gov.anl.aps.dm.portal.model.beans.ExperimentStationDbFacade;
 import gov.anl.aps.dm.portal.model.beans.SystemRoleTypeDbFacade;
 import gov.anl.aps.dm.portal.model.beans.UserSystemRoleDbFacade;
 import gov.anl.aps.dm.portal.model.entities.ExperimentStation;
+import gov.anl.aps.dm.portal.model.entities.ExperimentType;
 import gov.anl.aps.dm.portal.model.entities.SystemRoleType;
 import gov.anl.aps.dm.portal.model.entities.UserInfo;
 import gov.anl.aps.dm.portal.model.entities.UserSystemRole;
@@ -38,6 +39,7 @@ public class ExperimentStationController extends DmEntityController<ExperimentSt
     private UserSystemRoleDbFacade userSystemRoleDbFacade;
      
     UserSystemRole currentUserSystemRole;
+    ExperimentType currentExperimentType;
 
     public ExperimentStationController() {
     }
@@ -120,7 +122,7 @@ public class ExperimentStationController extends DmEntityController<ExperimentSt
     public void setCurrentUserSystemRole(UserSystemRole currentUserSystemRole) {
         this.currentUserSystemRole = currentUserSystemRole;
     }
-
+   
     public void prepareAddManager(ExperimentStation experimentStation) {
         UserSystemRole userSystemRole = getCurrentUserSystemRole();
         userSystemRole.setSystemRoleType(systemRoleTypeDbFacade.findManagerRoleType());
@@ -159,6 +161,42 @@ public class ExperimentStationController extends DmEntityController<ExperimentSt
         updateOnRemoval();
     }
 
+    public ExperimentType getCurrentExperimentType() {
+        if (currentExperimentType == null) {
+            currentExperimentType = new ExperimentType();
+        }
+        return currentExperimentType;
+    }
+
+    public void setCurrentExperimentType(ExperimentType currentExperimentType) {
+        this.currentExperimentType = currentExperimentType;
+    }    
+    public void prepareAddExperimentType(ExperimentType experimentType) {
+        setCurrentExperimentType(experimentType);
+    }
+
+    public void addExperimentType() {
+        ExperimentStation experimentStation = getCurrent();
+        List<ExperimentType> experimentTypeList = experimentStation.getExperimentTypeList();
+        if (experimentTypeList.contains(currentExperimentType)) {
+            SessionUtility.addErrorMessage("Error", "Experiment type "
+                    + currentExperimentType.getName()
+                    + " can already be used at "
+                    + experimentStation.getName() + " station.");
+            return;
+        }
+        experimentTypeList.add(0, currentExperimentType);
+        update();
+        currentExperimentType = null;
+    }
+    
+    public void deleteExperimentType(ExperimentType experimentType) {
+        ExperimentStation experimentStation = getCurrent();
+        List<ExperimentType> experimentTypeList = experimentStation.getExperimentTypeList();
+        experimentTypeList.remove(experimentType);
+        updateOnRemoval();
+    }
+    
     @FacesConverter(forClass = ExperimentStation.class)
     public static class ExperimentStationControllerConverter implements Converter {
 

src/java/DmWebPortal/src/java/gov/anl/aps/dm/portal/controllers/StorageController.java

+package gov.anl.aps.dm.portal.controllers;
+
+import gov.anl.aps.dm.common.exceptions.DmException;
+import gov.anl.aps.dm.common.exceptions.InvalidRequest;
+import gov.anl.aps.dm.common.exceptions.ObjectAlreadyExists;
+import gov.anl.aps.dm.portal.model.beans.StorageDbFacade;
+import gov.anl.aps.dm.portal.model.entities.Storage;
+import javax.ejb.EJB;
+import javax.inject.Named;
+import javax.enterprise.context.SessionScoped;
+import javax.faces.component.UIComponent;
+import javax.faces.context.FacesContext;
+import javax.faces.convert.Converter;
+import javax.faces.convert.FacesConverter;
+import org.apache.log4j.Logger;
+
+@Named("storageController")
+@SessionScoped
+public class StorageController extends DmEntityController<Storage, StorageDbFacade> {
+
+    private static final Logger logger = Logger.getLogger(StorageController.class.getName());
+
+    @EJB
+    private StorageDbFacade storageDbFacade;
+
+    public StorageController() {
+    }
+
+    @Override
+    protected StorageDbFacade getFacade() {
+        return storageDbFacade;
+    }
+
+    @Override
+    protected Storage createEntityInstance() {
+        return new Storage();
+    }
+
+    @Override
+    public String getEntityTypeName() {
+        return "storage";
+    }
+
+    @Override
+    public String getCurrentEntityInstanceName() {
+        if (getCurrent() != null) {
+            return getCurrent().getName();
+        }
+        return "";
+    }
+
+    @Override
+    public Storage findById(Integer id) {
+        return storageDbFacade.findById(id);
+    }
+    
+    @Override
+    public void prepareEntityInsert(Storage storage) throws DmException {
+        if ((storage.getName() == null) || (storage.getName().length() == 0)) {
+            throw new InvalidRequest("Storage name is missing.");
+        }
+        Storage existingStorage = storageDbFacade.findByName(storage.getName());
+        if (existingStorage != null) {
+            throw new ObjectAlreadyExists("Storage " + storage.getName() + " already exists.");
+        }
+        logger.debug("Inserting new storage " + storage.getName());
+    }
+
+    @Override
+    public void prepareEntityUpdate(Storage storage) throws DmException {
+        if ((storage.getName() == null) || (storage.getName().length() == 0)) {
+            throw new InvalidRequest("Storage name is missing.");
+        }
+        logger.debug("Updating storage " + storage.getName());
+    }
+
+    @Override
+    protected String getObjectAlreadyExistMessage(Storage storage) {
+        if (storage == null) {
+            return null;
+        }
+        return "Storage system " + storage.getName() + " already exists.";
+    }
+
+    @FacesConverter(forClass = Storage.class)
+    public static class StorageControllerConverter implements Converter {
+
+        @Override
+        public Object getAsObject(FacesContext facesContext, UIComponent component, String value) {
+            if (value == null || value.length() == 0) {
+                return null;
+            }
+            StorageController controller = (StorageController) facesContext.getApplication().getELResolver().
+                    getValue(facesContext.getELContext(), null, "storageController");
+            return controller.getEntity(getKey(value));
+        }
+
+        java.lang.Integer getKey(String value) {
+            java.lang.Integer key;
+            key = Integer.valueOf(value);
+            return key;
+        }
+
+        String getStringKey(java.lang.Integer value) {
+            StringBuilder sb = new StringBuilder();
+            sb.append(value);
+            return sb.toString();
+        }
+
+        @Override
+        public String getAsString(FacesContext facesContext, UIComponent component, Object object) {
+            if (object == null) {
+                return null;
+            }
+            if (object instanceof Storage) {
+                Storage o = (Storage) object;
+                return getStringKey(o.getId());
+            } else {
+                throw new IllegalArgumentException("Object " + object + " is of type " + object.getClass().getName() + "; expected type: " + Storage.class.getName());
+            }
+        }
+
+    }
+
+}

src/java/DmWebPortal/src/java/gov/anl/aps/dm/portal/model/beans/StorageDbFacade.java

@@ -8,6 +8,7 @@ package gov.anl.aps.dm.portal.model.beans;
 import gov.anl.aps.dm.portal.model.entities.Storage;
 import javax.ejb.Stateless;
 import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
 import javax.persistence.PersistenceContext;
 
 /**
@@ -28,5 +29,25 @@ public class StorageDbFacade extends DmEntityDbFacade<Storage> {
     public StorageDbFacade() {
         super(Storage.class);
     }
-    
+
+    public Storage findByName(String name) {
+        try {
+            return (Storage) em.createNamedQuery("Storage.findByName")
+                    .setParameter("name", name)
+                    .getSingleResult();
+        }
+        catch (NoResultException ex) {
+        }
+        return null;
+    }
+
+    public Storage findById(Integer id) {
+        try {
+            return (Storage) em.createNamedQuery("Storage.findById")
+                    .setParameter("id", id)
+                    .getSingleResult();
+        } catch (NoResultException ex) {
+        }
+        return null;
+    }        
 }

src/java/DmWebPortal/src/java/gov/anl/aps/dm/portal/model/beans/UsersLastUpdateDbFacade.java

-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
-package gov.anl.aps.dm.portal.model.beans;
-
-import gov.anl.aps.dm.portal.model.entities.UsersLastUpdate;
-import javax.ejb.Stateless;
-import javax.persistence.EntityManager;
-import javax.persistence.PersistenceContext;
-
-/**
- *
- * @author sveseli
- */
-@Stateless
-public class UsersLastUpdateDbFacade extends DmEntityDbFacade<UsersLastUpdate> {
-
-    @PersistenceContext(unitName = "DmWebPortalPU")
-    private EntityManager em;
-
-    @Override
-    protected EntityManager getEntityManager() {
-        return em;
-    }
-
-    public UsersLastUpdateDbFacade() {
-        super(UsersLastUpdate.class);
-    }
-    
-}