Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
# Need 1 terminal on DAQ node, 1 terminal on HPC node and 2 terminals on DS
# node
#
# SCENARIO 1: BASIC UPLOAD
#
# ssh -X dm@dmstorage: start services
cd /opt/DM/dev
source setup.sh
./etc/init.d/dm-postgresql start
./etc/init.d/dm-glassfish start
./etc/init.d/dm-mongodb start
./etc/init.d/dm-ds-web-service start
./etc/init.d/dm-cat-web-service start
# ssh -X dm@dmdaq: start services
cd /opt/DM/dev
source setup.sh
./etc/init.d/dm-daq-web-service start
# ssh -X dm@dmhpc: start services, check NFS
cd /opt/DM/dev
source setup.sh
./etc/init.d/dm-daq-web-service start
ls -l /net/dmstorage/opt/DM
#####################################################################
# dm@dmstorage: check directory content on the storage node
ls -l /opt/DM/data
# ssh sveseli@dmstorage: add and start experiment e1
source /opt/DM/etc/dm.setup.sh
dm-add-experiment --name e1 --type-id 1
dm-start-experiment --name e1
# dm@dmstorage: check directory content on the storage node
# note that experiment directory permissions are restricted
ls -l /opt/DM/data/ESAF
ls -l /opt/DM/data/ESAF/e1/
# ssh dm@dmdaq: source setup file, show test data
source /opt/DM/etc/dm.setup.sh
ls -lR /opt/DM/experiments/e1
cat /opt/DM/experiments/e1/file1
# dm@dmdaq: upload data for experiment e1
dm-upload --experiment e1 --data-directory /opt/DM/experiments/e1
# dm@dmstorage: check experiment storage directory content
# note permissions, ownership
ls -l /opt/DM/data/ESAF/e1/
ls -l /opt/DM/data/ESAF/e1/2015/07/09/
#
# SCENARIO 2: UPLOAD + METADATA CATALOG
#
# sveseli@dmstorage: get metadata for experiment files from catalogging service
dm-get-experiment-files --experiment e1
dm-get-experiment-file --experiment e1 --file file2 --display-keys=__all__
# dm@dmdaq: upload data for experiment e1, this time specify extra keys
dm-upload --experiment e1 --data-directory /opt/DM/experiments/e1 ownerUser:JohnC ownerGroup:APSU memo1:ApprovedByNDA memo2:DislikedByGD
# sveseli@dmstorage: get metadata for file 2 again
dm-get-experiment-file --experiment e1 --file file2 --display-keys=__all__
# sveseli@dmstorage: show metadata updates
dm-update-experiment-file --experiment e1 --file file3 quality:A --display-keys=id,fileName,quality
# sveseli@dmstorage: show metadata search
dm-get-experiment-files --experiment e1 quality:A
dm-get-experiment-files --experiment e1 storageFilePath:2015
#
# SCENARIO 3: UPLOAD + METADATA CATALOG + SDDS PARAMETERS
#
# sveseli@dmstorage: add and start experiment mm1
dm-add-experiment --name mm1 --type-id 1
dm-start-experiment --name mm1
# dm@dmdaq: upload data for experiment mm1, and request SDDS parameter
# processing
ls -lR /opt/DM/experiments/mm1
dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 ownerUser:JohnC ownerGroup:APSU processSddsParameters:True
# sveseli@dmstorage: get mm1 files, observe SDDS parameters
dm-get-experiment-files --experiment mm1
dm-get-experiment-files --experiment mm1 --display-keys=__all__ --display-format=dict
# dm@dmstorage: compare with sddsprintout (permissions do not allow sveseli
# account to access file)
export PATH=$PATH:/opt/epics/extensions/bin/linux-x86_64/
sddsprintout -parameters /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf
#
# SCENARIO 4: UPLOAD + METADATA CATALOG + SDDS PARAMETERS + SCRIPT PROCESSING
#
# dm@dmstorage: show processing script
cat /opt/DM/processing/find_sdds_row_count.sh
/opt/DM/processing/find_sdds_row_count.sh /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf
# sveseli@dmstorage: get mm1 files, note no key processingScriptOutput
dm-get-experiment-files --experiment mm1 --display-keys=fileName,processingScriptOutput
# dm@dmdaq: upload data for experiment mm1, request SDDS parameter
# processing, specify processing script
dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 processSddsParameters:True processingScript:/opt/DM/processing/find_sdds_row_count.sh
# sveseli@dmstorage: get mm1 files, note present key processingScriptOutput
dm-get-experiment-files --experiment mm1 --display-keys=fileName,processingScriptOutput
#
# SCENARIO 5: UPLOAD + METADATA CATALOG + SDDS PARAMETERS + HPC PROCESSING
#
# dm@dmstorage: show processing script
more /opt/DM/processing/sge_sdds_analysis.sh
# dm@dmstorage: show no png files in experiment directory
ls -l /opt/DM/data/ESAF/mm1/*.png
# dm@dmhpc: show empty home directory
cd
ls -l
# dm@dmhpc: show qstat
source /opt/sge/default/common/settings.sh
qstat -f
watch -d 'qstat -f'
# sveseli@dmstorage: get mm1 files, note only 1 file
dm-get-experiment-files --experiment mm1
# dm@dmdaq: upload data for experiment mm1, request SDDS parameter
# processing, specify SGE processing script
dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 processSddsParameters:True sgeJobScript:/opt/DM/processing/sge_sdds_analysis.sh
# sveseli@dmstorage: get mm1 files, note 2 files
dm-get-experiment-files --experiment mm1
# sveseli@dmstorage: get mm1 .png files, note parentFile key
dm-get-experiment-files --experiment mm1 fileName:.png --display-keys=__all__
# dm@dmhpc: show SGE output in home directory
ls -l
# dm@dmstorage: open processed file
xdg-open /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf.png
#
# SCENARIO 6: DAQ + METADATA CATALOG + SDDS PARAMETERS + HPC PROCESSING
#
# sveseli@dmstorage: add and start experiment mm2
dm-add-experiment --name mm2 --type-id 1
dm-start-experiment --name mm2
# sveseli@dmstorage: get mm2 files, note no files
dm-get-experiment-files --experiment mm2
# dm@dmstorage: show no png files in experiment directory
ls -l /opt/DM/data/ESAF/mm2/*.png
# dm@dmstorage: tail log file to observe processing
tail -f /opt/DM/var/log/dm.ds-web-service.log
# dm@dmdaq: start DAQ for experiment mm2, request SDDS parameter
# processing, specify SGE processing script
rm -rf /tmp/data/mm2
mkdir -p /tmp/data/mm2
dm-start-daq --experiment mm2 --data-directory /tmp/data/mm2 processSddsParameters:True sgeJobScript:/opt/DM/processing/sge_sdds_analysis.sh
# dm@dmhpc: show qstat
watch -d 'qstat -f'
# dm@dmdaq: copy experiment mm2 files into observed directory, watch qstat
ls -l /opt/DM/experiments/mm2/
cp /opt/DM/experiments/mm2/* /tmp/data/mm2/ && sleep 5 && touch /tmp/data/mm2/* &
tail -f /opt/DM/var/log/dm.daq-web-service.log
# sveseli@dmstorage: get mm2 files, note original + processed files
dm-get-experiment-files --experiment mm2
# dm@dmstorage: show png files in experiment directory
ls -l /opt/DM/data/ESAF/mm2/*.png
# dm@dmdaq: stop DAQ for experiment mm2
dm-stop-daq --experiment mm2
#
# SCENARIO 7: DATASET DEFINITION
#
# sveseli@dmstorage: add metadata for couple of experiment e2 files
# with different keys
dm-add-experiment-file --experiment e2 --file x1 status:good
dm-add-experiment-file --experiment e2 --file y1 status:bad
dm-get-experiment-files --experiment e2 --display-keys=fileName,status
# sveseli@dmstorage: add dataset metadata
dm-add-experiment-dataset --experiment e2 --dataset d1 status:g.*
# sveseli@dmstorage: get dataset files, note only one file matches
dm-get-experiment-dataset-files --experiment e2 --dataset d1
# sveseli@dmstorage: add metadata for anothare e2 file that
# should match dataset constraint
dm-add-experiment-file --experiment e2 --file x2 status:great
dm-get-experiment-files --experiment e2 --display-keys=fileName,status
# sveseli@dmstorage: get dataset files, note two files match
dm-get-experiment-dataset-files --experiment e2 --dataset d1