You are viewing an old version of this page. View the current version.
Compare with Current View Page History
« Previous Version 3 Next »
This example creates a request for a Time Series Scalar Data data product for the Barkley Canyon / Axis (POD 1) 150 kHz ADCP and returns information about the request. The returns a Request Id, which can be used to run the data product.
Python
import requests
import json
url = 'http://dmas.uvic.ca/api/dataProductDelivery'
parameters = {'method':'request',
'token':'YOUR_TOKEN_HERE', # replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
'locationCode':'BACAX', # Barkley Canyon / Axis (POD 1)
'deviceCategoryCode':'ADCP2MHZ', # 150 kHz Acoustic Doppler Current Profiler
'dataProductCode':'TSSD', # Time Series Scalar Data
'extension':'csv', # Comma Separated spreadsheet file
'begin':'2016-07-27T00:00:00.000Z',
'end':'2016-08-01T00:00:00.000Z',
'dpo_qualityControl':1,
'dpo_resample':'none',
'dpo_dataGaps':0}
response = requests.get(url,params=parameters)
if (response.ok):
requestInfo = json.loads(str(response.content,'utf-8')) # convert the json response to an object
print('Request Id: {}'.format(requestInfo['dpRequestId'])) # Print the Request Id
if ('numFiles' in requestInfo.keys()):
print('File Count: {}'.format(requestInfo['numFiles'])) # Print the Estimated File Size
if ('fileSize' in requestInfo.keys()):
print('File Size: {}'.format(requestInfo['fileSize'])) # Print the Estimated File Size
if 'downloadTimes' in requestInfo.keys():
print('Estimated download time:')
for e in sorted(requestInfo['downloadTimes'].items(),key=lambda t: t[1]):
print(' {} - {} sec'.format(e[0],'{:0.2f}'.format(e[1])))
if 'estimatedFileSize' in requestInfo.keys():
print('Estimated File Size: {}'.format(requestInfo['estimatedFileSize']))
if 'estimatedProcessingTime' in requestInfo.keys():
print('Estimated Processing Time: {}'.format(requestInfo['estimatedProcessingTime']))
else:
if(response.status_code == 400):
error = json.loads(str(response.content,'utf-8'))
print(error) # json response contains a list of errors, with an errorMessage and parameter
else:
print ('Error {} - {}'.format(response.status_code,response.reason))
MatLab
options = weboptions('Timeout',60);
try
requestInfo = webread('http://dmas.uvic.ca/api/dataProductDelivery',...
'method','request',...
'token','YOUR_TOKEN_HERE',... %replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
'locationCode','BACAX',... %Barkley Canyon / Axis (POD 1)
'deviceCategoryCode','ADCP2MHZ',... %150 kHz Acoustic Doppler Current Profiler
'dataProductCode','TSSD',... %Time Series Scalar Data
'extension','csv',... %Comma Separated spreadsheet file
'begin','2016-07-27T00:00:00.000Z',...
'end','2016-08-01T00:00:00.000Z',...
'dpo_qualityControl',1,...
'dpo_resample','none',...
'dpo_dataGaps',0,...
options);
disp(requestInfo)
catch ME
if (strcmp(ME.identifier,'MATLAB:webservices:HTTP400StatusCodeError'))
disp(ME.message);
else
disp(ME);
end
end
R
library(httr)
r <- GET("http://dmas.uvic.ca/api/dataProductDelivery",
query = list(method="request",
token="YOUR_TOKEN_HERE", #>replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
locationCode="BACAX", #>Barkley Canyon / Axis (POD 1)
deviceCategoryCode="ADCP2MHZ", #>150 kHz Acoustic Doppler Current Profiler
dataProductCode="TSSD", #>Time Series Scalar Data
extension="csv", #>Comma Separated spreadsheet file
begin="2016-07-27T00:00:00.000Z",
end="2016-08-01T00:00:00.000Z",
dpo_qualityControl=1,
dpo_resample="none",
dpo_dataGaps=0))
if (http_error(r)) {
if (r$status_code == 400){
error = content(r)
str(error)
} else {
str(http_status(r)$message)
}
} else {
requestInfo = content(r)
str(requestInfo)
}
This example runs a data product request using a Request Id returned from the 'Request Data Product' example
Python
import requests
import json
url = 'http://dmas.uvic.ca/api/dataProductDelivery'
parameters = {'method':'run',
'token':'YOUR_TOKEN_HERE', # replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
'dpRequestId':YOUR_REQUEST_ID_HERE} # replace YOUR_REQUEST_ID_HERE with a requestId number returned from the request method
response = requests.get(url,params=parameters)
if (response.ok):
runs = json.loads(str(response.content,'utf-8')) # convert the json response to an object
for runId in runs['dpRunIds']:
print('Run Id: {}'.format(runId)) # Print each of the Run Ids
else:
if(response.status_code == 400):
error = json.loads(str(response.content,'utf-8'))
print(error) # json response contains a list of errors, with an errorMessage and parameter
else:
print ('Error {} - {}'.format(response.status_code,response.reason))
MatLab
options = weboptions('Timeout',60);
try
runs = webread('http://dmas.uvic.ca/api/dataProductDelivery',...
'method','run',...
'token','YOUR_TOKEN_HERE',... %replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
'dpRequestId',YOUR_REQUEST_ID_HERE,... %replace YOUR_REQUEST_ID_HERE with a requestId number returned from the request method
options);
for i=1:numel(runs)
run = runs(i);
disp(sprintf('Run Id: %s',run));
end
catch ME
if (strcmp(ME.identifier,'MATLAB:webservices:HTTP400StatusCodeError'))
disp(ME.message);
else
disp(ME);
end
end
R
library(httr)
r <- GET("http://dmas.uvic.ca/api/dataProductDelivery",
query = list(method="run",
token="YOUR_TOKEN_HERE", #>replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
dpRequestId=YOUR_REQUEST_ID_HERE)) #>replace YOUR_REQUEST_ID_HERE with a requestId number returned from the request method
if (http_error(r)) {
if (r$status_code == 400){
error = content(r)
str(error)
} else {
str(http_status(r)$message)
}
} else {
runs = content(r)
for (run in runs){
cat(sprintf("Run Id: %s",run))
}
}
This example downloads all of the files generated by a data product request using a Run Id returned from the 'Run Data Product Request' example
Python
import requests
import json
import os
def downloadDataProductIndex(self,
runId, # The ID of the run process to download the files for. RunIds are returned from the dataProductDelivery run method
url, # The full URL of the dataProductDelivery service
parameters, # The http request parameters for method, token, runId and index
indx=1, # The index of the file to be downloaded. Data files have an index of 1 or higher. The Metadata has an index of 'meta'
fileCount=1, # The actual or estimated file count, which is returned from the dataProductDelivery request method
maxRetries=100): # Determines the maximum number of times the process will poll the service before it times out.
# The purpose of this property is to prevent hung processes on the Task server to hang this process.
defaultSleepTime = 2
requestUrl = '{}?{}'.format(url,('&'.join(['{}={}'.format(i[0],i[1]) for i in parameters.items()])))
downloadResult = {"url":requestUrl}
tryCount = 0
lastMessage = None
if (estimatedProcessingTime > 1):
sleepTime = estimatedProcessingTime * 0.5
else:
sleepTime = defaultSleepTime
downloadResult['message'] = []
while True:
tryCount+=1
downloadResult['status'] = 'running'
if tryCount >= maxRetries:
print('Maximum number of retries ({}) exceeded'.format(maxRetries))
downloadResult['message'].append(msg)
break
with closing(requests.get(url,params=parameters,stream=True)) as streamResponse:
if (streamResponse.ok): #Indicates that the request was successful and did not fail. The status code indicates if the stream contains a file (200) or
if streamResponse.status_code == 200: #OK
tryCount=0
if 'Content-Disposition' in streamResponse.headers.keys():
content = streamResponse.headers['Content-Disposition']
filename = content.split('filename=')[1]
else:
print('Error: Invalid Header')
streamResponse.close()
break
if 'Content-Length' in streamResponse.headers.keys():
size = streamResponse.headers['Content-Length']
downloadResult['size'] = float(size)
else:
size = 0
filePath = '{}/{}'.format(self.outPath,filename)
downloadResult['file'] = filePath
downloadResult['index'] = indx
try:
if (indx==1):
print('')
if (not os.path.isfile(filePath)):
#Create the directory structure if it doesn't already exist
try:
os.makedirs(self.outPath)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(self.outPath):
pass
else:
raise
print (" Downloading {}/{} '{}' ({})".format(indx,fileCount,filename,util.convertSize(float(size))))
with open(filePath,'wb') as handle:
try:
for block in streamResponse.iter_content(1024):
handle.write(block)
except KeyboardInterrupt:
print('Process interupted: Deleting {}'.format(filePath))
handle.close()
streamResponse.close()
os.remove(filePath)
sys.exit(-1)
else:
if fileCount == 0:
print (" Skipping {} '{}': File Already Exists".format(indx,filename))
else:
print (" Skipping {}/{} '{}': File Already Exists".format(indx,fileCount,filename))
downloadResult['downloaded'] = True
downloadResult['status'] = 'complete'
except:
msg = 'Error streaming response.'
print(msg)
downloadResult['message'].append(msg)
downloadResult['status'] = 'error'
streamResponse.close()
break
elif streamResponse.status_code == 202: #Accepted - Result is not complete -> Retry
payload = json.loads(util.toString(streamResponse.content))
if len(payload) >= 1:
msg = payload['message']
if (msg != lastMessage): #display a new message if it has changed
util.printWithEnd('\n {}'.format(msg))
sys.stdout.flush()
downloadResult['message'].append(msg)
lastMessage=msg
tryCount=0
else: #Add a dot to the end of the message to indicate that it is still receiving the same message
if (not multiThreadMessages):
util.printWithEnd('.')
sys.stdout.flush()
else:
if (not multiThreadMessages):
print('Retrying...')
elif streamResponse.status_code == 204: #No Content - No Data found
if not(util.toString(streamResponse.content) == ''):
payload = json.loads(util.toString(streamResponse.content))
msg = ' {} [{}]'.format(payload['message'],streamResponse.status_code)
else:
msg = 'No Data found'
if multiThreadMessages:
with lock:
print('{}: {}'.format(runId,msg))
else:
print('\n{}'.format(msg))
streamResponse.close()
downloadResult['message'].append(msg)
downloadResult['status'] = 'complete'
break
else:
msg = 'HTTP Status: {}'.format(streamResponse.status_code)
if multiThreadMessages:
with lock:
print('{}: {}'.format(runId,msg))
else:
print(msg)
downloadResult['message'].append(msg)
elif streamResponse.status_code == 400: #Error occurred
if (self.showInfo): print(' HTTP Status: {}'.format(streamResponse.status_code))
payload = json.loads(util.toString(streamResponse.content))
if len(payload) >= 1:
if ('errors' in payload):
for e in payload['errors']:
msg = e['errorMessage']
util.printErrorMesasge(streamResponse,parameters)
elif ('message' in payload):
msg = ' {} [{}]'.format(payload['message'],streamResponse.status_code)
if (not multiThreadMessages):
print('\n{}'.format(msg))
else:
msg = 'Error occurred processing data product request'
if multiThreadMessages:
with lock:
print('{}: {}'.format(runId,msg))
else:
print(msg)
else:
msg = 'Error occurred processing data product request'
if multiThreadMessages:
with lock:
print('{}: {}'.format(runId,msg))
else:
print(msg)
streamResponse.close()
downloadResult['status'] = 'error'
downloadResult['message'].append(msg)
break
elif streamResponse.status_code == 404: #Not Found - Beyond End of Index - Index # > Results Count
streamResponse.close()
downloadResult = None
break
elif streamResponse.status_code == 410: #Gone - file does not exist on the FTP server. It may not have been transfered to the FTP server yet
payload = json.loads(util.toString(streamResponse.content))
if len(payload) >= 1:
msg = payload['message']
if (msg != lastMessage):
if multiThreadMessages:
with lock:
print('{}: Waiting... {}'.format(runId,msg))
else:
util.printWithEnd('\n Waiting... {}'.format(msg))
sys.stdout.flush()
lastMessage=msg
tryCount=0
else:
if (not multiThreadMessages):
util.printWithEnd('.','')
sys.stdout.flush()
else:
if multiThreadMessages:
with lock:
print('{}: Running... Writing File'.format(runId))
else:
print('\nRunning... Writing File.')
elif streamResponse.status_code == 500: #Internal Server Error occurred
msg = util.printErrorMesasge(streamResponse,parameters)
if (self.showInfo): print(' URL: {}'.format(streamResponse.url))
streamResponse.close()
downloadResult['status'] = 'error'
downloadResult['message'].append(msg)
break
else:
try:
payload = json.loads(util.toString(streamResponse.content))
if len(payload) >= 1:
if ('errors' in payload):
for e in payload['errors']:
msg = e['errorMessage']
util.printErrorMesasge(streamResponse,parameters)
elif ('message' in payload):
msg = payload['message']
if multiThreadMessages:
with lock:
print('{}: {} [{}]'.format(runId,msg,streamResponse.status_code))
else:
print('\n {} [{}]'.format(msg,streamResponse.status_code))
downloadResult['status'] = 'error'
downloadResult['message'].append(msg)
streamResponse.close()
break
except:
util.printErrorMesasge(streamResponse,parameters)
if multiThreadMessages:
with lock:
print('{}: {} Retrying...'.format(runId,msg))
else:
print('{} Retrying...'.format(msg))
streamResponse.close()
break
streamResponse.close()
if (tryCount <= 5) and (sleepTime > defaultSleepTime):
sleepTime = sleepTime * 0.5
time.sleep(sleepTime)
return downloadResult
MatLab
options = weboptions('Timeout',60);
try
runs = webread('http://dmas.uvic.ca/api/dataProductDelivery',...
'method','run',...
'token','YOUR_TOKEN_HERE',... %replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
'dpRequestId',YOUR_REQUEST_ID_HERE,... %replace YOUR_REQUEST_ID_HERE with a requestId number returned from the request method
options);
for i=1:numel(runs)
run = runs(i);
disp(sprintf('Run Id: %s',run));
end
catch ME
if (strcmp(ME.identifier,'MATLAB:webservices:HTTP400StatusCodeError'))
disp(ME.message);
else
disp(ME);
end
end
R
library(httr)
r <- GET("http://dmas.uvic.ca/api/dataProductDelivery",
query = list(method="run",
token="YOUR_TOKEN_HERE", #>replace YOUR_TOKEN_HERE with your personal token obtained from the 'Web Services API' tab at http://dmas.uvic.ca/Profile
dpRequestId=YOUR_REQUEST_ID_HERE)) #>replace YOUR_REQUEST_ID_HERE with a requestId number returned from the request method
if (http_error(r)) {
if (r$status_code == 400){
error = content(r)
str(error)
} else {
str(http_status(r)$message)
}
} else {
runs = content(r)
for (run in runs){
cat(sprintf("Run Id: %s",run))
}
}
- No labels