Skip to content

Commit

Permalink
Merge pull request #16 from CelsoReyes/working
Browse files Browse the repository at this point in the history 
Working
  • Loading branch information
@CelsoReyes
CelsoReyes authored Jan 7, 2019
2 parents fd9f2c5 + 1b0414e commit db1a28f
Show file tree
Hide file tree
Showing 14 changed files with 519 additions and 266 deletions.
102 changes: 102 additions & 0 deletions MyProgressMonitorNew.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
classdef MyProgressMonitorNew < matlab.net.http.ProgressMonitor
properties
ProgHandle
Direction matlab.net.http.MessageType
Value uint64
NewDir matlab.net.http.MessageType = matlab.net.http.MessageType.Request
end

methods
function obj = MyProgressMonitorNew
obj.Interval = .01;
end

function done(obj)
obj.closeit();
end

function delete(obj)
obj.closeit();
end

function set.Direction(obj, dir)
obj.Direction = dir;
obj.changeDir();
end

function set.Value(obj, value)
obj.Value = value;
obj.update();
end
end

methods (Access = private)
function update(obj,~)
% called when Value is set
import matlab.net.http.*
if ~isempty(obj.Value)
if isempty(obj.Max)
% no maximum means we don't know length, so message changes on
% every call
value = 0;
if obj.Direction == MessageType.Request
msg = sprintf('Sent %d bytes...', obj.Value);
else
msg = sprintf('Received %d bytes...', obj.Value);
end
else
% maximum known; update proportional value
value = double(obj.Value)/double(obj.Max);
if obj.NewDir == MessageType.Request
% message changes only on change of direction
if obj.Direction == MessageType.Request
msg = 'Sending...';
else
msg = 'Receiving...';
end
end
end
if isempty(obj.ProgHandle)
% if we don't have a progress bar, display it for first time
obj.ProgHandle = ...
waitbar(value, msg, 'CreateCancelBtn', @(~,~)cancelAndClose(obj));

obj.NewDir = MessageType.Response;
elseif obj.NewDir == MessageType.Request || isempty(obj.Max)
% on change of direction or if no maximum known, change message
waitbar(value, obj.ProgHandle, msg);
obj.NewDir = MessageType.Response;
else
% no direction change else just update proportional value
waitbar(value, obj.ProgHandle);
end
end

function cancelAndClose(obj)
% Call the required CancelFcn and then close our progress bar. This is
% called when user clicks cancel or closes the window.
obj.CancelFcn();
obj.closeit();
end
end

function changeDir(obj,~)
% Called when Direction is set or changed. Leave the progress bar displayed.
obj.NewDir = matlab.net.http.MessageType.Request;
end
end

methods (Access=private)
function closeit(obj)
% Close the progress bar by deleting the handle so CloseRequestFcn isn't
% called, because waitbar calls our cancelAndClose(), which would cause
% recursion.
if ~isempty(obj.ProgHandle)
delete(obj.ProgHandle);
obj.ProgHandle = [];
end
end
end
end


222 changes: 105 additions & 117 deletions importfilters/import_fdsn_event.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,125 +127,58 @@
disp(['sending request to:' baseurl 'query with options'])
disp(varargin)

hasParallel=license('test','Distrib_Computing_Toolbox') && ZG.ParallelProcessingOpts.Enable;
try
if hasParallel
fn = tempname;
p=gcp();
f=parfeval(p,@websave,1,fn, [baseurl 'query'], varargin{:},'format','text',options);
warning('off','MATLAB:table:ModifiedAndSavedVarnames');
nQuakes = 0;
lastsize = 0;
infmt = string(missing);
earliest = datetime(missing);
latest=datetime(missing);
myfig = figure('Name','Downloading events');
ax1=subplot(2,1,1);
scDate=scatter(ax1,NaT,nan,'.');title(ax1,'Date of downloaded events');
ax2=subplot(2,1,2);
scLoc=scatter(ax2,nan,nan,'.');title(ax2,'Location of downloaded events');
while f.State=="running"
startDownloadTime = f.StartDateTime;
pause(1);
d=dir(fn);
if isempty(d) || d.bytes == lastsize
fprintf('.');
continue
end
lastsize=d.bytes;
rightnow = datetime;
rightnow.TimeZone = f.StartDateTime.TimeZone;
elapsedTime = rightnow - f.StartDateTime;
bytes_per_second = lastsize / seconds(elapsedTime);
try
tb=readtable(fn);
catch ME
continue
end
if ~isempty(tb) && any([ismissing(tb.Time(end)) ismissing(tb.Magnitude(end))])
tb(end,:)=[];
end
if isempty(tb)
continue
end
events_per_minute = height(tb) / minutes(elapsedTime);
if ismissing(infmt)
infmt = "uuuu-MM-dd'T'HH:mm:ss";
infmt{1}([5 8])=tb.Time{1}(5);
infmt{1}(12)=tb.Time{1}(11);
end
tb.Time = datetime(extractBefore(tb.Time,20),'InputFormat',infmt);
if isvalid(scDate)
set(scDate,'XData',tb.Time,'YData',tb.Magnitude);
end
if isvalid(scLoc)
set(scLoc,'XData',tb.Longitude,'YData',tb.Latitude);
end
drawnow nocallbacks
nQuakes=height(tb);
minTime = min(tb.Time);
maxTime=max(tb.Time);
minMag=min(tb.Magnitude);
maxMag=max(tb.Magnitude);
if minTime ~= earliest
earliest=minTime;
earlyStr = "<strong>"+string(earliest)+"</strong>";
else
earlyStr=string(earliest);
end
if maxTime ~= latest
latest=maxTime;
lateStr = "<strong>"+string(earliest)+"</strong>";
else
lateStr=string(latest);
end
timespan_per_minute = (latest - earliest) / minutes(elapsedTime);
if timespan_per_minute > years(1)
timespan_per_minute.Format='y';
elseif timespan_per_minute > days(5)
timespan_per_minute.Format='d';
end

msg.dbfprintf("[<strong>%d</strong> events found... still downloading ]\n"+...
" Download Statistics:\n"+...
" Elapsed Time : <strong>%s</strong>\n"+...
" Bytes / sec : %d\n"+...
" Events / min : %d\n"+...
" Timespan / min : %s\n\n" +...
" Catalog Statistics:\n"+...
" start time : %s\n"+...
" end time : %s\n"+...
" magnitude range : [%g to %g]\n",...
nQuakes, elapsedTime,...
round(bytes_per_second), round(events_per_minute), string(timespan_per_minute),...
earlyStr, lateStr, min(tb.Magnitude), max(tb.Magnitude));
fprintf('\nwaiting for next update');
% hasParallel=license('test','Distrib_Computing_Toolbox') && ZG.ParallelProcessingOpts.Enable;
myuri=[baseurl, 'query?', sprintf('%s=%s&',string(varargin)), 'format=text'];
resp = get_low_level_fdsn_query(myuri);
switch resp.StatusCode
case "OK"
ok=true;
case "NoContent"
warning("No Data was found");
uOutput=[];
ok=false;
case "BadRequest"
disp(resp.Body.Data)

% as of 2018-12-14, USGS returns this result when limit is exceeded. these depend on the error message wording
maxSearchLimit = double(extractBefore(extractAfter(resp.Body.Data,'exceeds search limit of '),'.'));
nFound = double(extractBefore(extractAfter(resp.Body.Data,'exceeds search limit of '),'.'));
if ~ismissing(maxSearchLimit)
warning("maximum number of events [%d] exceeded. atttempting to limit results", maxSearchLimit);
% try again, in chunks of maxSearchLimit
disp('* trying again while limiting results')
[resp, ok] = get_in_chunks(myuri, maxSearchLimit, nFound);
else
uOutput=[];
ok=false;
end
msg.dbfprintf('\nDone finding events. Final total: %d\n', nQuakes);
warning('on','MATLAB:table:ModifiedAndSavedVarnames');
fetchOutputs(f);
data = fileread(fn);
delete(fn);
if isvalid(myfig)
close(myfig)

case "PayloadTooLarge"
disp(resp.Body.Data)

% as of 2018-12-14, INGV returns this result when limit is exceeded. these depend on the error message wording
nFound = double(extractBefore(extractAfter(resp.Body.Data,'the number of requested events is "'),'";'));
maxSearchLimit = double(extractBefore(extractAfter(resp.Body.Data,'your request must be less then "'),'" events.'));
if ~ismissing(maxSearchLimit)
warning("maximum number of events [%d] exceeded. atttempting to limit results", maxSearchLimit);
% try again, in chunks of maxSearchLimit
disp('* trying again while limiting results')
[resp, ok] = get_in_chunks(myuri, maxSearchLimit, nFound);
else
uOutput=[];
ok=false;
end
else
data = webread([baseurl 'query'], varargin{:},'format','text',options);
end
catch ME
cancel(f)
if isvalid(myfig)
close(myfig)
end
switch ME.identifier
%case 'MATLAB:webservices:CopyContentToDataStreamError'
otherwise
txt = 'An error occurred attempting to reach the FDSN web services';
errordlg(sprintf('%s\n\n%s\n\nidentifier: ''%s''', txt, ME.message, ME.identifier),...
'Error retrieving data');
end
uOutput=[];
ok=false;

otherwise
warning("there was some sort of problem. The response follows")
disp(resp)
warning(resp.Body.Data)
uOutput=[];
ok=false;
end
if ok
data = char(resp.Body.Data);
else
return
end

Expand Down Expand Up @@ -369,4 +302,59 @@

end

function [resp,ok] = get_in_chunks(myuri, maxAllowed, expected)
% try again, in chunks of maxSearchLimit
starts=1:maxAllowed:expected;
uOutput='';
for n=starts
fprintf(' ** retrieving events %d to %d\n', n, min(n+maxAllowed-1, expected));
resp = get_low_level_fdsn_query(myuri + "&offset=" + n + "&limit="+ maxAllowed);
if resp.StatusCode == "OK"
if isempty(uOutput)
uOutput=string(resp.Body.Data);
else
uOutput=uOutput + newline + extractAfter(resp.Body.Data, newline);
end
ok=true;
else
ok=false;
break
end
end
if ok
resp.Body.Data=uOutput;
end
end

function resp = get_low_level_fdsn_query(uri)
U = matlab.net.URI(uri);
method = matlab.net.http.RequestMethod.GET;
type1 = matlab.net.http.MediaType('text/*');
acceptField = matlab.net.http.field.AcceptField(type1);
uafield=matlab.net.http.HeaderField('UserAgent','MATLAB 9.4.0.949201 (R2018a) Update 6 ZMAP/7.1');
contentTypeField = matlab.net.http.field.ContentTypeField('text/plain');
header = [acceptField contentTypeField uafield];
request = matlab.net.http.RequestMessage(method,header);

consumer=matlab.net.http.io.StringConsumer;
options=matlab.net.http.HTTPOptions(...'SavePayload',true ...
'ProgressMonitorFcn',@MyProgressMonitorNew ...
,'UseProgressMonitor',true ...
,'ConnectTimeout',30 ... % in seconds
);
[resp,req,hist] = request.send(U,options,consumer);
%{
% if there is an error, it would be shown in hist.Response.Body.Data
ss=strsplit(string(resp.Body.Data),newline)';
numel(ss)
%%
f=fopen('junkk.dat','w');
fprintf(f,"%s",resp.Body.Data); %resp.Body.Payload
fclose(f);
%%
ZG.primeCatalog = import_fdsn_event(1,'junk.dat')
% ZmapMainWindow(ZG.primeCatalog)
%}
end

33 changes: 33 additions & 0 deletions sample_low_level_http_request.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
% dealing with fdsn data more-or-less directly
% this is a temporary script, and should be rolled into the
% FDSN import routines
% with the anticipation that matlab.net.http.ProgressMonitor
% will be used to provide feedback.

% U = matlab.net.URI('http://service.iris.edu/fdsnws/event/1/query?starttime=2018-01-11T00:00:00&orderby=time&format=text&nodata=404');
tic
U = matlab.net.URI('http://service.iris.edu/fdsnws/event/1/query?starttime=2018-09-11T00:00:00&orderby=time&format=text&nodata=404');
method = matlab.net.http.RequestMethod.GET;
type1 = matlab.net.http.MediaType('text/*');
acceptField = matlab.net.http.field.AcceptField([type1]);
contentTypeField = matlab.net.http.field.ContentTypeField('text/plain');
header = [acceptField contentTypeField];
request = matlab.net.http.RequestMessage(method,header);

consumer=matlab.net.http.io.StringConsumer;

[resp,req,hist] = request.send(U,matlab.net.http.HTTPOptions('SavePayload',true,'ProgressMonitorFcn',@MyProgressMonitorNew,'UseProgressMonitor',true),consumer);
% show(request)
% resp.show

% if there is an error, it would be shown in hist.Response.Body.Data
ss=strsplit(string(char(resp.Body.Data')),newline)';
numel(ss)
%%
f=fopen('junkk.dat','w');
fprintf(f,"%s",resp.Body.Data); %resp.Body.Payload
fclose(f);
%%
ZG.primeCatalog = import_fdsn_event(1,'junk.dat')
% ZmapMainWindow(ZG.primeCatalog)
toc
2 changes: 1 addition & 1 deletion src/+XYfun/stressgrid.m
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
classdef stressgrid < ZmapHGridFunction
% STRESSGRID calculate stress grid for event that have Dip, DipDirection, and Rake
properties

calcmethod
end

properties(Constant)
Expand Down
1 change: 1 addition & 0 deletions src/@ZmapMainWindow/catalog_menu.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ function cb_recall(~,~)

hh=msgbox_nobutton('The catalog has been recalled.','Recall Catalog');
hh.delay_for_close(1);
%obj.replot_all();
else
warndlg('No catalog is currently memorized','Recall Catalog');
end
Expand Down
Loading

0 comments on commit db1a28f

Please sign in to comment.