diff --git a/cadc-registry/build.gradle b/cadc-registry/build.gradle index 5466381..e4f8b00 100644 --- a/cadc-registry/build.gradle +++ b/cadc-registry/build.gradle @@ -15,13 +15,13 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '1.9.1' +version = '1.9.2' description = 'OpenCADC Registry client library' def git_url = 'https://github.com/opencadc/reg' dependencies { - implementation 'org.opencadc:cadc-util:[1.12,)' + implementation 'org.opencadc:cadc-util:[1.12.17,)' testImplementation 'junit:junit:[4.12,)' testImplementation 'org.opencadc:cadc-quality:[1.0,)' diff --git a/cadc-registry/src/main/java/ca/nrc/cadc/reg/Capability.java b/cadc-registry/src/main/java/ca/nrc/cadc/reg/Capability.java index 2b143fa..41fe430 100644 --- a/cadc-registry/src/main/java/ca/nrc/cadc/reg/Capability.java +++ b/cadc-registry/src/main/java/ca/nrc/cadc/reg/Capability.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2019. (c) 2019. +* (c) 2026. (c) 2026. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -112,6 +112,21 @@ public List getInterfaces() { return interfaces; } + /** + * Find the interface with the specified type. If the arg is null, find and return + * the first interface. + * @param interfaceType Standards.INTERFACE_ type constant or null for first interface + * @return matching interface or null if not found + */ + public Interface findInterfaceByType(URI interfaceType) { + for (Interface i : interfaces) { + if (interfaceType == null || interfaceType.equals(i.getType())) { + return i; + } + } + return null; + } + /** * Find a ParamHTTP interface that uses the specified securityMethod. * This method returns the first matching interface. @@ -119,6 +134,7 @@ public List getInterfaces() { * @param securityMethod securityMethod to match * @return the first matching interface or null */ + @Deprecated public Interface findInterface(final URI securityMethod) { return findInterface(securityMethod, Standards.INTERFACE_PARAM_HTTP); } @@ -131,6 +147,7 @@ public Interface findInterface(final URI securityMethod) { * @param interfaceType interface type to match * @return the first matching interface or null */ + @Deprecated public Interface findInterface(final URI securityMethod, final URI interfaceType) { boolean anon = securityMethod == null || Standards.SECURITY_METHOD_ANON.equals(securityMethod); for (Interface intf : this.interfaces) { @@ -155,6 +172,7 @@ public Interface findInterface(final URI securityMethod, final URI interfaceType * @param authMethod AuthMethod to match * @return the first matching interface or null */ + @Deprecated public Interface findInterface(final AuthMethod authMethod) { return findInterface(authMethod, Standards.INTERFACE_PARAM_HTTP); } @@ -167,6 +185,7 @@ public Interface findInterface(final AuthMethod authMethod) { * @param interfaceType interface type to match * @return the first matching interface or null */ + @Deprecated public Interface findInterface(final AuthMethod authMethod, final URI interfaceType) { URI secMethod = Standards.getSecurityMethod(authMethod); // TODO: throw if non-null authMethod -> null securityMethod? diff --git a/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/CachingFile.java b/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/CachingFile.java index 51cf17d..b0e0243 100644 --- a/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/CachingFile.java +++ b/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/CachingFile.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2026. (c) 2026. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -71,6 +71,7 @@ import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.net.HttpTransfer; +import ca.nrc.cadc.net.RemoteServiceException; import ca.nrc.cadc.net.TransientException; import ca.nrc.cadc.profiler.Profiler; import java.io.ByteArrayOutputStream; @@ -80,6 +81,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.SocketException; import java.net.URL; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -111,13 +113,17 @@ public class CachingFile { // 10 minutes private static final int DEFAULT_EXPRIY_SECONDS = 10 * 60; - private int connectionTimeout = 3000; // millis + private int connectionTimeout = 12000; // millis private int readTimeout = 60000; // millis + private int maxRetries = 3; // for TransientException private File localCache; private URL remoteSource; private long expirySeconds; private File cacheDir; + + // response time of remote service + public Long responseTime = null; /** * Construct a caching file with the local file and @@ -162,7 +168,7 @@ public CachingFile(File localCache, URL remoteSource, long expirySeconds) { } /** - * HTTP connection timeout in milliseconds (default: 30000). + * HTTP connection timeout in milliseconds (default: 12000). * * @param connectionTimeout in milliseconds */ @@ -178,6 +184,15 @@ public void setConnectionTimeout(int connectionTimeout) { public void setReadTimeout(int readTimeout) { this.readTimeout = readTimeout; } + + /** + * Set the max number of retries for 503 errors. + * + * @param maxRetries maximum number of retries + */ + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } private File checkCacheDirectory(File cacheFile) { Profiler profiler = new Profiler(CachingFile.class); @@ -187,15 +202,11 @@ private File checkCacheDirectory(File cacheFile) { log.debug("cache file parent dir: " + dir); if (dir.exists() && !dir.isDirectory()) { - java.nio.file.Files.delete(dir.toPath()); + Files.delete(dir.toPath()); } if (!dir.exists()) { - // The NIO version seems to create the path properly, - // as opposed to the dir.mkdirs() - // jenkinsd 2017.03.17 - // - java.nio.file.Files.createDirectories(dir.toPath()); + Files.createDirectories(dir.toPath()); log.debug("Created directory " + dir); } return dir; @@ -206,7 +217,16 @@ private File checkCacheDirectory(File cacheFile) { } } - public String getContent() throws IOException { + /** + * Get the content. This method normally returns the content from the local cache + * and occasionally refreshes the cache from the remote URL (~10 min). + * @return the target content from the remote URL + * @throws IOException if local caching fails + * @throws RemoteServiceException if the call to the remote URL fails + * @throws TransientException if the call fails with a transient exception and retries have already been attempted + * @see setMaxRetries + */ + public String getContent() throws IOException, RemoteServiceException, TransientException { boolean cacheExists = localCache.exists() && localCache.canRead(); if (cacheExists && !hasExpired()) { @@ -221,6 +241,7 @@ public String getContent() throws IOException { } // read and cache the remote source + long t1 = System.currentTimeMillis(); try { // create a temp file in the directory File tmpFile = File.createTempFile(UUID.randomUUID().toString(), null, cacheDir); @@ -229,7 +250,7 @@ public String getContent() throws IOException { FileOutputStream fos = new FileOutputStream(tmpFile); try { loadRemoteContent(fos); - } catch (IOException | TransientException e) { + } catch (IOException e) { log.warn("Failed to cache " + remoteSource + " to " + cacheDir.getAbsolutePath() + " cause: " + e.getMessage()); log.debug("Deleting tmp cache file because download failed."); @@ -242,13 +263,14 @@ public String getContent() throws IOException { throw e; } finally { + this.responseTime = System.currentTimeMillis() - t1; try { fos.close(); } catch (IOException e) { log.warn("Failed to close output stream", e); } } - + // move the file to the real location Path source = Paths.get(tmpFile.getAbsolutePath()); Path dest = Paths.get(localCache.getAbsolutePath()); @@ -316,20 +338,27 @@ private String getRemoteContent() throws IOException { return out.toString("UTF-8"); } - private void loadRemoteContent(OutputStream dest) throws IOException, TransientException { + private void loadRemoteContent(OutputStream dest) throws IOException, RemoteServiceException, TransientException { Profiler profiler = new Profiler(CachingFile.class); try { + final long t1 = System.currentTimeMillis(); log.debug("loadRemoteContent: " + remoteSource + " " + connectionTimeout + "/" + readTimeout); HttpGet download = new HttpGet(remoteSource, dest); download.setConnectionTimeout(connectionTimeout); download.setReadTimeout(readTimeout); + download.setMaxRetries(maxRetries); download.run(); + long dt = System.currentTimeMillis() - t1; + log.debug("loadRemoteContent: dt=" + dt); if (download.getThrowable() != null) { log.warn("Could not get source from " + remoteSource + ": " + download.getThrowable()); if (download.getThrowable() instanceof TransientException) { throw (TransientException) download.getThrowable(); } + if (download.getThrowable() instanceof RemoteServiceException) { + throw (RemoteServiceException) download.getThrowable(); + } throw new IOException(download.getThrowable()); } } finally { @@ -343,5 +372,4 @@ private boolean hasExpired() { long now = System.currentTimeMillis(); return ((now - lastModified) > expiryMillis); } - } diff --git a/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/RegistryClient.java b/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/RegistryClient.java index 5d971da..f1d763b 100644 --- a/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/RegistryClient.java +++ b/cadc-registry/src/main/java/ca/nrc/cadc/reg/client/RegistryClient.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2026. (c) 2026. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -94,7 +94,10 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.apache.log4j.Logger; @@ -109,13 +112,12 @@ * @author pdowler */ public class RegistryClient { - private static Logger log = Logger.getLogger(RegistryClient.class); private static final String HOST_PROPERTY_KEY = RegistryClient.class.getName() + ".host"; private static final String CONFIG_BASE_URL_KEY = RegistryClient.class.getName() + ".baseURL"; - + public enum Query { APPLICATIONS("applications"), CAPABILITIES("resource-caps"); @@ -145,8 +147,10 @@ public String getValue() { private final List regBaseURLs = new ArrayList(); // private String capsDomain; private boolean isRegOverride = false; - private int connectionTimeout = 30000; // millis - private int readTimeout = 60000; // millis + private int connectionTimeout = 3000; // millis + private int readTimeout = 6000; // millis + private int readCapabilitiesTimeout = 5 * readTimeout; // millis + private int maxRetries = 1; /** * Default constructor, using DEFAULT_CONFIG_FILE_NAME. @@ -191,7 +195,7 @@ private RegistryClient(final PropertiesReader propReader) { } /** - * HTTP connection timeout in milliseconds (default: 30000). + * HTTP connection timeout in milliseconds (default: 3000). * * @param connectionTimeout in milliseconds */ @@ -200,14 +204,26 @@ public void setConnectionTimeout(int connectionTimeout) { } /** - * HTTP read timeout in milliseconds (default: 60000). + * HTTP read timeout in milliseconds (default: 6000 for canned queries). + * This also sets the timeout for reading capabilities documents to 5x + * the base timeout (default: 30000ms). * * @param readTimeout in milliseconds */ public void setReadTimeout(int readTimeout) { this.readTimeout = readTimeout; + this.readCapabilitiesTimeout = 5 * readTimeout; } - + + /** + * Maximum number of retries (default: 1) when encountering TransientException. + * + * @param maxRetries maximum number of retries + */ + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } + /** * Find out if registry lookup URL was modified by a system property. This * typically indicates that the code is running in a development/test environment. @@ -247,7 +263,6 @@ public URL getAccessURL(Query queryName, URI uri) throws ResourceNotFoundExcepti List exceptions = new ArrayList(); for (URL regBaseURL : regBaseURLs) { - try { log.debug("registry base URL [" + regBaseURL + "]"); File queryCacheFile = getQueryCacheFile(regBaseURL, queryName); @@ -259,6 +274,7 @@ public URL getAccessURL(Query queryName, URI uri) throws ResourceNotFoundExcepti CachingFile cachedCapSource = new CachingFile(queryCacheFile, queryURL); cachedCapSource.setConnectionTimeout(connectionTimeout); cachedCapSource.setReadTimeout(readTimeout); + cachedCapSource.setMaxRetries(1); String map = null ; try { @@ -291,9 +307,8 @@ public URL getAccessURL(Query queryName, URI uri) throws ResourceNotFoundExcepti ); } try { - return new URL( - values.get(0) - ); + URL ret = new URL(values.get(0)); + return ret; } catch (MalformedURLException e) { throw new RuntimeException( "Parsing accessURL [" + values.get(0) + "] from registry [" + regBaseURL + "] threw MalformedURLException [" + e.getMessage() + "]", @@ -319,8 +334,8 @@ public URL getAccessURL(Query queryName, URI uri) throws ResourceNotFoundExcepti * @param resourceID Identifies the resource. * @return The associated capabilities object. * - * @throws IOException If the capabilities could not be determined. - * @throws ca.nrc.cadc.net.ResourceNotFoundException if the resourceID cannot be found in the registry + * @throws IOException If the capabilities could not be read + * @throws ResourceNotFoundException if the resourceID cannot be found in the registry */ public Capabilities getCapabilities(URI resourceID) throws IOException, ResourceNotFoundException { if (resourceID == null) { @@ -335,12 +350,65 @@ public Capabilities getCapabilities(URI resourceID) throws IOException, Resource File capabilitiesFile = this.getCapabilitiesCacheFile(serviceCapsURL, resourceID); CachingFile cachedCapabilities = new CachingFile(capabilitiesFile, serviceCapsURL); cachedCapabilities.setConnectionTimeout(connectionTimeout); - cachedCapabilities.setReadTimeout(readTimeout); + cachedCapabilities.setReadTimeout(readCapabilitiesTimeout); + cachedCapabilities.setMaxRetries(1); String xml = cachedCapabilities.getContent(); CapabilitiesReader capReader = new CapabilitiesReader(); return capReader.read(xml); } + /** + * Convenience: get the accessURL for the specified capability (standardID) + * of a resource with the default interface type (Standards.INTERFACE_PARAM_HTTP). + * + * @param resourceID the resource identifier + * @param standardID the capability identifier + * @return the access URL or null if not found + */ + public URL getServiceURL(URI resourceID, URI standardID) { + return getServiceURL(resourceID, standardID, DEFAULT_ITYPE); + } + + /** + * Convenience: get the accessURL for the specified capability (standardID) + * of a resource. The interface type arg is optional and if null uses the + * default interface type (Standards.INTERFACE_PARAM_HTTP). + * + * @param resourceID the resource identifier + * @param standardID the capability identifier + * @param interfaceType the interface type identifier or null for default type + * @return the access URL or null if not found + */ + public URL getServiceURL(URI resourceID, URI standardID, URI interfaceType) { + if (resourceID == null || standardID == null) { + String msg = "No input parameters should be null"; + throw new IllegalArgumentException(msg); + } + if (interfaceType == null) { + interfaceType = DEFAULT_ITYPE; + } + + Capabilities caps = null; + try { + caps = getCapabilities(resourceID); + } catch (ResourceNotFoundException ex) { + log.debug("getCapabilities: " + ex); + return null; + } catch (IOException e) { + throw new RuntimeException("Could not obtain service URL", e); + } + + // locate the associated capability + Capability cap = caps.findCapability(standardID); + if (cap != null) { + Interface iface = cap.findInterfaceByType(interfaceType); + if (iface != null) { + return iface.getAccessURL().getURL(); + } + } + return null; + } + /** * Find the service URL for the service registered under the specified base resource * identifier and using the specified authentication method. The identifier must be an @@ -352,7 +420,9 @@ public Capabilities getCapabilities(URI resourceID) throws IOException, Resource * @param standardID IVOA standard identifier, e.g. ivo://ivo.net/std/TAP * @param authMethod authentication method to be used * @return service URL or null if a matching interface was not found + * @deprecated do not need to filter by AuthMethod */ + @Deprecated public URL getServiceURL(final URI resourceIdentifier, final URI standardID, final AuthMethod authMethod) { return getServiceURL(resourceIdentifier, standardID, authMethod, DEFAULT_ITYPE); } @@ -370,7 +440,9 @@ public URL getServiceURL(final URI resourceIdentifier, final URI standardID, fin * @param interfaceType Interface type indicating how to access the resource (e.g. HTTP). See IVOA * resource identifiers * @return service URL or null if a matching interface was not found + * @deprecated do not need to filter by AuthMethod */ + @Deprecated public URL getServiceURL(final URI resourceID, final URI standardID, final AuthMethod authMethod, URI interfaceType) { if (resourceID == null || standardID == null || interfaceType == null) { String msg = "No input parameters should be null"; diff --git a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerDroppedConnectionTests.java b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerDroppedConnectionTests.java index fb941ae..f9476a8 100644 --- a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerDroppedConnectionTests.java +++ b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerDroppedConnectionTests.java @@ -213,20 +213,20 @@ public void testMultipleDroppedConnection() } // - // The registry client should have called each of the bad 'resource-caps' endpoints at least twice, once to try to populate the cache, - // and a second time to try to get the content without using the cache. + // The registry client should have called each of the bad 'resource-caps' endpoints only once to populate the cache + // was: and a second time to try to get the content without using the cache but now this is a fatal RemoteServiceException // In fact MockServer will log multiple requests because the HttpGet library retries the request multiple times. mockServer.verify( request().withPath("/drop-connection-001/resource-caps"), - VerificationTimes.atLeast(2) + VerificationTimes.atLeast(1) ); mockServer.verify( request().withPath("/drop-connection-002/resource-caps"), - VerificationTimes.atLeast(2) + VerificationTimes.atLeast(1) ); mockServer.verify( request().withPath("/drop-connection-003/resource-caps"), - VerificationTimes.atLeast(2) + VerificationTimes.atLeast(1) ); // diff --git a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerTestBase.java b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerTestBase.java index 6a23290..c57b90c 100644 --- a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerTestBase.java +++ b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/MockServerTestBase.java @@ -23,6 +23,7 @@ package ca.nrc.cadc.reg.client; +import ca.nrc.cadc.auth.AuthMethod; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; @@ -51,6 +52,7 @@ import org.mockserver.model.MediaType; import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.StringUtil; import ca.nrc.cadc.xml.XmlUtil; @@ -215,12 +217,9 @@ protected RegistryClient buildRegistryClient(final String ...endpoints) protected void checkAvailibility(final URI targetIvoid, final RegistryClient registryClient ) throws Exception { + AuthMethod am = null; try { - URL endpoint = registryClient.getServiceURL( - targetIvoid, - new URI("ivo://ivoa.net/std/VOSI#availability"), - null - ); + URL endpoint = registryClient.getServiceURL(targetIvoid, Standards.VOSI_AVAILABILITY); assertNotNull("Null service endpoint",endpoint); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); diff --git a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/RegistryClientTest.java b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/RegistryClientTest.java index d1c6ecc..e8ffc63 100644 --- a/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/RegistryClientTest.java +++ b/cadc-registry/src/test/java/ca/nrc/cadc/reg/client/RegistryClientTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2026. (c) 2026. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -281,7 +281,12 @@ public void testGetServiceURLHappyPath() { URL serviceURL = rc.getServiceURL(resourceID, Standards.TAP_10, AuthMethod.ANON); Assert.assertNotNull("Service URL should not be null", serviceURL); Assert.assertEquals("got an incorrect URL", expected, serviceURL); -// Assert.assertNotNull("no caps domain", rc.getCapsDomain()); + + URI itype = null; + URL argusURL = rc.getServiceURL(resourceID, Standards.TAP_10, itype); + Assert.assertNotNull("Service URL should not be null", serviceURL); + Assert.assertEquals("got an incorrect URL", expected, serviceURL); + } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); @@ -336,7 +341,7 @@ public void testGetServiceURLModifyHostDeprecated() { URL resourceCapsURL = rc.getRegistryBaseURLs().get(0); Assert.assertNotNull("Service URL should not be null", resourceCapsURL); Assert.assertNotEquals(expected, resourceCapsURL); - Assert.assertEquals("wrong caps domain", "ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca", rc.getCapsDomain(resourceCapsURL)); + Assert.assertEquals("wrong caps domain", "cadc-west-01.canfar.net", rc.getCapsDomain(resourceCapsURL)); } catch (Exception t) { log.error("unexpected exception", t); Assert.fail("unexpected exception: " + t); diff --git a/cadc-registry/src/test/resources/cadc-registry.properties b/cadc-registry/src/test/resources/cadc-registry.properties index 3a93db5..1c02d3f 100644 --- a/cadc-registry/src/test/resources/cadc-registry.properties +++ b/cadc-registry/src/test/resources/cadc-registry.properties @@ -1,4 +1,5 @@ # configure RegistryClient +ca.nrc.cadc.reg.client.RegistryClient.baseURL = https://cadc-west-01.canfar.net/reg ca.nrc.cadc.reg.client.RegistryClient.baseURL = https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/reg # local authority map