diff --git a/.gitattributes b/.gitattributes index fcadb2c..db4c048 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,8 @@ * text eol=lf + +*.java text + +*.nc binary +*.bz2 binary + + diff --git a/api/build.gradle b/api/build.gradle index 9e13b98..ac3342c 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -10,6 +10,9 @@ group 'org.fairdatapipeline' version '1.0.0-rc1' repositories { + maven { + url "https://artifacts.unidata.ucar.edu/repository/unidata-all/" + } mavenCentral() } @@ -27,6 +30,7 @@ javadoc { dependencies { implementation project(':dataregistry') + implementation 'org.jetbrains:annotations:20.1.0' annotationProcessor 'org.immutables:value:2.9.0' implementation 'commons-codec:commons-codec:1.15' implementation 'com.google.guava:guava:31.1-jre' @@ -47,6 +51,8 @@ dependencies { implementation 'com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.2' implementation 'org.slf4j:slf4j-api:1.7.36' implementation 'org.slf4j:slf4j-simple:1.7.36' + implementation 'edu.ucar:cdm-core:5.5.3' + implementation 'edu.ucar:netcdf4:5.5.3' testAnnotationProcessor 'org.immutables:value:2.9.0' testImplementation(platform('org.junit:junit-bom:5.8.1')) testImplementation('org.junit.jupiter:junit-jupiter:5.8.2') @@ -54,6 +60,7 @@ dependencies { testImplementation 'org.mockito:mockito-core:4.3.1' testImplementation 'org.assertj:assertj-core:3.22.0' testImplementation 'org.javatuples:javatuples:1.2' + testImplementation 'org.apache.commons:commons-compress:1.21' } test { diff --git a/api/src/main/java/org/fairdatapipeline/api/Author.java b/api/src/main/java/org/fairdatapipeline/api/Author.java index 1dc9de2..63d1e34 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Author.java +++ b/api/src/main/java/org/fairdatapipeline/api/Author.java @@ -22,14 +22,14 @@ class Author { (RegistryUsers) restClient.getFirst(RegistryUsers.class, Collections.singletonMap("username", "admin")); if (u == null) { - throw (new RegistryObjectNotfoundException("Couldn't find a User in the local registry!")); + throw (new RegistryObjectNotFoundException("Couldn't find a User in the local registry!")); } RegistryUser_author ua = (RegistryUser_author) restClient.getFirst( RegistryUser_author.class, Collections.singletonMap("user", u.get_id().toString())); if (ua == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "Couldn't find a User_author in the local registry!")); } this.registryAuthor = (RegistryAuthor) restClient.get(RegistryAuthor.class, ua.getAuthor()); diff --git a/api/src/main/java/org/fairdatapipeline/api/Coderun.java b/api/src/main/java/org/fairdatapipeline/api/Coderun.java index 0128de3..f92ea8c 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Coderun.java +++ b/api/src/main/java/org/fairdatapipeline/api/Coderun.java @@ -48,7 +48,7 @@ * ImmutableSamples samples = ImmutableSamples.builder().addSamples(1, 2, 3).rng(rng).build(); * String dataProduct = "animal/dodo"; * String component1 = "example-samples-dodo1"; - * Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + * Data_product_write_toml dp = coderun.get_dp_for_write(dataProduct, "toml"); * Object_component_write oc1 = dp.getComponent(component1); * oc1.raise_issue("something is terribly wrong with this component", 10); * oc1.writeSamples(samples); @@ -77,7 +77,7 @@ * */ public class Coderun implements AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(Coderun.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Coderun.class); final Config config; final RestClient restClient; private final Map @@ -96,6 +96,8 @@ public class Coderun implements AutoCloseable { FileObject config_object; CodeRepo codeRepo; List authors; + private String wrong_dp_type = + "You have already opened Data_product with name {} but it was not a Data_product_write_{}."; /** * Constructor using only configFilePath - scriptPath is read from the config. @@ -133,6 +135,7 @@ public Coderun(Path configFilePath, @Nullable Path scriptPath) { * @param registryToken the authentication token of the local registry (or null if the token is to * be read from the config or from ~/.fair/registry/token) */ + @SuppressWarnings("squid:S3655") public Coderun(Path configFilePath, @Nullable Path scriptPath, @Nullable String registryToken) { YamlReader yamlReader = new YamlFactory().yamlReader(); if (!new File(configFilePath.toString()).isFile()) { @@ -177,18 +180,18 @@ public Coderun(Path configFilePath, @Nullable Path scriptPath, @Nullable String storageRootURI = URI.create(config.run_metadata().write_data_store().get()); if (storageRootURI.getScheme() == null) { storageRootURI = Path.of(storageRootURI.toString()).toUri(); - logger.trace( + LOGGER.trace( "added file:/// scheme to write_data_store from config: {}", config.run_metadata().write_data_store().get()); } } catch (Exception e) { Path wdsPath = Path.of(config.run_metadata().write_data_store().get()); storageRootURI = wdsPath.toUri(); - logger.trace("tried to create write_data_store URI after Exception: {}", storageRootURI); + LOGGER.trace("tried to create write_data_store URI after Exception: {}", storageRootURI); } } else { storageRootURI = configFilePath.getParent().getParent().getParent().toUri(); - logger.trace( + LOGGER.trace( "Using configFilePath.parent.parent.parent as storageRootURI: {}", storageRootURI); } this.write_data_store_root = new Storage_root(storageRootURI, restClient); @@ -320,16 +323,16 @@ public FileObject getCode_repo() { * @param dataProduct_name the name of the dataProduct to obtain. * @return the data product. */ - public Data_product_read get_dp_for_read(String dataProduct_name) { + public Data_product_read_link get_dp_for_read_link(String dataProduct_name) { if (dp_info_map.containsKey(dataProduct_name)) { // I could of course refuse to serve up the same DP twice, but let's be friendly. if (dp_info_map.get(dataProduct_name).getClass() != Data_product_read.class) { throw (new IllegalActionException( "You are trying to open the same data product twice in the same coderun, first for write and then for read. Please don't.")); } - return (Data_product_read) dp_info_map.get(dataProduct_name); + return (Data_product_read_link) dp_info_map.get(dataProduct_name); } - Data_product_read dp = new Data_product_read(dataProduct_name, this); + Data_product_read_link dp = new Data_product_read_link(dataProduct_name, this); dp_info_map.put(dataProduct_name, dp); return dp; } @@ -341,20 +344,50 @@ public Data_product_read get_dp_for_read(String dataProduct_name) { * @param extension the file extension representing the file type we will write, e.g. csv or toml * @return the data product */ - public Data_product_write get_dp_for_write(String dataProduct_name, String extension) { + public Data_product_write_link get_dp_for_write_link(String dataProduct_name, String extension) { if (dp_info_map.containsKey(dataProduct_name)) { // I could of course refuse to serve up the same DP twice, but let's be friendly. - if (dp_info_map.get(dataProduct_name).getClass() != Data_product_write.class) { + if (dp_info_map.get(dataProduct_name).getClass() != Data_product_write_link.class) { throw (new IllegalActionException( - "You are trying to open the same data product twice in the same coderun, first for read and then for write. Please don't.")); + String.format(this.wrong_dp_type, dataProduct_name, "link"))); } if (!dp_info_map.get(dataProduct_name).extension.equals(extension)) { throw (new IllegalActionException( "You are trying to open the same data product using two different file types. Please don't.")); } - return (Data_product_write) dp_info_map.get(dataProduct_name); + return (Data_product_write_link) dp_info_map.get(dataProduct_name); + } + Data_product_write_link dp = new Data_product_write_link(dataProduct_name, this, extension); + dp_info_map.put(dataProduct_name, dp); + return dp; + } + + /** + * Obtain a data product for writing. (gets the extension from config) + * + * @param dataProduct_name the name of the dataProduct to obtain. + * @return the data product + */ + public Data_product_write_link get_dp_for_write_link(String dataProduct_name) { + return this.get_dp_for_write_link(dataProduct_name, null); + } + + /** + * Obtain a data product for writing. (gets the extension from config) + * + * @param dataProduct_name the name of the dataProduct to obtain. + * @return the data product + */ + public Data_product_write_toml get_dp_for_write_toml(String dataProduct_name) { + if (dp_info_map.containsKey(dataProduct_name)) { + // I could of course refuse to serve up the same DP twice, but let's be friendly. + if (dp_info_map.get(dataProduct_name).getClass() != Data_product_write_toml.class) { + throw (new IllegalActionException( + String.format(this.wrong_dp_type, dataProduct_name, "json"))); + } + return (Data_product_write_toml) dp_info_map.get(dataProduct_name); } - Data_product_write dp = new Data_product_write(dataProduct_name, this, extension); + Data_product_write_toml dp = new Data_product_write_toml(dataProduct_name, this); dp_info_map.put(dataProduct_name, dp); return dp; } @@ -365,17 +398,54 @@ public Data_product_write get_dp_for_write(String dataProduct_name, String exten * @param dataProduct_name the name of the dataProduct to obtain. * @return the data product */ - public Data_product_write get_dp_for_write(String dataProduct_name) { + public Data_product_read_nc get_dp_for_read_nc(String dataProduct_name) { if (dp_info_map.containsKey(dataProduct_name)) { // I could of course refuse to serve up the same DP twice, but let's be friendly. - if (dp_info_map.get(dataProduct_name).getClass() != Data_product_write.class) { + if (dp_info_map.get(dataProduct_name).getClass() != Data_product_read_nc.class) { throw (new IllegalActionException( - "You are trying to open the same data product twice in the same coderun, first for read and then for write. Please don't.")); + String.format(this.wrong_dp_type, dataProduct_name, "nc"))); + } + return (Data_product_read_nc) dp_info_map.get(dataProduct_name); + } + Data_product_read_nc dp = new Data_product_read_nc(dataProduct_name, this); + dp_info_map.put(dataProduct_name, dp); + return dp; + } + + /** + * Obtain a data product for writing. (gets the extension from config) + * + * @param dataProduct_name the name of the dataProduct to obtain. + * @return the data product + */ + public Data_product_read_toml get_dp_for_read_toml(String dataProduct_name) { + if (dp_info_map.containsKey(dataProduct_name)) { + // I could of course refuse to serve up the same DP twice, but let's be friendly. + if (dp_info_map.get(dataProduct_name).getClass() != Data_product_read_toml.class) { + throw (new IllegalActionException(String.format(wrong_dp_type, dataProduct_name, "toml"))); + } + return (Data_product_read_toml) dp_info_map.get(dataProduct_name); + } + Data_product_read_toml dp = new Data_product_read_toml(dataProduct_name, this); + dp_info_map.put(dataProduct_name, dp); + return dp; + } + + /** + * Obtain a data product for writing. (gets the extension from config) + * + * @param dataProduct_name the name of the dataProduct to obtain. + * @return the data product + */ + public Data_product_write_nc get_dp_for_write_nc(String dataProduct_name) { + if (dp_info_map.containsKey(dataProduct_name)) { + // I could of course refuse to serve up the same DP twice, but let's be friendly. + if (dp_info_map.get(dataProduct_name).getClass() != Data_product_write_nc.class) { + throw (new IllegalActionException(String.format(wrong_dp_type, dataProduct_name, "json"))); } - return (Data_product_write) dp_info_map.get(dataProduct_name); + return (Data_product_write_nc) dp_info_map.get(dataProduct_name); } - Data_product_write dp = new Data_product_write(dataProduct_name, this); - // if dp.extension != dp_info_map.get(dataProduct_name).extension) -> FAIL + Data_product_write_nc dp = new Data_product_write_nc(dataProduct_name, this); dp_info_map.put(dataProduct_name, dp); return dp; } @@ -416,7 +486,7 @@ void append_code_run_uuid(String uuid) { try (FileWriter fw = new FileWriter(this.coderuns_txt.toString(), true)) { fw.write(uuid + "\n"); } catch (IOException e) { - logger.error("IOException: append_code_run_uuid() failed.", e); + LOGGER.error("IOException: append_code_run_uuid() failed.", e); } } diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product.java b/api/src/main/java/org/fairdatapipeline/api/Data_product.java index 271e7da..317946f 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Data_product.java +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product.java @@ -1,6 +1,7 @@ package org.fairdatapipeline.api; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; @@ -8,63 +9,105 @@ import java.util.Map; import org.fairdatapipeline.config.ImmutableConfigItem; import org.fairdatapipeline.dataregistry.content.*; -import org.fairdatapipeline.file.CleanableFileChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Data product is created: {@link Coderun#get_dp_for_write(String, String)} or {@link - * Coderun#get_dp_for_read(String)} + * Data product is created: {@link Coderun#get_dp_for_write_link(String, String)} or {@link + * Coderun#get_dp_for_read_link(String)} * *

Upon {@link Coderun#close()} it will try to register itself and its components in the * registry, and then register itself in the coderun. */ -public abstract class Data_product implements AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(Data_product.class); +abstract class Data_product implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(Data_product.class); + /** coderun is the Coderun that created me */ final Coderun coderun; + /** + * registryNamespace is the registry object for this.namespace_name; namespace must always exist + * (created by CLI) + */ final RegistryNamespace registryNamespace; + /** + * namespace_name: from config run_metadata default_input_namespace (for read) or + * default_output_namespace (for write) then possibly overwritten by configItem.use().namespace() + */ String namespace_name; + /** + * extension can be null upon creation, in that case, for a write item: it must be found in the + * configItem.file_type() for a read item: it can be left null. + */ String extension; + /** + * version MUST be provided by config. used upon creation by this.getRegistryData_product -- for + * read to check that it DOES exist, for write to check that it DOESN't. + */ final String version; + /** + * description is an optional config field, only used for 'write'; this will get copied into the + * Data_product in the registry. + */ final String description; + /** + * registryData_product created in populate_data_product() - for read this is the + * RegistryData_product received from registry; for write this is the new RegistryData_product() + * we will post to the registry upon close() + */ RegistryData_product registryData_product; + /** + * registryObject created in populate_data_product() - for read this is the RegistryObject + * received from registry; for write this is the new RegistryObject() we will post to the registry + * upon close() + */ RegistryObject registryObject; + /** + * registryStorage_location created in populate_data_product() - for read this is the + * RegistryStorage_location received from registry; for write this is the new + * RegistryStorage_location() we will post to the registry upon close() + */ RegistryStorage_location registryStorage_location; + /** + * registryStorage_root is this object's Storage_root - for read this is the + * registryStorage_location received from the registry (linked from + * this.registryStorage_location); for write this is taken from coderun.getWriteStorage_root() + */ RegistryStorage_root registryStorage_root; - Path filePath; - CleanableFileChannel filechannel; - Object_component whole_obj_oc; - /** the name given by the user (asked for in the FileApi.get_dp_for_xxx() call) */ + /** + * givenDataProduct_name is the name given by the user (asked for in the FileApi.get_dp_for_xxx() + * call) + */ final String givenDataProduct_name; /** - * usually the same as the given name, unless the config use section has given an alternative data - * product name to open. + * actualDataProduct_name is usually the same as the given name, unless the config use section has + * given an alternative data product name to open. */ String actualDataProduct_name; - + /** whole_obj_oc */ + Object_component whole_obj_oc; + /** componentMap */ final Map componentMap = new HashMap<>(); - final List configItems; + /** been_used to check at close() whether anything actually got read or written */ boolean been_used = false; Data_product(String dataProduct_name, Coderun coderun) { this(dataProduct_name, coderun, null); } + @SuppressWarnings("squid:S3655") Data_product(String dataProduct_name, Coderun coderun, String extension) { this.extension = extension; this.coderun = coderun; this.givenDataProduct_name = dataProduct_name; this.actualDataProduct_name = dataProduct_name; - this.configItems = this.getConfigItems(); this.namespace_name = this.getDefaultNamespace_name(); - ImmutableConfigItem configItem = this.getConfigItem(dataProduct_name); + ImmutableConfigItem configItem = this.getConfigItem(); if (configItem.use().namespace().isPresent()) namespace_name = configItem.use().namespace().get(); if (configItem.use().data_product().isPresent()) this.actualDataProduct_name = configItem.use().data_product().get(); if (configItem.file_type().isPresent()) { if (extension != null && !configItem.file_type().get().equals(extension)) - logger.warn( + LOGGER.warn( "file type conflict: code says {}, config says {}", extension, configItem.file_type().get()); @@ -79,20 +122,47 @@ public abstract class Data_product implements AutoCloseable { /* * this should populate the main data fields: data_product, storage_location and fdpObject * for READ: by reading these from the registry - * for WRITE: by preparing empty ones that can later by posted to registry + * for WRITE: by preparing empty ones that can later be posted to registry */ abstract void populate_data_product(); + /** + * @return the config writeItems (if this is a write dp) or config readItems (if this is a read + * dp) + */ abstract List getConfigItems(); + /** + * @return from config.run_metadata(): return default_input_namespace (for Data_product_read) of + * default_output_namespace (for Data_product_write) + */ abstract String getDefaultNamespace_name(); + /** + * getRegistryNamespace retrieves the registryNamespace from the registry. This should always + * exist as it has been created by the CLI. + * + * @param namespace_name the namespace name to look for in the registry + * @return the RegistryNamespace for the namespace with given name. + * @throws RegistryObjectNotFoundException if the namespace is not found in the registry. + */ RegistryNamespace getRegistryNamespace(String namespace_name) { - return (RegistryNamespace) - coderun.restClient.getFirst( - RegistryNamespace.class, Collections.singletonMap("name", namespace_name)); + RegistryNamespace ns = + (RegistryNamespace) + coderun.restClient.getFirst( + RegistryNamespace.class, Collections.singletonMap("name", namespace_name)); + if (ns == null) + throw (new RegistryObjectNotFoundException("Can't find the namespace " + namespace_name)); + return ns; } + /** + * getRegistryData_product() is used by data_product_read to retrieve the data_product from + * registry; it is used by data_product_write to check the data_product does not exist yet. + * + * @return The RegistryData_product from the registry with this.actualDataProduct_name, + * this.registryNamespace.get_id(), and this.version. returns null is not found. + */ RegistryData_product getRegistryData_product() { Map dp_map = Map.of( @@ -105,40 +175,76 @@ RegistryData_product getRegistryData_product() { return (RegistryData_product) coderun.restClient.getFirst(RegistryData_product.class, dp_map); } - ImmutableConfigItem getConfigItem(String dataProduct_name) { + /** + * getConfigItem searches the ConfigItems for this.givenDataProduct_name. Data_product_read will + * override this and throw ConfigException if not found. Data_product_write will override this to + * add globbing match for config items ending in /*, and also throw ConfigException if not found. + * + * @return the first of this.getConfigItems() that matches this dataProduct_name. returns null if + * not found. + */ + ImmutableConfigItem getConfigItem() { return this.getConfigItems().stream() - .filter(ci -> ci.data_product().equals(dataProduct_name)) + .filter(ci -> ci.data_product().equals(this.givenDataProduct_name)) .findFirst() .orElse(null); } - /* + /** + * getFilePath creates the path for this data_products file. it tries to create the directory if + * it doesn't exist. * + * @return the Path for this data product file, by appending registryStorage_location path to the + * registryStorage_root path. */ - abstract Path getFilePath(); - - /** please make sure the implementation set been_used = true; */ - abstract CleanableFileChannel getFilechannel() throws IOException; - - void closeFileChannel() { - if (this.filechannel != null) { - this.filechannel.close(); - this.filechannel = null; + Path getFilePath() { + this.been_used = true; + Path filePath = + this.registryStorage_root + .getPath() + .resolve(Path.of(this.registryStorage_location.getPath())); + if (!filePath.getParent().toFile().exists()) { + try { + Files.createDirectories(filePath.getParent()); + } catch (IOException e) { + LOGGER.error("failed to create directory {}", filePath.getParent()); + // throw, or continue? + return null; + } } + return filePath; } - abstract void objects_to_registry(); + /** + * do_hash() is used by data_product_write to set the registryStorage_location.hash upon close. + * nothing to do for data_product_read + */ + abstract void do_hash(); + + /** + * stolo_obj_and_dp_to_registry() will post my registryStorage_location, RegistryObject, and + * registryData_product to the registry. (find matching stolo hash and delete this file if there + * already exists an identical) nothing to do for data_product_read + */ + abstract void stolo_obj_and_dp_to_registry(); + + void components_to_registry() { + if (this.whole_obj_oc != null) this.whole_obj_oc.register_me_in_registry(); + this.componentMap.forEach((key, value) -> value.register_me_in_registry()); + } - void InputsOutputsToCoderun() { + void inputsOutputsToCoderun() { if (this.whole_obj_oc != null) this.whole_obj_oc.register_me_in_code_run(); this.componentMap.forEach((key, value) -> value.register_me_in_code_run()); } @Override public void close() { - this.closeFileChannel(); - this.objects_to_registry(); - this.InputsOutputsToCoderun(); + LOGGER.trace("Data_product.close()"); + this.do_hash(); + this.stolo_obj_and_dp_to_registry(); + this.components_to_registry(); + this.inputsOutputsToCoderun(); } } diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_read.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_read.java index 1799d16..eb70fcd 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Data_product_read.java +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_read.java @@ -1,36 +1,31 @@ package org.fairdatapipeline.api; -import static java.nio.file.StandardOpenOption.*; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Path; import java.util.List; import org.fairdatapipeline.config.ConfigException; import org.fairdatapipeline.config.ImmutableConfigItem; -import org.fairdatapipeline.dataregistry.content.RegistryNamespace; import org.fairdatapipeline.dataregistry.content.RegistryObject; import org.fairdatapipeline.dataregistry.content.RegistryStorage_location; import org.fairdatapipeline.dataregistry.content.RegistryStorage_root; -import org.fairdatapipeline.file.CleanableFileChannel; /** - * Data_product_read is created by Coderun: {@link Coderun#get_dp_for_read(String)} + * Data_product_read_* are created by Coderun. * *

Upon {@link Coderun#close()} it will register its components in the coderun. */ -public class Data_product_read extends Data_product { - private boolean hash_checked = false; - +abstract class Data_product_read extends Data_product { Data_product_read(String dataProduct_name, Coderun coderun) { super(dataProduct_name, coderun); } + /** + * requires: this.registryData_product is set this.registryNamespace is set + * this.actualDataProduct_name is set + */ void populate_data_product() { // called from the constructor this.registryData_product = this.getRegistryData_product(); if (this.registryData_product == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "Trying to read from non-existing data_product " + this.actualDataProduct_name + "; NS " @@ -42,7 +37,7 @@ void populate_data_product() { (RegistryObject) coderun.restClient.get(RegistryObject.class, this.registryData_product.getObject()); if (this.registryObject == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "couldn't retrieve the fdpObject for this READ dp " + this.givenDataProduct_name + " (" @@ -54,7 +49,7 @@ void populate_data_product() { coderun.restClient.get( RegistryStorage_location.class, this.registryObject.getStorage_location()); if (this.registryStorage_location == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "Couldn't retrieve the StorageLocation for this READ dp " + this.givenDataProduct_name + " (" @@ -66,17 +61,13 @@ void populate_data_product() { coderun.restClient.get( RegistryStorage_root.class, this.registryStorage_location.getStorage_root()); if (this.registryStorage_root == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "Couldn't retrieve the StorageRoot for this READ dp " + this.givenDataProduct_name + " (" + this.actualDataProduct_name + ")")); } - this.filePath = - this.registryStorage_root - .getPath() - .resolve(Path.of(this.registryStorage_location.getPath())); } List getConfigItems() { @@ -87,87 +78,27 @@ String getDefaultNamespace_name() { return this.coderun.config.run_metadata().default_input_namespace().orElse(""); } + /** + * getConfigItem searches the ConfigItems for this.givenDataProduct_name. + * + * @return the configItem that matches this.givenDataProduct_name + * @throws ConfigException if no configItem matches. + */ @Override - RegistryNamespace getRegistryNamespace(String namespace_name) { - // for a READ dp we must have the namespace from the config or we will have to give up - RegistryNamespace ns = super.getRegistryNamespace(namespace_name); - if (ns == null) { - throw (new RegistryObjectNotfoundException("Can't find the namespace " + namespace_name)); - } - return ns; - } - - @Override - ImmutableConfigItem getConfigItem(String dataProduct_name) { - ImmutableConfigItem configItem = super.getConfigItem(dataProduct_name); + ImmutableConfigItem getConfigItem() { + ImmutableConfigItem configItem = super.getConfigItem(); if (configItem == null) { - throw (new ConfigException("dataProduct " + dataProduct_name + " not found in config")); + throw (new ConfigException( + "dataProduct " + this.givenDataProduct_name + " not found in config")); } return configItem; } - private void check_hash() { - // Do nothing for now. Unsure if we will implement hash checking. - } - - private void executeOnCloseFileHandleDP() { - // don't need to Hash READ objects - } - - protected Path getFilePath() { - this.been_used = true; - if (!hash_checked) this.check_hash(); - return this.filePath; - } - - CleanableFileChannel getFilechannel() throws IOException { - this.been_used = true; - Runnable onClose = this::executeOnCloseFileHandleDP; - if (!hash_checked) this.check_hash(); - if (this.filechannel == null) { - this.filechannel = new CleanableFileChannel(FileChannel.open(this.filePath, READ), onClose); - } else { - if (!this.filechannel.isOpen()) { - this.filechannel = new CleanableFileChannel(FileChannel.open(this.filePath, READ), onClose); - } - } - return this.filechannel; - } - - /** - * Obtain an Object_component for reading. - * - * @param component_name the name of the object component to read. - * @return the Object_component class - */ - public Object_component_read getComponent(String component_name) { - if (componentMap.containsKey(component_name)) - return (Object_component_read) componentMap.get(component_name); - Object_component_read dc = new Object_component_read(this, component_name); - componentMap.put(component_name, dc); - return dc; - } - - /** - * Obtain an Object_component (whole_object) for reading. - * - * @return the Object_component class - */ - public Object_component_read getComponent() { - if (this.whole_obj_oc == null) this.whole_obj_oc = new Object_component_read(this); - return (Object_component_read) this.whole_obj_oc; - } - - void components_to_registry() { - // this is just to make sure the components can register their issues - this.componentMap.entrySet().stream() - .filter(c -> c.getValue().been_used) - .forEach(component -> component.getValue().register_me_in_registry()); + void do_hash() { + // no hashing to be done for read } - void objects_to_registry() { - // this is called upon close; but since this is a READ dp, it won't have a DP, Stolo, fdpObj, or - // any new components to register - this.components_to_registry(); + void stolo_obj_and_dp_to_registry() { + // nothing to post to registry for read } } diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_read_filechannel.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_filechannel.java new file mode 100644 index 0000000..0185f02 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_filechannel.java @@ -0,0 +1,47 @@ +package org.fairdatapipeline.api; + +import static java.nio.file.StandardOpenOption.READ; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import org.fairdatapipeline.file.CleanableFileChannel; + +abstract class Data_product_read_filechannel extends Data_product_read { + CleanableFileChannel filechannel; + + Data_product_read_filechannel(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + CleanableFileChannel getFilechannel() throws IOException { + this.been_used = true; + Runnable onClose = this::executeOnCloseFileHandleDP; + if (this.filechannel == null) { + this.filechannel = + new CleanableFileChannel(FileChannel.open(this.getFilePath(), READ), onClose); + } else { + if (!this.filechannel.isOpen()) { + this.filechannel = + new CleanableFileChannel(FileChannel.open(this.getFilePath(), READ), onClose); + } + } + return this.filechannel; + } + + private void executeOnCloseFileHandleDP() { + // don't need to Hash READ objects + } + + void closeFileChannel() { + if (this.filechannel != null) { + this.filechannel.close(); + this.filechannel = null; + } + } + + @Override + public void close() { + closeFileChannel(); + super.close(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_read_link.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_link.java new file mode 100644 index 0000000..22d75be --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_link.java @@ -0,0 +1,18 @@ +package org.fairdatapipeline.api; + +public class Data_product_read_link extends Data_product_read_filechannel { + Data_product_read_link(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + /** + * Obtain an Object_component (whole_object) for reading. + * + * @return this.whole_obj_oc the Object_component_read_filechannel_link + */ + public Object_component_read_filechannel_link getComponent() { + if (this.whole_obj_oc == null) + this.whole_obj_oc = new Object_component_read_filechannel_link(this); + return (Object_component_read_filechannel_link) this.whole_obj_oc; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_read_nc.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_nc.java new file mode 100644 index 0000000..fae2f2e --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_nc.java @@ -0,0 +1,69 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import javax.annotation.Nonnull; +import org.fairdatapipeline.netcdf.NetcdfReader; +import org.fairdatapipeline.netcdf.VariableName; + +public class Data_product_read_nc extends Data_product_read { + NetcdfReader reader; + + Data_product_read_nc(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + /** + * Obtain an Object_component_read_nc for reading. + * + * @param component_name the name of the object component. + * @return the Object_component_read_nc object. + * @throws IOException if the file cannot be opened for reading. + */ + public Object_component_read_nc getComponent(@Nonnull String component_name) throws IOException { + if (componentMap.containsKey(component_name)) { + // check its on Object_component_read_nc + return (Object_component_read_nc) componentMap.get(component_name); + } + Object_component_read_nc dc; + VariableName vn = new VariableName(component_name); + dc = new Object_component_read_nc(this, vn); + componentMap.put(component_name, dc); + return dc; + } + + /** + * Obtain an Object_component_read_table for reading. + * + * @param component_name the name of the object component. + * @return the Object_component_read_nc object. + * @throws IOException if the file cannot be opened for reading. + */ + public Object_component_read_table getTable(@Nonnull String component_name) throws IOException { + if (componentMap.containsKey(component_name)) { + // check it's an Object_component_read_table + return (Object_component_read_table) componentMap.get(component_name); + } + VariableName vn = new VariableName(component_name); + Object_component_read_table dc = new Object_component_read_table(this, vn); + componentMap.put(component_name, dc); + return dc; + } + + NetcdfReader getNetcdfReader() throws IOException { + this.been_used = true; + if (this.reader == null) { + this.reader = new NetcdfReader(this.getFilePath().toString()); + } + return reader; + } + + void closeNetcdfReader() { + // this.netCDFBuilder.build?? + } + + @Override + public void close() { + closeNetcdfReader(); + super.close(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_read_toml.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_toml.java new file mode 100644 index 0000000..7199482 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_read_toml.java @@ -0,0 +1,24 @@ +package org.fairdatapipeline.api; + +import java.util.Objects; + +public class Data_product_read_toml extends Data_product_read_filechannel { + Data_product_read_toml(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + /** + * Obtain an Object_component for reading. + * + * @param component_name the name of the object component. + * @return the Object_component_read_filechannel_toml object. + */ + public Object_component_read_filechannel_toml getComponent(String component_name) { + if (componentMap.containsKey(component_name)) + return (Object_component_read_filechannel_toml) componentMap.get(component_name); + Object_component_read_filechannel_toml dc; + dc = new Object_component_read_filechannel_toml(this, Objects.requireNonNull(component_name)); + componentMap.put(component_name, dc); + return dc; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_write.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_write.java index 2766c21..9191f96 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Data_product_write.java +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_write.java @@ -1,31 +1,26 @@ package org.fairdatapipeline.api; -import static java.nio.file.StandardOpenOption.*; - import java.io.IOException; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Map; -import java.util.Objects; import org.fairdatapipeline.config.ConfigException; import org.fairdatapipeline.config.ImmutableConfigItem; import org.fairdatapipeline.dataregistry.content.*; -import org.fairdatapipeline.file.CleanableFileChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Data_product_write is created by Coderun: {@link Coderun#get_dp_for_write(String, String)} + * Data_product_write is created by Coderun: {@link Coderun#get_dp_for_write_link(String, String)} * *

Upon {@link Coderun#close()} it will register itself and its components in the registry, and * then register its components in the coderun. */ -public class Data_product_write extends Data_product { - private static final Logger logger = LoggerFactory.getLogger(Data_product_write.class); - private boolean is_hashed; +abstract class Data_product_write extends Data_product { + private static final Logger LOGGER = LoggerFactory.getLogger(Data_product_write.class); + boolean is_hashed; Data_product_write(String dataProduct_name, Coderun coderun) { super(dataProduct_name, coderun); @@ -54,7 +49,6 @@ void populate_data_product() { String filename = this.version + "." + this.extension; Path my_stolo_path = Paths.get(this.namespace_name).resolve(this.actualDataProduct_name).resolve(filename); - this.filePath = this.registryStorage_root.getPath().resolve(my_stolo_path); this.registryStorage_location.setPath(my_stolo_path.toString()); this.registryObject = new RegistryObject(); this.registryObject.setDescription(this.description); @@ -74,124 +68,53 @@ String getDefaultNamespace_name() { return this.coderun.config.run_metadata().default_output_namespace().orElse(""); } - @Override - RegistryNamespace getRegistryNamespace(String namespace_name) { - RegistryNamespace ns = super.getRegistryNamespace(namespace_name); - if (ns == null) { - ns = (RegistryNamespace) coderun.restClient.post(new RegistryNamespace(namespace_name)); - if (ns == null) { - throw (new RegistryException( - "Failed to create in registry: namespace '" + namespace_name + "'")); - } - } - return ns; - } - - private boolean globMatch(String pattern, String dataProduct_name) { + /** + * @param pattern is the data_product name from the config, that may end in /* to allow matching + * for glob matching. + * @return if pattern ends in /*, we return true if this.givenDataProduct_name starts with pattern + * without the * (matching up to and including the /) + */ + private boolean globMatch(String pattern) { if (pattern.endsWith("/*")) { - return dataProduct_name.startsWith(pattern.substring(0, pattern.length() - 1)); + return this.givenDataProduct_name.startsWith(pattern.substring(0, pattern.length() - 1)); } - return pattern.equals(dataProduct_name); + return pattern.equals(this.givenDataProduct_name); } + /** + * getConfigItem searches the ConfigItems for this.givenDataProduct_name. if there is no exact + * match, try globMatch for configItems ending in /* + * + * @return the matching configItem + * @throws ConfigException if none matches. + */ @Override - ImmutableConfigItem getConfigItem(String dataProduct_name) { - ImmutableConfigItem configItem = super.getConfigItem(dataProduct_name); + ImmutableConfigItem getConfigItem() { + ImmutableConfigItem configItem = super.getConfigItem(); if (configItem == null) { // for WRITING dp's; we allow /* globbing if there is no exact match we look for a /* match configItem = this.getConfigItems().stream() - .filter(ci -> globMatch(ci.data_product(), dataProduct_name)) + .filter(ci -> globMatch(ci.data_product())) .findFirst() .orElse(null); } if (configItem == null) { - throw (new ConfigException("DataProduct " + dataProduct_name + " not found in config")); + throw (new ConfigException( + "DataProduct " + this.givenDataProduct_name + " not found in config")); } return configItem; } - private void executeOnCloseFileHandleDP() { - this.do_hash(); - } - - private void do_hash() { + void do_hash() { if (this.is_hashed) return; - String hash = coderun.hasher.fileHash(this.filePath.toString()); + String hash = coderun.hasher.fileHash(this.getFilePath().toString()); this.registryStorage_location.setHash(hash); this.is_hashed = true; } - Path getFilePath() { - this.been_used = true; - this.is_hashed = false; - if (!this.filePath.getParent().toFile().exists()) { - try { - Files.createDirectories(this.filePath.getParent()); - } catch (IOException e) { - logger.error("failed to create directory {}", this.filePath.getParent()); - // throw, or continue? - return null; - } - } - return this.filePath; - } - - CleanableFileChannel getFilechannel() throws IOException { - this.been_used = true; - Runnable onClose = this::executeOnCloseFileHandleDP; - if (this.filechannel == null) { - if (!this.filePath.getParent().toFile().exists()) { - Files.createDirectories(this.filePath.getParent()); - } - this.filechannel = - new CleanableFileChannel(FileChannel.open(this.filePath, CREATE_NEW, WRITE), onClose); - } else { - if (!this.filechannel.isOpen()) { - this.filechannel = - new CleanableFileChannel(FileChannel.open(this.filePath, APPEND, WRITE), onClose); - } - } - this.is_hashed = false; - return this.filechannel; - } - - @Override - void closeFileChannel() { - if (this.filechannel != null) { - this.filechannel.close(); - this.filechannel = null; - } - } - - /** - * Obtain an Object_component for writing. - * - * @param component_name the name of the object component. - * @return the Object_component_write object. - */ - public Object_component_write getComponent(String component_name) { - if (componentMap.containsKey(component_name)) - return (Object_component_write) componentMap.get(component_name); - Object_component_write dc; - dc = - new Object_component_write( - this, Objects.requireNonNullElse(component_name, "whole_object")); - componentMap.put(component_name, dc); - return dc; - } - - /** - * Obtain an Object_component (whole_object) for writing. - * - * @return the Object_component class - */ - public Object_component_write getComponent() { - if (this.whole_obj_oc == null) this.whole_obj_oc = new Object_component_write(this); - return (Object_component_write) this.whole_obj_oc; - } - void stolo_obj_and_dp_to_registry() { + LOGGER.trace("data_product_write.stolo_obj_and_dp_to_registry()"); if (this.registryStorage_location.getUrl() == null) { Map find_identical = Map.of( @@ -207,10 +130,11 @@ void stolo_obj_and_dp_to_registry() { this.coderun.restClient.getFirst(RegistryStorage_location.class, find_identical); if (identical_sl != null) { // we've found an existing stolo with matching hash. delete this one. + LOGGER.trace("found an existing stolo with matching hash.. delete."); try { - Files.delete(this.filePath); + Files.delete(this.getFilePath()); } catch (IOException e) { - logger.warn( + LOGGER.warn( "Failed to delete current data file which is identical to a file already in the local registry.", e); } @@ -226,6 +150,7 @@ void stolo_obj_and_dp_to_registry() { + this.registryStorage_location.getPath())); } else { this.registryStorage_location = sl; + LOGGER.trace("registered the stolo."); } } } @@ -235,6 +160,7 @@ void stolo_obj_and_dp_to_registry() { throw (new RegistryException( "Failed to create in registry: Object " + this.registryObject.getDescription())); } + LOGGER.trace("registered the RegistryObject."); this.registryObject = o; this.registryData_product.setObject(o.getUrl()); RegistryData_product dp = @@ -243,17 +169,7 @@ void stolo_obj_and_dp_to_registry() { throw (new RegistryException( "Failed to create in registry: Data_product " + this.registryData_product.getName())); } + LOGGER.trace("registered the Data_product"); this.registryData_product = dp; } - - void components_to_registry() { - if (this.whole_obj_oc != null) this.whole_obj_oc.register_me_in_registry(); - this.componentMap.forEach((key, value) -> value.register_me_in_registry()); - } - - void objects_to_registry() { - this.do_hash(); - this.stolo_obj_and_dp_to_registry(); - this.components_to_registry(); - } } diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_write_filechannel.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_filechannel.java new file mode 100644 index 0000000..51d0aeb --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_filechannel.java @@ -0,0 +1,59 @@ +package org.fairdatapipeline.api; + +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.nio.file.StandardOpenOption.WRITE; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import org.fairdatapipeline.file.CleanableFileChannel; + +abstract class Data_product_write_filechannel extends Data_product_write { + CleanableFileChannel filechannel; + + Data_product_write_filechannel(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + Data_product_write_filechannel(String dataProduct_name, Coderun coderun, String extension) { + super(dataProduct_name, coderun, extension); + } + + private void executeOnCloseFileHandleDP() { + this.do_hash(); + } + + CleanableFileChannel getFilechannel() throws IOException { + this.been_used = true; + Runnable onClose = this::executeOnCloseFileHandleDP; + if (this.filechannel == null) { + if (!this.getFilePath().getParent().toFile().exists()) { + Files.createDirectories(this.getFilePath().getParent()); + } + this.filechannel = + new CleanableFileChannel( + FileChannel.open(this.getFilePath(), CREATE_NEW, WRITE), onClose); + } else { + if (!this.filechannel.isOpen()) { + this.filechannel = + new CleanableFileChannel(FileChannel.open(this.getFilePath(), APPEND, WRITE), onClose); + } + } + this.is_hashed = false; + return this.filechannel; + } + + void closeFileChannel() { + if (this.filechannel != null) { + this.filechannel.close(); + this.filechannel = null; + } + } + + @Override + public void close() { + closeFileChannel(); + super.close(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_write_link.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_link.java new file mode 100644 index 0000000..c3f3aa9 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_link.java @@ -0,0 +1,30 @@ +package org.fairdatapipeline.api; + +/** + * Data_product_write_link is created by Coderun: {@link Coderun#get_dp_for_write_link(String, + * String)} + * + *

Upon {@link Coderun#close()} it will register itself and its components in the registry, and + * then register its components in the coderun. + */ +public class Data_product_write_link extends Data_product_write_filechannel { + + Data_product_write_link(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + Data_product_write_link(String dataProduct_name, Coderun coderun, String extension) { + super(dataProduct_name, coderun, extension); + } + + /** + * Obtain an Object_component_write_filechannel_link (whole_object) for writing. + * + * @return the Object_component_write_filechannel_link + */ + public Object_component_write_filechannel_link getComponent() { + if (this.whole_obj_oc == null) + this.whole_obj_oc = new Object_component_write_filechannel_link(this); + return (Object_component_write_filechannel_link) this.whole_obj_oc; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_write_nc.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_nc.java new file mode 100644 index 0000000..a40fc8c --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_nc.java @@ -0,0 +1,170 @@ +package org.fairdatapipeline.api; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.fairdatapipeline.netcdf.NetcdfBuilder; +import org.fairdatapipeline.netcdf.NetcdfWriter; +import org.fairdatapipeline.objects.CoordinateVariableDefinition; +import org.fairdatapipeline.objects.DimensionalVariableDefinition; +import org.fairdatapipeline.objects.TableDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.write.Nc4Chunking; + +/** + * Data_product_write_nc is created by Coderun: {@link Coderun#get_dp_for_write_nc(String)} + * + *

Upon {@link Coderun#close()} it will register itself and its components in the registry, and + * then register its components in the coderun. + */ +public class Data_product_write_nc extends Data_product_write { + private static final Logger LOGGER = LoggerFactory.getLogger(Data_product_write_nc.class); + private NetcdfBuilder netCDFBuilder; + private NetcdfWriter netCDFWriter; + private static final String TOOLATE = + "too late.. you can only set chunking details BEFORE any of the components has prepared any array variables."; + + public enum chunkingStrategies { + GRIB, + NONE, + STANDARD; + + private Nc4Chunking.Strategy getNc4Strategy() { + switch (this) { + case GRIB: + return Nc4Chunking.Strategy.grib; + case NONE: + return Nc4Chunking.Strategy.none; + case STANDARD: + return Nc4Chunking.Strategy.standard; + } + return Nc4Chunking.Strategy.standard; + } + } + + private chunkingStrategies chunkingStrategy = chunkingStrategies.STANDARD; + private int nc4deflateLevel = 2; + private boolean nc4shuffle = true; + + Data_product_write_nc(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun, "nc"); + } + + public Object_component_write_array getComponent(@NonNull DimensionalVariableDefinition nadef) { + if (componentMap.containsKey(nadef.getVariableName().getFullPath())) { + // TODO: check that component in map is an Object_component_write_array + return (Object_component_write_array) componentMap.get(nadef.getVariableName().getFullPath()); + } + Object_component_write_array dc; + dc = new Object_component_write_array(this, nadef); + LOGGER.trace( + "new Object_component (DimensionalVariable) created: {}", + nadef.getVariableName().getFullPath()); + componentMap.put(nadef.getVariableName().getFullPath(), dc); + return dc; + } + + public Object_component_write_table getComponent(@NonNull TableDefinition tabledef) { + if (componentMap.containsKey(tabledef.getGroupName().toString())) { + // TODO: check that component in map is an Object_component_write_table + return (Object_component_write_table) componentMap.get(tabledef.getGroupName().toString()); + } + Object_component_write_table tc; + tc = new Object_component_write_table(this, tabledef); + LOGGER.trace("new Object_component (table) created: {}", tabledef.getGroupName()); + componentMap.put(tabledef.getGroupName().toString(), tc); + return tc; + } + + public Object_component_write_dimension getComponent( + @NonNull CoordinateVariableDefinition dimdef) { + if (componentMap.containsKey(dimdef.getVariableName().getFullPath())) { + // TODO: check that component in map is an Object_component_write_nc + return (Object_component_write_dimension) + componentMap.get(dimdef.getVariableName().getFullPath()); + } + Object_component_write_dimension dc; + dc = new Object_component_write_dimension(this, dimdef); + LOGGER.trace( + "new Object_component (dimension) created: {}", dimdef.getVariableName().getFullPath()); + componentMap.put(dimdef.getVariableName().getFullPath(), dc); + return dc; + } + + private void executeOnCloseDP() { + this.do_hash(); + } + + public void setChunkingStrategy(chunkingStrategies chunkingStrategy) { + if (this.netCDFWriter != null || this.netCDFBuilder != null) + throw (new IllegalActionException(TOOLATE)); + this.chunkingStrategy = chunkingStrategy; + } + + public void setChunkingDeflate(int deflateLevel) { + if (this.netCDFWriter != null || this.netCDFBuilder != null) + throw (new IllegalActionException(TOOLATE)); + this.nc4deflateLevel = deflateLevel; + } + + public void setChunkingShuffle(boolean shuffle) { + if (this.netCDFWriter != null || this.netCDFBuilder != null) + throw (new IllegalActionException(TOOLATE)); + this.nc4shuffle = shuffle; + } + + NetcdfBuilder getNetCDFBuilder() { + if (this.netCDFWriter != null) { + throw (new IllegalActionException( + "you've already started writing data to the netCDF file, you can't go back preparing arrays once writing has started.")); + } + this.been_used = true; + Runnable onClose = this::executeOnCloseDP; + if (this.netCDFBuilder == null) { + this.netCDFBuilder = + new NetcdfBuilder( + this.getFilePath().toString(), + this.chunkingStrategy.getNc4Strategy(), + this.nc4deflateLevel, + this.nc4shuffle, + onClose); + } + return this.netCDFBuilder; + } + + NetcdfWriter getNetCDFWriter() { + if (this.netCDFWriter != null) return this.netCDFWriter; + if (this.netCDFBuilder == null) { + throw (new IllegalActionException( + "you must prepare one or more arrays to write to before trying to write data to netCDF")); + } + Runnable onClose = + this::executeOnCloseDP; // is it going to get called twice, both for the closing of + // netcdfwriter AND for netcdfBuilder? + this.netCDFWriter = new NetcdfWriter(this.netCDFBuilder, onClose); + this.netCDFBuilder.close(); + this.netCDFBuilder = null; + this.componentMap.forEach((key, value) -> ((Object_component_write) value).write_preset_data()); + return this.netCDFWriter; + } + + void closeNetcdfBuilder() { + if (this.netCDFWriter != null) { + LOGGER.trace("closeNetcdfBuilder() closing writer"); + this.netCDFWriter.close(); + this.netCDFWriter = null; + } else if (this.netCDFBuilder != null) { + LOGGER.trace("closeNetcdfBuilder() closing builder"); + this.netCDFBuilder.close(); + this.netCDFBuilder = null; + } else { + LOGGER.trace("closeNetcdfBuilder() doing nothing"); + } + } + + @Override + public void close() { + LOGGER.trace("close()"); + closeNetcdfBuilder(); + super.close(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Data_product_write_toml.java b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_toml.java new file mode 100644 index 0000000..a841c58 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Data_product_write_toml.java @@ -0,0 +1,31 @@ +package org.fairdatapipeline.api; + +import java.util.Objects; + +/** + * Data_product_write is created by Coderun: {@link Coderun#get_dp_for_write_toml(String)} + * + *

Upon {@link Coderun#close()} it will register itself and its components in the registry, and + * then register its components in the coderun. + */ +public class Data_product_write_toml extends Data_product_write_filechannel { + + Data_product_write_toml(String dataProduct_name, Coderun coderun) { + super(dataProduct_name, coderun); + } + + /** + * Obtain an Object_component for writing. + * + * @param component_name the name of the object component. + * @return the Object_component_write_filechannel_toml object. + */ + public Object_component_write_filechannel_toml getComponent(String component_name) { + if (componentMap.containsKey(component_name)) + return (Object_component_write_filechannel_toml) componentMap.get(component_name); + Object_component_write_filechannel_toml dc; + dc = new Object_component_write_filechannel_toml(this, Objects.requireNonNull(component_name)); + componentMap.put(component_name, dc); + return dc; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component.java b/api/src/main/java/org/fairdatapipeline/api/Object_component.java index ab62eaf..c922a5a 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Object_component.java +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component.java @@ -1,20 +1,21 @@ package org.fairdatapipeline.api; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import org.fairdatapipeline.dataregistry.content.RegistryObject_component; -import org.fairdatapipeline.file.CleanableFileChannel; /** * This represents an object_component to read from or write to (or raise issues with) An * object_component without a name is the 'whole_object' component. */ -public abstract class Object_component { +abstract class Object_component { + /** component_name is the name of this component; if not given we will use "whole_object" */ final String component_name; + /** is this a 'whole_object' or a named part for a toml or netcdf file? */ final boolean whole_object; + /** the Data_product we are part of */ final Data_product dp; + /** registryObject_component is the */ RegistryObject_component registryObject_component; + boolean been_used = false; Object_component(Data_product dp, String component_name) { @@ -43,22 +44,12 @@ public void raise_issue(String description, Integer severity) { i.add_components(this); } - CleanableFileChannel getFileChannel() throws IOException { - this.been_used = true; - return this.dp.getFilechannel(); - } - + /** + * populate_component() - sets this.registryObject_component - for READ component: retrieve from + * registry - for WRITE component: create an empty new registryObject_component. + */ abstract void populate_component(); - RegistryObject_component retrieveObject_component() { - Map objcompmap = new HashMap<>(); - objcompmap.put("object", dp.registryObject.get_id().toString()); - if (this.whole_object) objcompmap.put("whole_object", "true"); - else objcompmap.put("name", component_name); - return (RegistryObject_component) - dp.coderun.restClient.getFirst(RegistryObject_component.class, objcompmap); - } - /** POST this registryObject_component to the registry, unless it's not been used */ abstract void register_me_in_registry(); diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java index 9df0b69..02a8f5a 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java @@ -1,11 +1,8 @@ package org.fairdatapipeline.api; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import org.fairdatapipeline.distribution.Distribution; -import org.fairdatapipeline.file.CleanableFileChannel; -import org.fairdatapipeline.parameters.ReadComponent; +import java.util.HashMap; +import java.util.Map; +import org.fairdatapipeline.dataregistry.content.RegistryObject_component; /** * This represents an object_component to read from (or raise issues with) An object_component @@ -13,102 +10,46 @@ * components on toml and h5 files, and only read from the 'whole_object' on any other files. This * is not enforced at the moment. */ -public class Object_component_read extends Object_component { +abstract class Object_component_read extends Object_component { - Object_component_read(Data_product_read dp, String component_name) { + Object_component_read(Data_product dp, String component_name) { super(dp, component_name); } - Object_component_read(Data_product_read dp) { + Object_component_read(Data_product dp) { super(dp); } - protected void populate_component() { - this.registryObject_component = this.retrieveObject_component(); - if (this.registryObject_component == null) { - throw (new RegistryObjectNotfoundException( - "Object Component '" - + this.component_name - + "' for Object " - + this.dp.registryObject.get_id().toString() - + " not found in registry.")); - } - } - - /** - * get the filePath to read from; only for whole_object component - * - * @return Path the Path of the data object. - */ - public Path readLink() { - if (!this.whole_object) { - throw (new IllegalActionException( - "You shouldn't try to read directly from a Data Product with named components.")); - } - this.been_used = true; - return this.dp.getFilePath(); - } - - /** - * get the CleanableFileChannel to read directly from the file. only for whole_object component. - * - * @return CleanableFileChannel the filechannel to read from. - * @throws IOException if the file can't be opened. - */ - public CleanableFileChannel readFileChannel() throws IOException { - if (!this.whole_object) { - throw (new IllegalActionException( - "You shouldn't try to read directly from a Data Product with named components.")); - } - this.been_used = true; - return this.getFileChannel(); - } - /** - * read the Estimate that was stored as this component in a TOML file. + * retrieves the registryObject_component from the registry (given dp.registryObject.get_id(), and + * either whole_object=true OR name=component_name) * - * @return the estimate as Number + * @return the registryObject_component found, or null if none found. */ - public Number readEstimate() { - ReadComponent data; - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - data = - dp.coderun.parameterDataReader.read(fileChannel, this.registryObject_component.getName()); - } catch (IOException e) { - throw (new RuntimeException("readEstimate() -- IOException trying to read from file", e)); - } - return data.getEstimate(); + RegistryObject_component retrieveObject_component() { + Map objcompmap = new HashMap<>(); + objcompmap.put("object", dp.registryObject.get_id().toString()); + if (this.whole_object) objcompmap.put("whole_object", "true"); + else objcompmap.put("name", component_name); + return (RegistryObject_component) + dp.coderun.restClient.getFirst(RegistryObject_component.class, objcompmap); } /** - * read the Distribution that was stored as this component in a TOML file. + * sets this.registryObject_component by retrieving it from the registry. * - * @return the Distribution + * @throws RegistryObjectNotFoundException if the Object_component does not exist. */ - public Distribution readDistribution() { - ReadComponent data; - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); - } catch (IOException e) { - throw (new RuntimeException( - "readDistribution() -- IOException trying to read from file.", e)); - } - return data.getDistribution(); - } - - /** - * read the Samples that were stored as this component in a TOML file. - * - * @return the Samples object - */ - public List readSamples() { - ReadComponent data; - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); - } catch (IOException e) { - throw (new RuntimeException("readSamples() -- IOException trying to read from file.", e)); + protected void populate_component() { + this.registryObject_component = this.retrieveObject_component(); + if (this.registryObject_component == null) { + throw (new RegistryObjectNotFoundException( + "Object Component '" + + this.component_name + + "' for Object " + + this.dp.registryObject.get_id().toString() + + " not found in registry.")); } - return data.getSamples(); } void register_me_in_registry() { diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel.java new file mode 100644 index 0000000..9102a23 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel.java @@ -0,0 +1,34 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import org.fairdatapipeline.file.CleanableFileChannel; + +abstract class Object_component_read_filechannel extends Object_component_read { + Object_component_read_filechannel(Data_product dp, String component_name) { + super(dp, component_name); + } + + Object_component_read_filechannel(Data_product dp) { + super(dp); + } + + /** + * get the CleanableFileChannel to read directly from the file. only for whole_object component. + * + * @return CleanableFileChannel the filechannel to read from. + * @throws IOException if the file can't be opened. + */ + public CleanableFileChannel readFileChannel() throws IOException { + if (!this.whole_object) { + throw (new IllegalActionException( + "You shouldn't try to read directly from a Data Product with named components.")); + } + this.been_used = true; + return this.getFileChannel(); + } + + CleanableFileChannel getFileChannel() throws IOException { + this.been_used = true; + return ((Data_product_read_filechannel) this.dp).getFilechannel(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_link.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_link.java new file mode 100644 index 0000000..945934d --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_link.java @@ -0,0 +1,23 @@ +package org.fairdatapipeline.api; + +import java.nio.file.Path; + +public class Object_component_read_filechannel_link extends Object_component_read_filechannel { + Object_component_read_filechannel_link(Data_product dp) { + super(dp); + } + + /** + * get the filePath to read from; only for whole_object component + * + * @return Path the Path of the data object. + */ + public Path readLink() { + if (!this.whole_object) { + throw (new IllegalActionException( + "You shouldn't try to read directly from a Data Product with named components.")); + } + this.been_used = true; + return this.dp.getFilePath(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_toml.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_toml.java new file mode 100644 index 0000000..9815342 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_filechannel_toml.java @@ -0,0 +1,60 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import java.util.List; +import org.fairdatapipeline.distribution.Distribution; +import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.parameters.ReadComponent; + +public class Object_component_read_filechannel_toml extends Object_component_read_filechannel { + Object_component_read_filechannel_toml(Data_product dp, String component_name) { + super(dp, component_name); + } + + /** + * read the Estimate that was stored as this component in a TOML file. + * + * @return the estimate as Number + */ + public Number readEstimate() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = + dp.coderun.parameterDataReader.read(fileChannel, this.registryObject_component.getName()); + } catch (IOException e) { + throw (new RuntimeException("readEstimate() -- IOException trying to read from file", e)); + } + return data.getEstimate(); + } + + /** + * read the Distribution that was stored as this component in a TOML file. + * + * @return the Distribution + */ + public Distribution readDistribution() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); + } catch (IOException e) { + throw (new RuntimeException( + "readDistribution() -- IOException trying to read from file.", e)); + } + return data.getDistribution(); + } + + /** + * read the Samples that were stored as this component in a TOML file. + * + * @return the Samples object + */ + public List readSamples() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); + } catch (IOException e) { + throw (new RuntimeException("readSamples() -- IOException trying to read from file.", e)); + } + return data.getSamples(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read_nc.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_nc.java new file mode 100644 index 0000000..27a78ee --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_nc.java @@ -0,0 +1,131 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import java.util.Arrays; +import org.fairdatapipeline.netcdf.NetcdfReader; +import org.fairdatapipeline.netcdf.VariableName; +import org.fairdatapipeline.objects.NumericalArray; +import org.fairdatapipeline.objects.VariableDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.Variable; + +public class Object_component_read_nc extends Object_component_read { + private static final Logger LOGGER = LoggerFactory.getLogger(Object_component_read_nc.class); + int[] origin_read_pointer; + boolean eof = false; + Variable variable; + + /** + * @param dp the data product that we belong to + * @param variableName the variable name (groupname and local name) of the variable to read. + * @throws IOException if the file cannot be opened for reading. + */ + Object_component_read_nc(Data_product dp, VariableName variableName) throws IOException { + super(dp, variableName.getFullPath()); + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + this.variable = r.getVariable(variableName); + origin_read_pointer = r.getShape(this.variable); + for (int i = 0; i < origin_read_pointer.length; i++) origin_read_pointer[i] = 0; + } + + public VariableDefinition getVardef() throws IOException { + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + return r.getArray(this.component_name); + } + + /** + * read the complete data + * + * @return + * @throws IOException + */ + public NumericalArray readArray() throws IOException { + this.been_used = true; + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + return r.read(r.getVariable(this.component_name)); + } + + void check_match(int[] shape, int[] vshape, int from) { + if (shape.length != vshape.length) + throw new IllegalArgumentException("shape has to match number of dims in variable"); + for (int i = from; i < shape.length; i++) { + if (shape[i] != vshape[i]) + throw new IllegalArgumentException( + "final dimensions in shape must match final dimensions in variable"); + } + LOGGER.trace("shape and vshape match from {}", from); + } + + /** + * read a slice of data, of size 'shape'. + * + * @param shape + * @return + * @throws IOException + */ + public NumericalArray readArray(int[] shape) throws IOException { + if (shape.length != origin_read_pointer.length) + throw new IllegalArgumentException("shape has to match number of dims in variable"); + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + int[] vshape = r.getShape(this.variable); + this.been_used = true; + // + NumericalArray na = r.read(this.variable, origin_read_pointer, shape); + // which is the 'update' dimension; ie the dimension that we increment with each read? + int update_dim = shape.length - 1; + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("readArray({})", Arrays.toString(shape)); + LOGGER.trace("origin_read_pointer: {}", Arrays.toString(origin_read_pointer)); + LOGGER.trace("variable shape: {}", Arrays.toString(vshape)); + } + for (int i = 0; i < shape.length; i++) { + if (shape[i] != 1) { + if (shape[i] == vshape[i]) { + // make sure further shape matches vshape + check_match(shape, vshape, i + 1); + if (i > 0) { + update_dim = i - 1; + LOGGER.trace("i = {}; update_dim will be i-1={}", i, i - 1); + } else { + LOGGER.trace("i = 0; EOF=true"); + eof = true; + } + } else if (shape[i] > vshape[i]) { + throw new IllegalArgumentException( + "shape[" + i + "] is larger than the variable shape[" + i + "]"); + } else { + check_match(shape, vshape, i + 1); + update_dim = i; + } + break; + } + } + LOGGER.trace("update dim: {}", update_dim); + LOGGER.trace("EOF? {}", eof); + if (!eof) { + origin_read_pointer[update_dim] += shape[update_dim]; + LOGGER.trace( + "updated origin_read_pointer[{}]: += {} becomes {}", + update_dim, + shape[update_dim], + origin_read_pointer[update_dim]); + while (update_dim > 0 && origin_read_pointer[update_dim] >= vshape[update_dim]) { + LOGGER.trace( + "reducing origin_read_pointer[{}] to 0, (and increasing origin_read_pointer[{}] to {})", + update_dim, + update_dim - 1, + origin_read_pointer[update_dim - 1] + 1); + origin_read_pointer[update_dim] = 0; + origin_read_pointer[update_dim - 1] += 1; + update_dim -= 1; + } + if (update_dim == 0 && origin_read_pointer[0] >= vshape[0]) eof = true; + } + return na; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read_table.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_table.java new file mode 100644 index 0000000..11b109f --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read_table.java @@ -0,0 +1,69 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.fairdatapipeline.netcdf.NetcdfComponentNotfoundException; +import org.fairdatapipeline.netcdf.NetcdfComponentWrongTypeException; +import org.fairdatapipeline.netcdf.NetcdfReader; +import org.fairdatapipeline.netcdf.VariableName; +import org.fairdatapipeline.objects.TableDefinition; +import ucar.nc2.Variable; + +public class Object_component_read_table extends Object_component_read { + int[] origin_read_pointer; + boolean[] eof; + + TableDefinition tableDef; + List variables; + int length; + + /** + * @param dp the data product that we belong to + * @param variableName the variable name (groupname and local name) of the variable to read. + * @throws IOException if the file cannot be opened for reading. + */ + Object_component_read_table(Data_product dp, VariableName variableName) + throws IOException, NetcdfComponentNotfoundException, NetcdfComponentWrongTypeException { + super(dp, variableName.getFullPath()); + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + this.tableDef = r.getTable(variableName); + this.length = tableDef.getSize(); + this.eof = new boolean[this.length]; + this.origin_read_pointer = new int[this.length]; + } + + public TableDefinition getTabledef() { + return this.tableDef; + } + + void getVariables(NetcdfReader r) { + this.variables = new ArrayList<>(); + for (int i = 0; i < this.tableDef.getColumns().length; i++) { + this.variables.add(r.getVariable(this.tableDef.getVariableName(i))); + } + } + + public Object readData(int variableNr) throws IOException { + this.been_used = true; + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + if (this.variables == null) this.getVariables(r); + this.eof[variableNr] = true; + return r.readObj(variables.get(variableNr)); + } + + public Object readData(int variableNr, int num_items) throws IOException { + this.been_used = true; + Data_product_read_nc dpr = (Data_product_read_nc) this.dp; + NetcdfReader r = dpr.getNetcdfReader(); + if (this.variables == null) this.getVariables(r); + int[] origin = new int[] {origin_read_pointer[variableNr]}; + int[] shape = new int[] {num_items}; + Object o = r.readObj(variables.get(variableNr), origin, shape); + this.origin_read_pointer[variableNr] = this.origin_read_pointer[variableNr] + num_items; + if (num_items >= this.length) this.eof[variableNr] = true; + return o; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java index 5204143..fdaa24a 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java @@ -1,13 +1,9 @@ package org.fairdatapipeline.api; -import java.io.IOException; -import java.nio.file.Path; import java.util.Map; import org.fairdatapipeline.dataregistry.content.RegistryObject_component; -import org.fairdatapipeline.distribution.Distribution; -import org.fairdatapipeline.estimate.ImmutableEstimate; -import org.fairdatapipeline.file.CleanableFileChannel; -import org.fairdatapipeline.samples.Samples; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This represents an object_component to write to (or raise issues with) An object_component @@ -16,105 +12,39 @@ * not enforced at the moment. You should only ever either write to the whole_object, OR to the * named components, not both. This also is not enforced at the moment. */ -public class Object_component_write extends Object_component { +abstract class Object_component_write extends Object_component { + private static final Logger LOGGER = LoggerFactory.getLogger(Object_component_write.class); + Object_component_write(Data_product dp, String component_name) { super(dp, component_name); } Object_component_write(Data_product dp) { - super(dp, "whole_object", true); + super(dp); } protected void populate_component() { this.registryObject_component = new RegistryObject_component(component_name); } - /** - * get the filePath to write; only for whole_object component - * - * @return Path the Path of the data object. - */ - public Path writeLink() { - if (!this.whole_object) { - throw (new IllegalActionException( - "You shouldn't try to write directly to a Data Product with named components.")); - } - this.been_used = true; - return this.dp.getFilePath(); - } - - /** - * get the CleanableFileChannel to write directly to the file. only for whole_object component. - * - * @return CleanableFileChannel the filechannel to write to. - * @throws IOException if the file can't be opened. - */ - public CleanableFileChannel writeFileChannel() throws IOException { - if (!this.whole_object) { - throw (new IllegalActionException( - "You shouldn't try to write directly to a Data Product with named components.")); - } - this.been_used = true; - return this.getFileChannel(); - } - - /** - * write a Number as an Estimate, as this named component in the data product. - * - * @param estimateNumber the number to write. - */ - public void writeEstimate(Number estimateNumber) { - var estimate = - ImmutableEstimate.builder().internalValue(estimateNumber).rng(this.dp.coderun.rng).build(); - - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, estimate); - } catch (IOException e) { - throw (new RuntimeException("writeEstimate() -- IOException trying to write to file.", e)); - } - } - - /** - * write a Distribution, as this named component in the data product. - * - * @param distribution the Distribution to write - */ - public void writeDistribution(Distribution distribution) { - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, distribution); - } catch (IOException e) { - throw (new RuntimeException( - "writeDistribution() -- IOException trying to write to file.", e)); - } - } - - /** - * write Samples, as this named component in the data product. - * - * @param samples a Samples object containing the samples - */ - public void writeSamples(Samples samples) { - try (CleanableFileChannel fileChannel = this.getFileChannel()) { - this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, samples); - } catch (IOException e) { - throw (new RuntimeException("writeSamples() -- IOException trying to write to file.", e)); - } - } - void register_me_in_code_run() { if (this.been_used) this.dp.coderun.addOutput(this.registryObject_component.getUrl()); } void register_me_in_registry() { - if (!been_used) return; // don't register a component unless it has been written to + if (!been_used) { + LOGGER.trace("unused Object_component_write not being stored in registry."); + return; // don't register a component unless it has been written to + } if (this.whole_object) { + LOGGER.trace("storing Object_component_write (whole_object)"); Map find_whole_object = Map.of("object", dp.registryObject.get_id().toString(), "whole_object", "true"); RegistryObject_component objComponent = (RegistryObject_component) dp.coderun.restClient.getFirst(RegistryObject_component.class, find_whole_object); if (objComponent == null) { - throw (new RegistryObjectNotfoundException( + throw (new RegistryObjectNotFoundException( "Can't find the 'whole_object' component for obj " + dp.registryObject.get_id())); } this.registryObject_component = objComponent; @@ -122,6 +52,7 @@ void register_me_in_registry() { // the referenced Object_component_write so that this can later be stored as a // code_run output. } else { + LOGGER.trace("storing Object_component_write (not whole)"); // component != whole_object this.registryObject_component.setObject(dp.registryObject.getUrl()); RegistryObject_component objComponent = @@ -139,4 +70,6 @@ void register_me_in_registry() { // output } } + + abstract void write_preset_data(); } diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_array.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_array.java new file mode 100644 index 0000000..63d3729 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_array.java @@ -0,0 +1,124 @@ +package org.fairdatapipeline.api; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import org.fairdatapipeline.netcdf.NetcdfBuilder; +import org.fairdatapipeline.netcdf.NetcdfWriter; +import org.fairdatapipeline.objects.DimensionalVariableDefinition; +import org.fairdatapipeline.objects.NumericalArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +public class Object_component_write_array extends Object_component_write { + private static final Logger LOGGER = LoggerFactory.getLogger(Object_component_write_array.class); + DimensionalVariableDefinition nadef; + Variable variable; + int[] write_pointer; + int[] shape; + boolean eof = false; + + Object_component_write_array(Data_product_write_nc dp, DimensionalVariableDefinition nadef) { + super(dp, nadef.getVariableName().toString()); + this.nadef = nadef; + NetcdfBuilder nBuilder = ((Data_product_write_nc) this.dp).getNetCDFBuilder(); + nBuilder.prepare(nadef); + } + + private void getVariable() { + if (this.variable != null) return; + this.been_used = true; + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + this.variable = nWriter.getVariable(this.nadef.getVariableName()); + this.shape = this.variable.getShape(); + this.write_pointer = new int[this.shape.length]; + } + + void check_match(int[] shape, int[] vshape, int from) { + if (shape.length != vshape.length) + throw new IllegalArgumentException("shape has to match number of dims in variable"); + for (int i = from; i < shape.length; i++) { + if (shape[i] != vshape[i]) + throw new IllegalArgumentException( + "final dimensions in shape must match final dimensions in variable"); + } + LOGGER.trace("shape and vshape match from {}", from); + } + + /** + * writes the whole data (if nadat.getShape equals variable.getShape) or write a slice and update + * a write-pointer. + * + * @param nadat the data to be written + * @throws EOFException if we are trying to write beyong the end of the file. + */ + public void writeArrayData(NumericalArray nadat) throws IOException { + LOGGER.trace("writeArrayData"); + if (eof) throw (new EOFException("trying to write beyond end of data")); + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + if (this.variable == null) + this.getVariable(); // this.variable = nWriter.getVariable(this.component_name, this.nadef); + if (Arrays.equals(nadat.getShape(), this.variable.getShape())) { + // nadat contains ALL the data for the variable.. write it from its start. + try { + nWriter.writeArrayData(variable, nadat); + } catch (InvalidRangeException e) { + // TODO: error handling + } + eof = true; + } else { + try { + Array a = Array.makeFromJavaArray(nadat.asObject()); + while (a.getShape().length < this.shape.length) { + a = Array.makeArrayRankPlusOne(a); + if (LOGGER.isTraceEnabled()) + LOGGER.trace("makeArrayRankPlusOne: {}", Arrays.toString(a.getShape())); + } + + if (LOGGER.isTraceEnabled()) + LOGGER.trace( + "nWriter.writeArrayData() a = {}; write_pointer: {}", + Arrays.toString(a.getShape()), + Arrays.toString(this.write_pointer)); + nWriter.writeArrayData(variable, a, this.write_pointer); + int update_dimension = this.shape.length - 1; + for (int i = 0; i < this.shape.length; i++) { + if (a.getShape()[i] != 1) { + if (a.getShape()[i] == this.shape[i]) { + update_dimension = i - 1; + } else { + update_dimension = i; + } + break; + } + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("line 96 .. update_dimension: {}", update_dimension); + LOGGER.trace("this.shape: {}", Arrays.toString(this.shape)); + LOGGER.trace("a.shape: {}", Arrays.toString(a.getShape())); + LOGGER.trace("update_dim: {}", update_dimension); + } + this.write_pointer[update_dimension] += a.getShape()[update_dimension]; + while (this.shape[update_dimension] != 0 + && this.write_pointer[update_dimension] >= this.shape[update_dimension]) { + if (update_dimension == 0) { + eof = true; + return; + } + update_dimension -= 1; + this.write_pointer[update_dimension] += 1; + this.write_pointer[update_dimension + 1] = 0; + } + } catch (InvalidRangeException e) { + // TODO: error handling + } + } + } + + void write_preset_data() { + // do nothing. nadef doesn't have preset data. + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_dimension.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_dimension.java new file mode 100644 index 0000000..c850b30 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_dimension.java @@ -0,0 +1,90 @@ +package org.fairdatapipeline.api; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import org.fairdatapipeline.netcdf.NetcdfBuilder; +import org.fairdatapipeline.netcdf.NetcdfWriter; +import org.fairdatapipeline.objects.CoordinateVariableDefinition; +import org.fairdatapipeline.objects.NumericalArray; +import org.fairdatapipeline.objects.NumericalArrayImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +public class Object_component_write_dimension extends Object_component_write { + private static final Logger LOGGER = + LoggerFactory.getLogger(Object_component_write_dimension.class); + CoordinateVariableDefinition dimdef; + Variable variable; + int[] write_pointer; + int[] shape; + boolean eof = false; + + Object_component_write_dimension(Data_product_write_nc dp, CoordinateVariableDefinition dimdef) { + // dimdef.getVariableName().toString: /my/path + / + dimensionName = component_name + super(dp, dimdef.getVariableName().toString()); + this.dimdef = dimdef; + NetcdfBuilder nBuilder = ((Data_product_write_nc) this.dp).getNetCDFBuilder(); + nBuilder.prepare(dimdef); + } + + private void getVariable() { + if (this.variable != null) return; + this.been_used = true; + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + this.variable = nWriter.getVariable(this.dimdef.getVariableName()); + this.shape = this.variable.getShape(); + this.write_pointer = new int[] {0}; + } + + public void writeData() throws IOException { + if (this.dimdef.getValues() != null) { + this.writeData(new NumericalArrayImpl(this.dimdef.getValues())); + } + } + + /** + * writes the whole data (if nadat.getShape equals variable.getShape) or write a slice and update + * a write-pointer. + * + * @param nadat the data we want to write. + * @throws IOException if the write fails. + */ + public void writeData(NumericalArray nadat) throws /*EOFException, */ IOException { + if (eof) throw (new EOFException("trying to write beyond end of data")); + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + if (this.variable == null) this.getVariable(); + if (Arrays.equals(nadat.getShape(), this.shape)) { + // nadat contains ALL the data for the variable.. write it from its start. + try { + nWriter.writeArrayData(variable, nadat); + } catch (InvalidRangeException e) { + // TODO: error handling + } + eof = true; + } else { + try { + Array a = Array.makeFromJavaArray(nadat.asObject()); + if (a.getShape().length == 0) a = Array.makeArrayRankPlusOne(a); + nWriter.writeArrayData(variable, a, this.write_pointer); + this.write_pointer[0] += 1; + if (this.shape[0] != 0 && this.write_pointer[0] >= this.shape[0]) { + eof = true; + } + } catch (InvalidRangeException e) { + // TODO: error handling + } + } + } + + void write_preset_data() { + try { + writeData(); + } catch (IOException e) { + LOGGER.error("failed to write preset data", e); + } + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel.java new file mode 100644 index 0000000..109d051 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel.java @@ -0,0 +1,34 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import org.fairdatapipeline.file.CleanableFileChannel; + +abstract class Object_component_write_filechannel extends Object_component_write { + Object_component_write_filechannel(Data_product_write_filechannel dp, String component_name) { + super(dp, component_name); + } + + Object_component_write_filechannel(Data_product_write_filechannel dp) { + super(dp); + } + + /** + * get the CleanableFileChannel to write directly to the file. only for whole_object component. + * + * @return CleanableFileChannel the filechannel to write to. + * @throws IOException if the file can't be opened. + */ + public CleanableFileChannel writeFileChannel() throws IOException { + if (!this.whole_object) { + throw (new IllegalActionException( + "You shouldn't try to write directly to a Data Product with named components.")); + } + this.been_used = true; + return this.getFileChannel(); + } + + CleanableFileChannel getFileChannel() throws IOException { + this.been_used = true; + return ((Data_product_write_filechannel) this.dp).getFilechannel(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_link.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_link.java new file mode 100644 index 0000000..ec460fa --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_link.java @@ -0,0 +1,23 @@ +package org.fairdatapipeline.api; + +import java.nio.file.Path; + +public class Object_component_write_filechannel_link extends Object_component_write_filechannel { + Object_component_write_filechannel_link(Data_product_write_link dp) { + super(dp); + } + + /** + * get the filePath to write; only for whole_object component + * + * @return Path the Path of the data object. + */ + public Path writeLink() { + this.been_used = true; + return this.dp.getFilePath(); + } + + void write_preset_data() { + // no preset data to write. + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_toml.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_toml.java new file mode 100644 index 0000000..341d78a --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_filechannel_toml.java @@ -0,0 +1,59 @@ +package org.fairdatapipeline.api; + +import java.io.IOException; +import org.fairdatapipeline.distribution.Distribution; +import org.fairdatapipeline.estimate.ImmutableEstimate; +import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.samples.Samples; + +public class Object_component_write_filechannel_toml extends Object_component_write_filechannel { + Object_component_write_filechannel_toml( + Data_product_write_filechannel dp, String component_name) { + super(dp, component_name); + } + /** + * write a Number as an Estimate, as this named component in the data product. + * + * @param estimateNumber the number to write. + */ + public void writeEstimate(Number estimateNumber) { + var estimate = + ImmutableEstimate.builder().internalValue(estimateNumber).rng(this.dp.coderun.rng).build(); + + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, estimate); + } catch (IOException e) { + throw (new RuntimeException("writeEstimate() -- IOException trying to write to file.", e)); + } + } + /** + * write a Distribution, as this named component in the data product. + * + * @param distribution the Distribution to write + */ + public void writeDistribution(Distribution distribution) { + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, distribution); + } catch (IOException e) { + throw (new RuntimeException( + "writeDistribution() -- IOException trying to write to file.", e)); + } + } + + /** + * write Samples, as this named component in the data product. + * + * @param samples a Samples object containing the samples + */ + public void writeSamples(Samples samples) { + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, samples); + } catch (IOException e) { + throw (new RuntimeException("writeSamples() -- IOException trying to write to file.", e)); + } + } + + void write_preset_data() { + // no preset data to write. + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write_table.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_table.java new file mode 100644 index 0000000..93876a9 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write_table.java @@ -0,0 +1,94 @@ +package org.fairdatapipeline.api; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import org.fairdatapipeline.netcdf.NetcdfBuilder; +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.NetcdfWriter; +import org.fairdatapipeline.netcdf.VariableName; +import org.fairdatapipeline.objects.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; + +public class Object_component_write_table extends Object_component_write { + private static final Logger LOGGER = LoggerFactory.getLogger(Object_component_write_table.class); + TableDefinition tabledef; + int[] write_index; + // boolean[] eof; + + Variable[] variables; + + Object_component_write_table(Data_product_write_nc dp, TableDefinition tabledef) { + super(dp, tabledef.getGroupName().toString()); + this.tabledef = tabledef; + NetcdfBuilder nBuilder = ((Data_product_write_nc) this.dp).getNetCDFBuilder(); + nBuilder.prepare(tabledef); + } + + private void getVariables() { + LOGGER.trace("getVariables()"); + if (this.variables != null) return; + this.been_used = true; + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + LocalVariableDefinition[] columns = tabledef.getColumns(); + this.variables = new Variable[columns.length]; + // this.eof = new boolean[columns.length]; + this.write_index = new int[columns.length]; + LOGGER.trace("write_index created"); + for (int i = 0; i < columns.length; i++) { + // this.eof[i] = false; + this.write_index[i] = 0; + this.variables[i] = + nWriter.getVariable(new VariableName(columns[i].getLocalName(), tabledef.getGroupName())); + } + + // this.shape = this.variable.getShape(); + // this.write_pointer = new int[this.shape.length]; + } + + public void writeData(int column_index, int[] ints) throws IOException { + writeData(ints, column_index); + } + + public void writeData(int column_index, double[] doubles) throws IOException { + writeData(doubles, column_index); + } + + public void writeData(int column_index, String[] strings) throws IOException { + writeData(strings, column_index); + } + + private void writeData(Object data, int column_index) throws IOException { + int[] origin = new int[1]; + if (this.variables == null) + this.getVariables(); // this.variable = nWriter.getVariable(this.component_name, this.nadef); + if (this.tabledef.getSize() != 0 && this.write_index[column_index] >= this.tabledef.getSize()) + throw (new EOFException("trying to write beyond end of data")); + NetcdfWriter nWriter = ((Data_product_write_nc) this.dp).getNetCDFWriter(); + if (column_index >= this.variables.length) + throw (new IllegalArgumentException("This table doesn't have that many columns..")); + origin[0] = this.write_index[column_index]; + try { + Array a = NetcdfDataType.translateArray(data); + if (LOGGER.isTraceEnabled()) + LOGGER.trace( + "writeArrayData({}, {}, {})", + column_index, + Arrays.toString(a.getShape()), + Arrays.toString(origin)); + nWriter.writeArrayData(this.variables[column_index], a, origin); + this.write_index[column_index] += a.getShape()[0]; + } catch (InvalidRangeException e) { + LOGGER.error("writeData to table failed..", e); + // TODO: error handling + } + } + + void write_preset_data() { + // there is no preset data on tables. + } +} diff --git a/api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotfoundException.java b/api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotFoundException.java similarity index 65% rename from api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotfoundException.java rename to api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotFoundException.java index 4a66045..0eba412 100644 --- a/api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotfoundException.java +++ b/api/src/main/java/org/fairdatapipeline/api/RegistryObjectNotFoundException.java @@ -1,13 +1,13 @@ package org.fairdatapipeline.api; /** Failure to retrieve an expected object from the registry. */ -public class RegistryObjectNotfoundException extends RuntimeException { +public class RegistryObjectNotFoundException extends RuntimeException { /** * Constructor * * @param message The error message. */ - public RegistryObjectNotfoundException(String message) { + public RegistryObjectNotFoundException(String message) { super(message); } @@ -17,7 +17,7 @@ public RegistryObjectNotfoundException(String message) { * @param message The error message. * @param e The Exception that caused it. */ - public RegistryObjectNotfoundException(String message, Exception e) { + public RegistryObjectNotFoundException(String message, Exception e) { super(message, e); } } diff --git a/api/src/main/java/org/fairdatapipeline/api/Storage_location.java b/api/src/main/java/org/fairdatapipeline/api/Storage_location.java index cf6022c..649b120 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Storage_location.java +++ b/api/src/main/java/org/fairdatapipeline/api/Storage_location.java @@ -13,7 +13,7 @@ /** This is used to store a file or a remote repo to the registry as a RegistryStorage_location. */ class Storage_location { - private static final Logger logger = LoggerFactory.getLogger(Storage_location.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Storage_location.class); RegistryStorage_location registryStorage_location; /** @@ -73,15 +73,15 @@ void create_storagelocation( if (this.registryStorage_location != null) { // there is an already existing StorageLocation for this hash; we may need to delete the file if (filePath_to_delete_if_hash_exists == null) { - logger.trace( + LOGGER.trace( "Not deleting file after finding existing storage location with identical hash."); } else { try { Files.delete(filePath_to_delete_if_hash_exists); - logger.trace( + LOGGER.trace( "Deleting file after finding existing storage location with identical hash."); } catch (IOException e) { - logger.error( + LOGGER.error( "Failed to delete file after finding existing storage location with identical hash: {}", filePath_to_delete_if_hash_exists); } diff --git a/api/src/main/java/org/fairdatapipeline/arrays/NumericalArray.java b/api/src/main/java/org/fairdatapipeline/arrays/NumericalArray.java new file mode 100644 index 0000000..e05f93f --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/arrays/NumericalArray.java @@ -0,0 +1,3 @@ +package org.fairdatapipeline.arrays; + +public class NumericalArray {} diff --git a/api/src/main/java/org/fairdatapipeline/config/Config.java b/api/src/main/java/org/fairdatapipeline/config/Config.java index 9161e63..6bb160a 100644 --- a/api/src/main/java/org/fairdatapipeline/config/Config.java +++ b/api/src/main/java/org/fairdatapipeline/config/Config.java @@ -86,8 +86,8 @@ interface ConfigItem { ImmutableConfigUseItem use(); /** - * Config write items may specify the file_type (such as csv, h5, or toml) but this can also be - * specified using {@link Coderun#get_dp_for_write(String, String)} method. + * Config write items may specify the file_type (such as csv, txt) but this can also be + * specified using {@link Coderun#get_dp_for_write_link(String, String)} method. * * @return String the file_type file extension, if given. */ diff --git a/api/src/main/java/org/fairdatapipeline/hash/Sha1Hasher.java b/api/src/main/java/org/fairdatapipeline/hash/Sha1Hasher.java index c70d8af..8a15784 100644 --- a/api/src/main/java/org/fairdatapipeline/hash/Sha1Hasher.java +++ b/api/src/main/java/org/fairdatapipeline/hash/Sha1Hasher.java @@ -2,7 +2,7 @@ import static org.apache.commons.codec.digest.DigestUtils.sha1Hex; -class Sha1Hasher { +public class Sha1Hasher { public String hash(String key) { return sha1Hex(key); } diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfBuilder.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfBuilder.java new file mode 100644 index 0000000..8b40a78 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfBuilder.java @@ -0,0 +1,309 @@ +package org.fairdatapipeline.netcdf; + +import java.io.IOException; +import java.lang.ref.Cleaner; +import java.lang.ref.Cleaner.Cleanable; +import java.util.*; +import org.fairdatapipeline.objects.CoordinateVariableDefinition; +import org.fairdatapipeline.objects.DimensionalVariableDefinition; +import org.fairdatapipeline.objects.TableDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.DataType; +import ucar.nc2.*; +import ucar.nc2.write.Nc4Chunking; +import ucar.nc2.write.Nc4ChunkingStrategy; +import ucar.nc2.write.NetcdfFileFormat; +import ucar.nc2.write.NetcdfFormatWriter; + +/** */ +public class NetcdfBuilder implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfBuilder.class); + private static final Cleaner cleaner = Cleaner.create(); + // todo: we should not create more cleaners than necessary + private final Cleanable cleanable; + private final NetcdfBuilderWrapper netcdfBuilderWrapper; + + public NetcdfBuilder( + String filePath, + Nc4Chunking.Strategy nc4chunkingStrategy, + int nc4deflateLevel, + boolean nc4shuffle, + Runnable onClose) { + LOGGER.trace("NetcdfBuilder({}) ", filePath); + this.netcdfBuilderWrapper = + new NetcdfBuilderWrapper( + filePath, nc4chunkingStrategy, nc4deflateLevel, nc4shuffle, onClose); + this.cleanable = cleaner.register(this, this.netcdfBuilderWrapper); + } + + // Defining a resource that requires cleaning + private static class NetcdfBuilderWrapper implements Runnable { + private final NetcdfFormatWriter.Builder builder; + private boolean has_been_built = false; + + private final Runnable runOnClose; + + NetcdfBuilderWrapper( + String filePath, + Nc4Chunking.Strategy nc4chunkingStrategy, + int nc4deflateLevel, + boolean nc4shuffle, + Runnable runOnClose) { + Nc4Chunking chunker = + Nc4ChunkingStrategy.factory(nc4chunkingStrategy, nc4deflateLevel, nc4shuffle); + + this.builder = + NetcdfFormatWriter.createNewNetcdf4(NetcdfFileFormat.NETCDF4, filePath, chunker); + + this.runOnClose = runOnClose; + } + + public NetcdfFormatWriter build() throws IOException { + NetcdfFormatWriter w = builder.build(); + has_been_built = true; + return w; + } + + // Invoked by close method or cleaner + @Override + public void run() { + LOGGER.trace("run() invoked by cleaner"); + runOnClose.run(); + if (!has_been_built) { + // write the file: + try (NetcdfFormatWriter w = builder.build()) { + // do nothing + } catch (IOException e) { + LOGGER.error("failed to write netcdf file", e); + } + } else { + // it has already been built. do nothing. + } + } + } + + NetcdfFormatWriter build() throws IOException { + return this.netcdfBuilderWrapper.build(); + } + + /** + * prepare a separate dimension in its own group, which other arrays can later refer to. this + * allows different arrays to share dimensions. these shared dimensions MUST reside in parent + * groups of the array group that is trying to use it. i.e. /time_group/time_dim can be used as a + * dimension of the array in /time_group/personal_data/ but cannot be used as a dimension of the + * array in /other_group/somedata + * + * @param coordinateVariable + */ + public void prepare(CoordinateVariableDefinition coordinateVariable) { + Group.Builder gb = + getGroup( + netcdfBuilderWrapper.builder.getRootGroup(), + coordinateVariable.getVariableName().getGroupName().toString()); + Dimension d; + if (coordinateVariable.isUnlimited()) { + d = + new Dimension( + coordinateVariable.getVariableName().getName().toString(), 0, true, true, false); + } else { + d = + new Dimension( + coordinateVariable.getVariableName().getName().toString(), + coordinateVariable.getSize()); + } + gb.addDimension(d); + Variable.Builder varbuilder = + Variable.builder() + .setName(coordinateVariable.getVariableName().getName().toString()) + .setDataType(coordinateVariable.getDataType().translate()) + .setDimensions(Collections.singletonList(d)); + if (coordinateVariable.getDescription().length() > 0) + varbuilder.addAttribute( + new Attribute(NetcdfNames.ATTRIB_DESC, coordinateVariable.getDescription())); + if (coordinateVariable.getUnits().length() > 0) + varbuilder.addAttribute( + new Attribute(NetcdfNames.ATTRIB_UNITS, coordinateVariable.getUnits())); + if (coordinateVariable.getLong_name().length() > 0) + varbuilder.addAttribute( + new Attribute(NetcdfNames.ATTRIB_LNAME, coordinateVariable.getLong_name())); + if (coordinateVariable.getMissingValue() != null) { + // coordinateVariables should not have any missing values. + if (coordinateVariable.getMissingValue().getClass() == String.class) { + varbuilder.addAttribute( + new Attribute( + NetcdfNames.ATTRIB_FILLVALUE, (String) coordinateVariable.getMissingValue())); + } else { + varbuilder.addAttribute( + new Attribute( + NetcdfNames.ATTRIB_FILLVALUE, (Number) coordinateVariable.getMissingValue())); + } + } + gb.addVariable(varbuilder); + } + + public void prepare(TableDefinition tabledef) { + String dimensionName = "index"; + Group.Builder gb = + getGroup( + netcdfBuilderWrapper.builder.getRootGroup(), tabledef.getGroupName().toString(), true); + Dimension d; + if (tabledef.isUnlimited()) { + d = new Dimension(dimensionName, 0, true, true, false); + } else { + d = new Dimension(dimensionName, tabledef.getSize()); + } + gb.addDimension(d); + if (tabledef.getDescription().length() > 0) + gb.addAttribute(new Attribute(NetcdfNames.ATTRIB_DESC, tabledef.getDescription())); + if (tabledef.getLong_name().length() > 0) + gb.addAttribute(new Attribute(NetcdfNames.ATTRIB_LNAME, tabledef.getLong_name())); + tabledef + .getOptional_attribs() + .forEach( + (key, value) -> + gb.addAttribute( + Attribute.builder() + .setName(key) + .setDataType(DataType.STRING) + .setValues(NetcdfDataType.translateArray(value)) + .build())); + for (int i = 0; i < tabledef.getColumns().length; i++) { + this.prepare( + new DimensionalVariableDefinition( + tabledef.getColumns()[i], tabledef.getGroupName(), dimensionName), + i); + } + gb.addAttribute( + new Attribute(NetcdfNames.ATTRIB_GROUP_TYPE, NetcdfNames.ATTRIB_GROUP_TYPE_TABLE)); + } + + public void prepare(DimensionalVariableDefinition nadef) { + prepare(nadef, null); + } + + /** + * prepareArray creates the dimensions and dimension variables for the given + * NumericalArrayDefinition. It then creates the actual data variable. + * + * @param nadef + */ + public void prepare(DimensionalVariableDefinition nadef, Integer columnIndex) { + List dims = new ArrayList<>(); + VariableName vbn = nadef.getVariableName(); + NetcdfGroupName gn = vbn.getGroupName(); + String groupName = gn.toString(); + Group.Builder gb = getGroup(netcdfBuilderWrapper.builder.getRootGroup(), groupName); + for (int i = 0; i < nadef.getDimensions().length; i++) { + if (nadef.getDimensions()[i].is_size()) { + int[] values = new int[nadef.getDimensions()[i].size()]; + for (int j = 0; j < nadef.getDimensions()[i].size(); j++) values[j] = j + 1; + CoordinateVariableDefinition cvdef = + new CoordinateVariableDefinition( + new VariableName( + new NetcdfName(NetcdfNames.generatedDimName(vbn.getName().getName(), i)), + vbn.getGroupName()), + values, + "", + "", + ""); + this.prepare(cvdef); + } + } + + for (int i = 0; i < nadef.getDimensions().length; i++) { + String dimName; + if (nadef.getDimensions()[i].is_size()) { + dimName = NetcdfNames.generatedDimName(vbn.getName().getName(), i); + } else { + dimName = nadef.getDimensions()[i].name().getName(); + } + Optional optionalDimension = gb.findDimension(dimName); + if (optionalDimension.isEmpty()) + throw (new IllegalArgumentException("Can't find dimension " + dimName)); + Dimension d = optionalDimension.get(); + dims.add(d); + } + Variable.Builder varbuilder = + Variable.builder() + .setName(nadef.getVariableName().getName().toString()) + .setDataType(nadef.getDataType().translate()) + .setDimensions(dims); + if (nadef.getDescription().length() > 0) + varbuilder.addAttribute(new Attribute(NetcdfNames.ATTRIB_DESC, nadef.getDescription())); + if (nadef.getUnits().length() > 0) + varbuilder.addAttribute(new Attribute(NetcdfNames.ATTRIB_UNITS, nadef.getUnits())); + if (nadef.getLong_name().length() > 0) + varbuilder.addAttribute(new Attribute(NetcdfNames.ATTRIB_LNAME, nadef.getLong_name())); + if (nadef.getMissingValue() != null) { + if (nadef.getMissingValue().getClass() == String.class) { + varbuilder.addAttribute( + new Attribute(NetcdfNames.ATTRIB_FILLVALUE, (String) nadef.getMissingValue())); + } else { + varbuilder.addAttribute( + new Attribute(NetcdfNames.ATTRIB_FILLVALUE, (Number) nadef.getMissingValue())); + } + } + if (columnIndex != null) { + varbuilder.addAttribute(new Attribute(NetcdfNames.ATTRIB_COLUMN_INDEX, columnIndex)); + } + + gb.addVariable(varbuilder); + } + + Group.Builder getGroup(Group.Builder start_group, String group_name) { + return getGroup(start_group, group_name, false); + } + + Group.Builder getGroup(Group.Builder start_group, String group_name, boolean mustBeFresh) { + LOGGER.trace("getGroup({}, {}, {}})", start_group, group_name, mustBeFresh); + if (group_name.startsWith("/")) group_name = group_name.substring(1); + if (start_group == null) start_group = netcdfBuilderWrapper.builder.getRootGroup(); + if (group_name.equals("")) { + if (mustBeFresh) throw (new IllegalArgumentException("this group already exists")); + if (start_group.getAttributeContainer().findAttribute(NetcdfNames.ATTRIB_GROUP_TYPE) != null) + throw (new IllegalArgumentException( + "we can't create anything new in groups marked with group_type attribute.")); + return start_group; + } + String[] split = group_name.split("/", 2); + Optional optGroupBuilder = start_group.findGroupLocal(split[0]); + if (optGroupBuilder.isPresent()) { + Group.Builder found_group = optGroupBuilder.get(); + if (found_group.getAttributeContainer().findAttribute(NetcdfNames.ATTRIB_GROUP_TYPE) != null) + throw (new IllegalArgumentException( + "we can't create anything new in groups marked with group_type attribute.")); + if (split.length == 2) { + return getGroup(found_group, split[1], mustBeFresh); + } else { + if (mustBeFresh) throw (new IllegalArgumentException("this group already exists")); + return found_group; + } + } else { + if (split.length == 2) { + return createGroup(start_group, split[0], split[1]); + } else { + return createGroup(start_group, split[0], null); + } + } + } + + Group.Builder createGroup(Group.Builder start_group, String groupName, String subgroups) { + LOGGER.trace("createGroup({}, {}, {})", start_group, groupName, subgroups); + Group.Builder newGroup = Group.builder().setParentGroup(start_group).setName(groupName); + start_group.addGroup(newGroup); + if (subgroups == null) { + return newGroup; + } + String[] split = subgroups.split("/", 2); + if (split.length == 2) return createGroup(newGroup, split[0], split[1]); + return createGroup(newGroup, split[0], null); + } + + @Override + public void close() { + LOGGER.trace("close()"); + cleanable.clean(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentNotfoundException.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentNotfoundException.java new file mode 100644 index 0000000..131d102 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentNotfoundException.java @@ -0,0 +1,23 @@ +package org.fairdatapipeline.netcdf; + +/** The consumer is probably trying to do something they aren't supposed to be doing. */ +public class NetcdfComponentNotfoundException extends RuntimeException { + /** + * Constructor + * + * @param message The error message. + */ + public NetcdfComponentNotfoundException(String message) { + super(message); + } + + /** + * Constructor + * + * @param message The error message. + * @param e The Exception that caused it. + */ + public NetcdfComponentNotfoundException(String message, Exception e) { + super(message, e); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentWrongTypeException.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentWrongTypeException.java new file mode 100644 index 0000000..ef248ac --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfComponentWrongTypeException.java @@ -0,0 +1,23 @@ +package org.fairdatapipeline.netcdf; + +/** The consumer is probably trying to do something they aren't supposed to be doing. */ +public class NetcdfComponentWrongTypeException extends RuntimeException { + /** + * Constructor + * + * @param message The error message. + */ + public NetcdfComponentWrongTypeException(String message) { + super(message); + } + + /** + * Constructor + * + * @param message The error message. + * @param e The Exception that caused it. + */ + public NetcdfComponentWrongTypeException(String message, Exception e) { + super(message, e); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfDataType.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfDataType.java new file mode 100644 index 0000000..36e860a --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfDataType.java @@ -0,0 +1,100 @@ +package org.fairdatapipeline.netcdf; + +import java.lang.reflect.Array; +import org.jetbrains.annotations.NotNull; +import ucar.ma2.DataType; + +public enum NetcdfDataType { + BOOLEAN, + BYTE, + CHAR, + SHORT, + INT, + LONG, + FLOAT, + DOUBLE, + SEQUENCE, + STRING; + + DataType translate() { + return NetcdfDataType.translate(this); + } + + public static DataType translate(NetcdfDataType dataType) { + switch (dataType) { + case BOOLEAN: + return DataType.BOOLEAN; + case BYTE: + return DataType.BYTE; + case CHAR: + return DataType.CHAR; + case SHORT: + return DataType.SHORT; + case INT: + return DataType.INT; + case LONG: + return DataType.LONG; + case FLOAT: + return DataType.FLOAT; + case DOUBLE: + return DataType.DOUBLE; + case SEQUENCE: + return DataType.SEQUENCE; + case STRING: + return DataType.STRING; + } + throw (new IllegalArgumentException("unknown datatype: " + dataType)); + } + + public static NetcdfDataType translate(DataType dataType) { + if (dataType == DataType.BOOLEAN) return NetcdfDataType.BOOLEAN; + if (dataType == DataType.BYTE) return NetcdfDataType.BYTE; + if (dataType == DataType.CHAR) return NetcdfDataType.CHAR; + if (dataType == DataType.SHORT) return NetcdfDataType.SHORT; + if (dataType == DataType.INT) return NetcdfDataType.INT; + if (dataType == DataType.LONG) return NetcdfDataType.LONG; + if (dataType == DataType.FLOAT) return NetcdfDataType.FLOAT; + if (dataType == DataType.DOUBLE) return NetcdfDataType.DOUBLE; + if (dataType == DataType.SEQUENCE) return NetcdfDataType.SEQUENCE; + if (dataType == DataType.STRING) return NetcdfDataType.STRING; + throw (new IllegalArgumentException("unknown datatype: " + dataType)); + } + + public static NetcdfDataType translateDatatype(@NotNull Object o) { + if (Array.newInstance(Integer.class, 0).getClass().equals(o.getClass()) + || Array.newInstance(int.class, 0).getClass().equals(o.getClass())) { + return NetcdfDataType.INT; + } else if (Array.newInstance(Long.class, 0).getClass().equals(o.getClass()) + || Array.newInstance(long.class, 0).getClass().equals(o.getClass())) { + return NetcdfDataType.LONG; + } else if (Array.newInstance(Double.class, 0).getClass().equals(o.getClass()) + || Array.newInstance(double.class, 0).getClass().equals(o.getClass())) { + return NetcdfDataType.DOUBLE; + } else if (Array.newInstance(String.class, 0).getClass().equals(o.getClass())) { + return NetcdfDataType.STRING; + } else if (Array.newInstance(Float.class, 0).getClass().equals(o.getClass()) + || Array.newInstance(float.class, 0).getClass().equals(o.getClass())) { + return NetcdfDataType.FLOAT; + } + throw (new UnsupportedOperationException( + "can't translate object of class " + + o.getClass().getSimpleName() + + " to NetCDF data type.")); + } + + public static ucar.ma2.Array translateArray(@NotNull Object o) { + return ucar.ma2.Array.factory( + translate(translateDatatype(o)), new int[] {Array.getLength(o)}, o); + } + + /** + * @param dataType + * @param dimSizes + * @param o + * @return + */ + public static ucar.ma2.Array translateArray( + @NotNull NetcdfDataType dataType, @NotNull int[] dimSizes, @NotNull Object o) { + return ucar.ma2.Array.factory(dataType.translate(), dimSizes, o); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfGroupName.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfGroupName.java new file mode 100644 index 0000000..353d9ae --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfGroupName.java @@ -0,0 +1,33 @@ +package org.fairdatapipeline.netcdf; + +import java.util.regex.Pattern; +import javax.annotation.Nonnull; + +public class NetcdfGroupName { + @Nonnull String groupName; + + public static final Pattern groupnameP = + Pattern.compile("^(?:\\p{Alnum}[\\p{Alnum}_-]*+(?:/\\p{Alnum}[\\p{Alnum}_-]*+)*+)?$"); + + public NetcdfGroupName(@Nonnull String groupName) { + if (groupName.startsWith("/")) groupName = groupName.substring(1); + if (!groupnameP.matcher(groupName).find()) + throw (new IllegalArgumentException("not a valid netCDF group name: " + groupName)); + this.groupName = groupName; + } + + public @Nonnull String getGroupName() { + return groupName; + } + + public @Nonnull String toString() { + return getGroupName(); + } + + @Override + public boolean equals(Object other) { + return (other != null + && other.getClass() == getClass() + && ((NetcdfGroupName) other).getGroupName().equals(this.groupName)); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfName.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfName.java new file mode 100644 index 0000000..35d0a60 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfName.java @@ -0,0 +1,35 @@ +package org.fairdatapipeline.netcdf; + +import java.util.regex.Pattern; +import javax.annotation.Nonnull; + +public class NetcdfName { + @Nonnull String name; + // public static Pattern THIS PATTERN IS SIMPLIFIED TO MATCH THE VERY RESTRICTED + // FORMAT OF Datapipeline object_component + // this was based on the definition of 'bare key' from TOML. + public static final Pattern NAME_P = Pattern.compile("^\\p{Alnum}[\\p{Alnum}_-]*+$"); + public static final Pattern NAME_P_INTERNAL = + Pattern.compile("^__fdp_\\p{Alnum}[\\p{Alnum}_-]*+$"); + + public NetcdfName(@Nonnull String name) { + if (!NAME_P_INTERNAL.matcher(name).find() && !NAME_P.matcher(name).find()) + throw (new IllegalArgumentException("not a valid netCDF name: " + name)); + this.name = name; + } + + public @Nonnull String getName() { + return name; + } + + public @Nonnull String toString() { + return getName(); + } + + @Override + public boolean equals(Object other) { + return (other != null + && other.getClass() == getClass() + && ((NetcdfName) other).getName().equals(this.name)); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfNames.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfNames.java new file mode 100644 index 0000000..44b6ec0 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfNames.java @@ -0,0 +1,19 @@ +package org.fairdatapipeline.netcdf; + +public class NetcdfNames { + public static final String[] attrib_names = + new String[] {"description", "units", "long_name", "standard_name"}; + + public static String FDP_PREFIX = "__fdp_"; + public static final String ATTRIB_DESC = attrib_names[0]; + public static final String ATTRIB_UNITS = attrib_names[1]; + public static final String ATTRIB_LNAME = attrib_names[2]; + public static final String ATTRIB_FILLVALUE = "_FillValue"; + public static final String ATTRIB_GROUP_TYPE = FDP_PREFIX + "group_type"; + public static final String ATTRIB_GROUP_TYPE_TABLE = "table"; + public static final String ATTRIB_COLUMN_INDEX = FDP_PREFIX + "column_index"; + + public static String generatedDimName(String vName, int i) { + return FDP_PREFIX + vName + "_dim_" + i; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfReader.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfReader.java new file mode 100644 index 0000000..1fd6df0 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfReader.java @@ -0,0 +1,340 @@ +package org.fairdatapipeline.netcdf; + +import java.io.IOException; +import java.util.*; +import org.fairdatapipeline.objects.*; +import org.fairdatapipeline.objects.Dimension; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.*; + +public class NetcdfReader { + NetcdfFile file; + + /** + * @param fileName the fileName of the file to open. + * @throws IOException if the file cannot be opened for reading. + */ + public NetcdfReader(String fileName) throws IOException { + this.file = NetcdfFiles.open(fileName); + } + + /** + * retrieves the variable definition (all the metadata) for the given VariableName. + * + * @param variableName + * @return the variable definition (all the metadata) for the given VariableName. + */ + public VariableDefinition getArray(String variableName) { + return getArray(new VariableName(variableName)); + } + + /** + * retrieves the variable definition (all the metadata) for the given VariableName. + * + * @param variableName + * @return the variable definition (all the metadata) for the given VariableName. + * @throws NetcdfComponentNotfoundException + * @throws NetcdfComponentWrongTypeException + */ + public VariableDefinition getArray(VariableName variableName) + throws NetcdfComponentWrongTypeException, NetcdfComponentNotfoundException { + Variable v = this.getVariable(variableName); + Map argument_attribs = new HashMap<>(); + Arrays.stream(NetcdfNames.attrib_names).forEach(s -> argument_attribs.put(s, "")); + Map optional_attribs = new HashMap<>(); + for (Attribute attribute : v.attributes()) { + if (attribute.isString()) { + if (argument_attribs.containsKey(attribute.getName())) { + argument_attribs.put(attribute.getName(), attribute.getStringValue()); + } else { + if (attribute.getValues() != null) { + optional_attribs.put( + attribute.getName(), + (String[]) attribute.getValues().get1DJavaArray(DataType.STRING)); + } + } + } + } + int[] sh = v.getShape(); + + Dimension[] dims = + v.getDimensions().stream() + .map( + dim -> { + if (dim.getName().startsWith("__fdp")) + return new Dimension(sh[Integer.valueOf(dim.getName().split("_dim_")[1])]); + else return new Dimension(dim.getName()); + }) + .toArray(Dimension[]::new); + + NetcdfDataType dataType = NetcdfDataType.translate(v.getDataType()); + if (v.isCoordinateVariable()) { + return new CoordinateVariableDefinition( + variableName, + dataType, + v.getShape(0), + argument_attribs.get(NetcdfNames.attrib_names[0]), + argument_attribs.get(NetcdfNames.attrib_names[1]), + argument_attribs.get(NetcdfNames.attrib_names[2]), + optional_attribs); + + } else { + + return new DimensionalVariableDefinition( + variableName, + dataType, + dims, + argument_attribs.get(NetcdfNames.attrib_names[0]), + argument_attribs.get(NetcdfNames.attrib_names[1]), + argument_attribs.get(NetcdfNames.attrib_names[2]), + optional_attribs); + } + } + + private Object getMissingValue(Variable v) { + Attribute a = v.findAttribute(NetcdfNames.ATTRIB_FILLVALUE); + if (a == null) return null; + if (a.isString()) return a.getStringValue(); + return a.getNumericValue(); + } + + private LocalVariableDefinition makeVarDef(Variable v) { + Integer columnIndex = null; + NetcdfName localName = new NetcdfName(v.getShortName()); + NetcdfDataType dataType = NetcdfDataType.translate(v.getDataType()); + Map argument_attribs = new HashMap<>(); + Arrays.stream(NetcdfNames.attrib_names).forEach(s -> argument_attribs.put(s, "")); + Map optional_attribs = new HashMap<>(); + Object missingValue = getMissingValue(v); + for (Attribute attribute : v.attributes()) { + if (attribute.isString()) { + if (argument_attribs.containsKey(attribute.getName())) { + argument_attribs.put(attribute.getName(), attribute.getStringValue()); + } else { + if (attribute.getValues() != null) { + optional_attribs.put( + attribute.getName(), + Arrays.stream((Object[]) attribute.getValues().get1DJavaArray(DataType.STRING)) + .map(String.class::cast) + .toArray(String[]::new)); + } + } + } else { + if (attribute.getName().equals(NetcdfNames.ATTRIB_COLUMN_INDEX)) + columnIndex = (Integer) attribute.getNumericValue(); + } + } + + return new LocalVariableDefinition( + localName, + dataType, + argument_attribs.get(NetcdfNames.attrib_names[0]), + argument_attribs.get(NetcdfNames.attrib_names[1]), + argument_attribs.get(NetcdfNames.attrib_names[2]), + optional_attribs, + missingValue, + columnIndex); + } + + public TableDefinition getTable(VariableName tablename) + throws NetcdfComponentWrongTypeException, NetcdfComponentNotfoundException { + if (file.findVariable(tablename.getFullPath()) != null) + throw new NetcdfComponentWrongTypeException( + "component named " + tablename.getFullPath() + " is an array/variable, not a table"); + Group g = file.findGroup(tablename.getFullPath()); + if (g == null) + throw (new NetcdfComponentNotfoundException( + "tablename " + tablename.getFullPath() + " group not found")); + Attribute a = g.findAttribute(NetcdfNames.ATTRIB_GROUP_TYPE); + if (a == null + || a.getStringValue() == null + || !a.getStringValue().equals(NetcdfNames.ATTRIB_GROUP_TYPE_TABLE)) + throw (new NetcdfComponentNotfoundException( + "tablename " + tablename.getFullPath() + " group not found")); + List variables = getVariables(tablename); + Map argument_attribs = new HashMap<>(); + Arrays.stream(NetcdfNames.attrib_names).forEach(s -> argument_attribs.put(s, "")); + Map optional_attribs = new HashMap<>(); + for (Attribute attribute : g.attributes()) { + if (attribute.isString()) { + if (argument_attribs.containsKey(attribute.getName())) { + argument_attribs.put(attribute.getName(), attribute.getStringValue()); + } else { + if (attribute.getValues() != null + && !attribute.getName().startsWith(NetcdfNames.FDP_PREFIX)) { + optional_attribs.put( + attribute.getName(), + Arrays.stream((Object[]) attribute.getValues().get1DJavaArray(DataType.STRING)) + .map(String.class::cast) + .toArray(String[]::new)); + } + } + } + } + LocalVariableDefinition[] columns = + variables.stream().map(this::makeVarDef).toArray(LocalVariableDefinition[]::new); + + Arrays.sort( + columns, + Comparator.comparing( + p -> { + if (p.getColumnIndex() == null) return -1; + else return p.getColumnIndex(); + })); + + return new TableDefinition( + new NetcdfGroupName(tablename.getFullPath()), + (int) variables.get(0).getSize(), + argument_attribs.get(NetcdfNames.attrib_names[0]), + argument_attribs.get(NetcdfNames.attrib_names[2]), + optional_attribs, + columns); + } + + public List getVariables(VariableName tableName) { + Group g = file.findGroup(tableName.getFullPath()); + if (g == null) { + if (file.findVariable(tableName.getFullPath()) != null) + throw new NetcdfComponentWrongTypeException( + "trying to read variable " + + tableName.getFullPath() + + " as a table but it might be an array."); + throw new NetcdfComponentNotfoundException("can't find table " + tableName); + } + Attribute a = g.findAttribute(NetcdfNames.ATTRIB_GROUP_TYPE); + if (a == null + || a.getStringValue() == null + || !a.getStringValue().equals(NetcdfNames.ATTRIB_GROUP_TYPE_TABLE)) + throw new NetcdfComponentNotfoundException("can't find table " + tableName); + return g.getVariables(); + } + + /** + * get the Variable, which is needed for the read methods. + * + * @param variableName the name of the variable. + * @return the Variable (needed for read methods) + * @throws IllegalArgumentException when the variable can't be found + */ + public Variable getVariable(VariableName variableName) + throws NetcdfComponentWrongTypeException, NetcdfComponentNotfoundException { + return this.getVariable(variableName.getFullPath()); + } + + /** + * get the Variable, which is needed for the read methods. + * + * @param variablefullname the name of the variable (as a string) + * @return the Variable (needed for read methods) + * @throws IllegalArgumentException when the variable can't be found + */ + public Variable getVariable(String variablefullname) + throws NetcdfComponentWrongTypeException, NetcdfComponentNotfoundException { + Variable v = file.findVariable(variablefullname); + if (v == null) { + Group g = file.findGroup(variablefullname); + if (g != null) { + Attribute a = g.findAttribute(NetcdfNames.ATTRIB_GROUP_TYPE); + if (a != null + && a.getStringValue() != null + && a.getStringValue().equals(NetcdfNames.ATTRIB_GROUP_TYPE_TABLE)) + throw new NetcdfComponentWrongTypeException( + "Component " + + variablefullname + + " is a table, you are trying to open it as an Array"); + } + throw (new NetcdfComponentNotfoundException("can't find variable " + variablefullname)); + } + return v; + } + + /** + * get the shape of the Variable. (int array containing the sizes of all dimensions) + * + * @param v + * @return int array containing the sizes of all dimensions + */ + public int[] getShape(Variable v) { + return v.getShape(); + } + + /** + * read ALL data in the Variable. + * + * @param v the variable + * @return all data in this array + * @throws IllegalArgumentException if it runs into a problem reading. + */ + public NumericalArray read(Variable v) throws IllegalArgumentException { + return new NumericalArrayImpl(this.readObj(v)); + } + + public Object readObj(Variable v) throws IllegalArgumentException { + Array a; + try { + a = v.read(); + } catch (IOException e) { + throw (new IllegalArgumentException("problem")); + } + return a.copyToNDJavaArray(); + } + + /** + * read ALL data in the Variable. + * + * @param v the variable + * @return all data in this array + * @throws IllegalArgumentException if it runs into a problem reading. + */ + public Object readObj(Variable v, int[] origin, int[] shape) throws IllegalArgumentException { + Array a; + try { + a = v.read(origin, shape); + } catch (IOException e) { + throw (new IllegalArgumentException("problem")); + } catch (InvalidRangeException e) { + throw new IllegalArgumentException("invalid range"); + } + return a.copyToNDJavaArray(); + } + + /** + * read part of the data from Variable. + * + *

example for variable with shape {2, 3, 4} + * int[] origin = new int[] {0,0,0}; + * int[] shape = new int[] {1, 3, 4}; + * for(int i=0;i<2;i++) { + * origin[0] = i; + * NumericalArray na = read(v,origin,shape); + * // na now contains a 3d-array with shape 1, 3, 4 + * } + * + * + * @param v the variable to read from + * @param origin where to start reading. + * @param shape how big a chunk to read. + * @return the data read + */ + public NumericalArray read(Variable v, int[] origin, int[] shape) { + Array a; + try { + a = v.read(origin, shape); + } catch (IOException e) { + throw (new IllegalArgumentException("problem")); + } catch (InvalidRangeException e) { + throw (new IllegalArgumentException("other problem")); + } + return new NumericalArrayImpl(a.copyToNDJavaArray()); + } + + public void close() { + try { + if (this.file != null) this.file.close(); + } catch (IOException e) { + // + } + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfWriter.java b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfWriter.java new file mode 100644 index 0000000..6117c3b --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/NetcdfWriter.java @@ -0,0 +1,160 @@ +package org.fairdatapipeline.netcdf; + +import java.io.IOException; +import java.lang.ref.Cleaner; +import java.lang.ref.Cleaner.Cleanable; +import java.util.Arrays; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.fairdatapipeline.api.IllegalActionException; +import org.fairdatapipeline.objects.CoordinateVariableDefinition; +import org.fairdatapipeline.objects.NumericalArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Group; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; +import ucar.nc2.write.NetcdfFormatWriter; + +public class NetcdfWriter implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(NetcdfWriter.class); + private static final Cleaner cleaner = Cleaner.create(); + // todo: we should not create more cleaners than necessary + private final Cleanable cleanable; + private final NetcdfWriterWrapper netcdfWriterWrapper; + + public NetcdfWriter(NetcdfBuilder b, Runnable onClose) { + LOGGER.trace("NetcdfWriter.NetcdfWriter()"); + this.netcdfWriterWrapper = new NetcdfWriterWrapper(b, onClose); + this.cleanable = cleaner.register(this, this.netcdfWriterWrapper); + } + + // Defining a resource that requires cleaning + private static class NetcdfWriterWrapper implements Runnable { + private final NetcdfFormatWriter writer; + private final NetcdfFile netcdfFile; + + private final Runnable runOnClose; + + NetcdfWriterWrapper(NetcdfBuilder b, Runnable runOnClose) { + try { + this.writer = b.build(); + this.netcdfFile = this.writer.getOutputFile(); + } catch (Exception e) { + throw (new IllegalActionException("failed to build the netCDF file", e)); + } + + this.runOnClose = runOnClose; + } + + // Invoked by close method or cleaner + @Override + public void run() { + LOGGER.trace("run() invoked by cleaner"); + runOnClose.run(); + try { + this.writer.close(); + } catch (IOException e) { + LOGGER.error("can't close the netCDF writer.", e); + } + } + } + + public Variable getVariable(VariableName variableName) { + Group g = this.netcdfWriterWrapper.netcdfFile.findGroup("/" + variableName.getGroupName()); + if (g == null) + throw (new IllegalActionException( + "group /" + variableName.getGroupName() + " not found for writing.")); + Variable v = g.findVariableLocal(variableName.getName().toString()); + if (v == null) + throw (new IllegalActionException( + "variable " + + variableName.getName() + + " (in group /" + + variableName.getGroupName() + + ") not found for writing.")); + return v; + } + + public void writeArrayData(Variable v, NumericalArray nadat) + throws InvalidRangeException, IOException { + writeArrayData(v, nadat, null); + } + + public void writeArrayData(Variable v, Array data) throws InvalidRangeException, IOException { + writeArrayData(v, data, null); + } + + public void writeArrayData(Variable v, NumericalArray nadat, @Nullable int[] origin) + throws InvalidRangeException, IOException { + Array data = ucar.ma2.Array.makeFromJavaArray(nadat.asObject()); + writeArrayData(v, data, origin); + } + + /** + * write data to the file; from position 'origin' or from the start if origin is null. + * + * @param v + * @param data + * @param origin + * @throws InvalidRangeException + */ + public void writeArrayData(Variable v, Array data, @Nullable int[] origin) + throws InvalidRangeException, IOException { + // TODO: check if data fill fit? correct stride and size.. + // Array data = NetcdfDataType.translate_array(nadef.getDataType(), nadef.getDimension_sizes(), + // nadat.asOA()); + // the below works for primitive arrays.. if it might contain non primitives we need + // NetcdfDataType.translate_array instead. + if (LOGGER.isDebugEnabled()) { + String datashape = Arrays.toString(data.getShape()); + String vshape = Arrays.toString(v.getShape()); + LOGGER.trace("data.shape: {}", datashape); + LOGGER.trace("v.shape: {}", vshape); + } + if (origin == null) { + LOGGER.debug("origin == null"); + this.netcdfWriterWrapper.writer.write(v, data); + } else { + if (LOGGER.isDebugEnabled()) { + String originstring = Arrays.toString(origin); + LOGGER.debug("origin: {}", originstring); + } + this.netcdfWriterWrapper.writer.write(v, origin, data); + } + } + + public void writeDimensionVariable(CoordinateVariableDefinition coordinateVariable) { + if (coordinateVariable.getValues() == null) return; + Group g = + this.netcdfWriterWrapper.netcdfFile.findGroup( + coordinateVariable.getVariableName().getGroupName().toString()); + if (g == null) + throw (new IllegalActionException( + "can't find group " + coordinateVariable.getVariableName().getGroupName())); + Variable v = g.findVariableLocal(coordinateVariable.getVariableName().getName().toString()); + if (v == null) + throw (new IllegalActionException( + "can't find variable " + coordinateVariable.getVariableName())); + Array data = NetcdfDataType.translateArray(coordinateVariable.getValues()); + + try { + this.netcdfWriterWrapper.writer.write(v, data); + } catch (IOException e) { + throw (new IllegalActionException( + "failed to write dimension values to file for variable " + + coordinateVariable.getVariableName())); + } catch (InvalidRangeException e) { + throw (new IllegalActionException( + "invalid range to write dimension values for variable " + + coordinateVariable.getVariableName())); + } + } + + @Override + public void close() { + LOGGER.trace("close()"); + cleanable.clean(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/VariableName.java b/api/src/main/java/org/fairdatapipeline/netcdf/VariableName.java new file mode 100644 index 0000000..b9967c9 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/VariableName.java @@ -0,0 +1,70 @@ +package org.fairdatapipeline.netcdf; + +import java.util.regex.Pattern; +import javax.annotation.Nonnull; + +/** + * VariableName stores the object_component (e.g. 'human/mortality_data') as a name + * ('mortality_data') and a group ('human') + */ +public class VariableName { + @Nonnull NetcdfName name; + @Nonnull NetcdfGroupName groupName; + private static final Pattern netcdffullpath = + Pattern.compile("^\\p{Alnum}[\\p{Alnum}_-]*+(?:/\\p{Alnum}[\\p{Alnum}_-]*+)*+$"); + + /** + * create the VariableName from a full path with variable name (e.g. 'human/mortality_data') will + * be stored as a name ('mortality_data') and a group ('human') if the full path starts with a '/' + * we remove the slash. + * + * @param fullPath the full path to this variable + */ + public VariableName(@Nonnull String fullPath) { + if (fullPath.startsWith("/")) fullPath = fullPath.substring(1); + if (!netcdffullpath.matcher(fullPath).find()) + throw (new IllegalArgumentException("not a valid netCDF full path: " + fullPath)); + int last_slash = fullPath.lastIndexOf("/"); + if (last_slash == -1) { + this.name = new NetcdfName(fullPath); + this.groupName = new NetcdfGroupName(""); + } else { + this.name = new NetcdfName(fullPath.substring(last_slash + 1)); + this.groupName = new NetcdfGroupName(fullPath.substring(0, last_slash)); + } + } + + public VariableName(@Nonnull String name, @Nonnull String groupName) { + this(new NetcdfName(name), new NetcdfGroupName(groupName)); + } + + public VariableName(@Nonnull NetcdfName name, @Nonnull NetcdfGroupName groupName) { + this.name = name; + this.groupName = groupName; + } + + public @Nonnull NetcdfGroupName getGroupName() { + return groupName; + } + + public @Nonnull NetcdfName getName() { + return name; + } + + public @Nonnull String getFullPath() { + if (groupName.toString().length() == 0) return name.toString(); + return groupName.toString() + '/' + name; + } + + public @Nonnull String toString() { + return getFullPath(); + } + + @Override + public boolean equals(Object other) { + return (other != null + && other.getClass() == getClass() + && ((VariableName) other).getGroupName().equals(this.groupName) + && ((VariableName) other).getName().equals(this.name)); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/netcdf/package-info.java b/api/src/main/java/org/fairdatapipeline/netcdf/package-info.java new file mode 100644 index 0000000..a51fafb --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/netcdf/package-info.java @@ -0,0 +1,2 @@ +/** The NetCDF interface for reading and writing arrays and tables to netCDF files. */ +package org.fairdatapipeline.netcdf; diff --git a/api/src/main/java/org/fairdatapipeline/objects/CoordinateVariableDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/CoordinateVariableDefinition.java new file mode 100644 index 0000000..054e35b --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/CoordinateVariableDefinition.java @@ -0,0 +1,150 @@ +package org.fairdatapipeline.objects; + +import java.lang.reflect.Array; +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.VariableName; + +/** + * CoordinateVariableDefinition is used to create a variable that describes one of the dimensions of + * a DimensionalVariableDefinition. + * + *

A CoordinateVariable is represented in the netCDF file by a variable and a dimension with the + * same name, in the same group. + */ +public class CoordinateVariableDefinition extends VariableDefinition { + private final int size; + public static final int UNLIMITED = 0; + private final @Nullable Object values; + + @Nonnull VariableName variableName; + + /** + * if we don't supply values, we need to set size and type. size can be + * CoordinateVariable.UNLIMITED + * + * @param variableName Name and Group of this Coordinate Variable + * @param dataType dataType of the Variable + * @param size length of the Dimension. (or UNLIMITED) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + */ + public CoordinateVariableDefinition( + @Nonnull VariableName variableName, + @Nonnull NetcdfDataType dataType, + int size, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name) { + super(dataType, description, units, long_name, Collections.emptyMap()); + this.variableName = variableName; + this.size = size; + this.values = null; + } + + /** + * if we don't supply values, we need to set size and type. size can be + * CoordinateVariable.UNLIMITED + * + * @param variableName Name and Group of this Coordinate Variable + * @param dataType dataType of the Variable + * @param size length of the Dimension. (or UNLIMITED) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + * @param optional_attribs extra metadata attributes to be added to this variable. the values are + * String[] as we mostly want to add a single String but sometimes more than one String. + */ + public CoordinateVariableDefinition( + @Nonnull VariableName variableName, + @Nonnull NetcdfDataType dataType, + int size, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + super(dataType, description, units, long_name, optional_attribs); + this.variableName = variableName; + this.size = size; + this.values = null; + } + + /** + * if we give 'values', we don't need to give type and length as these can be taken from the + * 'values' array. + * + * @param variableName Name and Group of this Coordinate Variable + * @param values an Array of values for this Variable. (ie. [1, 2, 3] will make an Int variable of + * length 3.) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + */ + public CoordinateVariableDefinition( + @Nonnull VariableName variableName, + @Nonnull Object values, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name) { + super( + NetcdfDataType.translateDatatype(values), + description, + units, + long_name, + Collections.emptyMap()); + this.variableName = variableName; + this.size = Array.getLength(values); + this.values = values; + } + + /** + * if we give 'values', we don't need to give type and length as these can be taken from the + * 'values' array. + * + * @param variableName Name and Group of this Coordinate Variable + * @param values an Array of values for this Variable. (ie. [1, 2, 3] will make an Int variable of + * length 3.) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + * @param optional_attribs extra metadata attributes to be added to this variable. the values are + * String[] as we mostly want to add a single String but sometimes more than one String. + */ + public CoordinateVariableDefinition( + @Nonnull VariableName variableName, + @Nonnull Object values, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + super( + NetcdfDataType.translateDatatype(values), description, units, long_name, optional_attribs); + this.variableName = variableName; + this.size = Array.getLength(values); + this.values = values; + } + + public boolean isUnlimited() { + return this.size == UNLIMITED; + } + + public int getSize() { + return size; + } + + public @Nonnull VariableName getVariableName() { + return this.variableName; + } + + public @Nullable Object getValues() { + return values; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/Dimension.java b/api/src/main/java/org/fairdatapipeline/objects/Dimension.java new file mode 100644 index 0000000..2bb7662 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/Dimension.java @@ -0,0 +1,44 @@ +package org.fairdatapipeline.objects; + +import javax.annotation.Nullable; +import org.fairdatapipeline.netcdf.NetcdfName; + +public class Dimension { + @Nullable NetcdfName name; + @Nullable Integer size; + boolean is_size = false; + + public Dimension(NetcdfName name) { + this.name = name; + } + + public Dimension(String name) { + this.name = new NetcdfName(name); + } + + public Dimension(int size) { + this.size = size; + is_size = true; + } + + public boolean is_size() { + return is_size; + } + + public Integer size() { + return size; + } + + public NetcdfName name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof Dimension)) return false; + Dimension d = (Dimension) o; + return ((this.is_size() && d.is_size() && d.size == this.size) + || (!this.is_size() && !d.is_size() && d.name().getName().equals(this.name.getName()))); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/DimensionalVariableDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/DimensionalVariableDefinition.java new file mode 100644 index 0000000..7fc0f2f --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/DimensionalVariableDefinition.java @@ -0,0 +1,121 @@ +package org.fairdatapipeline.objects; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nonnull; +import org.fairdatapipeline.netcdf.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * DimensionVariableDefinition, aka 'Array'.. multidimensional array. + * + *

The dimensions are defined by linking to CoordinateVariables, with a link only to the 'local' + * NetcdfName part of the variable name; this means the name COULD be ambiguous, as there could be + * different variables with the same name, located in the current group and its parent groups. + */ +public class DimensionalVariableDefinition extends VariableDefinition { + private static final Logger LOGGER = LoggerFactory.getLogger(DimensionalVariableDefinition.class); + private final @Nonnull Dimension[] dimensions; + @Nonnull VariableName variableName; + + /** + * @param name Name and Group of this Variable + * @param dataType dataType of the Variable + * @param dimensions names/sizes of the dimensions. (no group names given; the dimensions used + * will be the nearest dimension with the given name) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + */ + public DimensionalVariableDefinition( + @Nonnull VariableName name, + @Nonnull NetcdfDataType dataType, + @Nonnull Dimension[] dimensions, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name) { + this(name, dataType, dimensions, description, units, long_name, Collections.emptyMap()); + } + + /** + * @param name Name and Group of this Variable + * @param dataType dataType of the Variable + * @param dimensions names of the dimensions. (no group names given; the dimensions used will be + * the nearest dimension with the given name) + * @param description a short description of the variable. (ie. "temperature at ground level") + * @param units the units used for this variable. (ie. "C" or "K" for temperature) + * @param long_name a more descriptive name than can be fitted in the VariableName (could be used + * on a plot axis) + * @param optional_attribs extra metadata attributes to be added to this variable. the values are + * String[] as we mostly want to add a single String but sometimes more than one String. + */ + public DimensionalVariableDefinition( + @Nonnull VariableName name, + @Nonnull NetcdfDataType dataType, + @Nonnull Dimension[] dimensions, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + this(name, dataType, dimensions, description, units, long_name, optional_attribs, null); + } + + /** + * To turn a localVarDef (local variable + * + * @param localVarDef + * @param groupName + * @param dimensionName + */ + public DimensionalVariableDefinition( + @Nonnull LocalVariableDefinition localVarDef, + NetcdfGroupName groupName, + String dimensionName) { + super( + localVarDef.dataType, + localVarDef.description, + localVarDef.units, + localVarDef.long_name, + localVarDef.optional_attribs, + localVarDef.missingValue); + LOGGER.trace( + "Creating DimensionalVariableDefinition({}, {}) from LocalVariableDefinition({})", + localVarDef.getLocalName(), + groupName, + localVarDef.getLocalName()); + this.variableName = new VariableName(localVarDef.getLocalName(), groupName); + this.dimensions = new Dimension[] {new Dimension(dimensionName)}; + } + + public DimensionalVariableDefinition( + @Nonnull VariableName name, + @Nonnull NetcdfDataType dataType, + @Nonnull Dimension[] dimensions, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs, + Object missingValue) { + super(dataType, description, units, long_name, optional_attribs, missingValue); + LOGGER.trace( + "Creating DimensionalVariableDefinition({},{},{},{},{},{})", + name, + dataType, + description, + units, + long_name, + missingValue); + this.variableName = name; + this.dimensions = dimensions.clone(); + } + + public @Nonnull VariableName getVariableName() { + return this.variableName; + } + + public Dimension[] getDimensions() { + return this.dimensions.clone(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/LocalVariableDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/LocalVariableDefinition.java new file mode 100644 index 0000000..5f98ad8 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/LocalVariableDefinition.java @@ -0,0 +1,72 @@ +package org.fairdatapipeline.objects; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.NetcdfName; + +/** + * LocalVariableDefinition is a variable without a group - it is used as part of a Table: the group + * is defined on the table, and the variables within the table are always sharing the tables group. + */ +public class LocalVariableDefinition extends VariableDefinition { + NetcdfName localName; + @Nullable Integer columnIndex; + + public LocalVariableDefinition( + @Nonnull NetcdfName localName, + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs, + Object missingValue, + @Nullable Integer columnIndex) { + super(dataType, description, units, long_name, optional_attribs, missingValue); + this.columnIndex = columnIndex; + this.localName = localName; + } + + public LocalVariableDefinition( + @Nonnull NetcdfName localName, + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs, + Object missingValue) { + super(dataType, description, units, long_name, optional_attribs, missingValue); + this.localName = localName; + } + + public LocalVariableDefinition( + @Nonnull NetcdfName localName, + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + super(dataType, description, units, long_name, optional_attribs); + this.localName = localName; + } + + public LocalVariableDefinition( + @Nonnull NetcdfName localName, + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name) { + super(dataType, description, units, long_name, Collections.emptyMap()); + this.localName = localName; + } + + public @Nonnull NetcdfName getLocalName() { + return localName; + } + + public @Nullable Integer getColumnIndex() { + return columnIndex; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/NetcdfDataComponentDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/NetcdfDataComponentDefinition.java new file mode 100644 index 0000000..3e244f0 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/NetcdfDataComponentDefinition.java @@ -0,0 +1,44 @@ +package org.fairdatapipeline.objects; + +import java.util.Map; +import javax.annotation.Nonnull; +import org.fairdatapipeline.netcdf.NetcdfNames; + +/** + * basic DataComponentDef - the only fields shared between all types of variables/dimensions/tables + * are the attributes: description, long_name, and optional_attribs. + */ +abstract class NetcdfDataComponentDefinition { + final @Nonnull String description; + final @Nonnull String long_name; + final @Nonnull Map optional_attribs; + + NetcdfDataComponentDefinition( + @Nonnull String description, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + this.description = description; + this.long_name = long_name; + optional_attribs.keySet().stream() + .forEach( + k -> { + if (k.startsWith(NetcdfNames.FDP_PREFIX)) + throw new IllegalArgumentException( + "attribute names are not allowed to start with prefix " + + NetcdfNames.FDP_PREFIX); + }); + this.optional_attribs = optional_attribs; + } + + public @Nonnull String getDescription() { + return description; + } + + public @Nonnull String getLong_name() { + return long_name; + } + + public @Nonnull Map getOptional_attribs() { + return optional_attribs; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/NumericalArray.java b/api/src/main/java/org/fairdatapipeline/objects/NumericalArray.java index 39e00dc..eff7f29 100644 --- a/api/src/main/java/org/fairdatapipeline/objects/NumericalArray.java +++ b/api/src/main/java/org/fairdatapipeline/objects/NumericalArray.java @@ -1,6 +1,12 @@ package org.fairdatapipeline.objects; public interface NumericalArray { + Object asObject(); + + int[] getShape(); + + public NumericalArray matchShape(int[] longershape); + Number[] as1DArray(); Number[][] as2DArray(); diff --git a/api/src/main/java/org/fairdatapipeline/objects/NumericalArrayImpl.java b/api/src/main/java/org/fairdatapipeline/objects/NumericalArrayImpl.java index 3d7977b..0e57da7 100644 --- a/api/src/main/java/org/fairdatapipeline/objects/NumericalArrayImpl.java +++ b/api/src/main/java/org/fairdatapipeline/objects/NumericalArrayImpl.java @@ -1,34 +1,155 @@ package org.fairdatapipeline.objects; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import org.fairdatapipeline.api.IllegalActionException; + public class NumericalArrayImpl implements NumericalArray { - private final Object[] nDArray; + private final Object nDArray; + private int[] shape; - public NumericalArrayImpl(Object[] nDArray) { + public NumericalArrayImpl(Object nDArray) { + if (!nDArray.getClass().isArray()) throw (new IllegalActionException("no array")); + this.shape = checkShape(nDArray, true).stream().mapToInt(Integer::intValue).toArray(); this.nDArray = nDArray; } + @Override + public int[] getShape() { + return this.shape; + } + + /** + * wrap my current array in extra dimensions until it matches the number of dimensions in shape. + * the added (top level) dimensions will have length 1. make sure that the dimensions that already + * existed have the same length as in shape. (the first (top-level) lengths in shape are ignored) + * ie. if current shape is {3, 4} and longershape is {5, 3, 4}, we will return a NumericalArray + * with shape {1, 3, 4} (the array with shape {1, 3, 4} can we written to a variable with shape + * {5, 3, 4} with offsets {0, 0, 0} up to {4, 0, 0}) + * + * @param longershape the shape we want to match. + * @return a new NumericalArray with the number of dimensions of longershape. + */ + @Override + public NumericalArray matchShape(int[] longershape) { + if (longershape.length <= this.shape.length) + throw (new IllegalArgumentException( + "matchShape(): the asked for longershape must be longer than the current shape.")); + if (!Arrays.equals( + this.shape, + 0, + this.shape.length, + longershape, + longershape.length - shape.length, + longershape.length)) + throw (new IllegalArgumentException( + "the non-padded/added right-hand-side dimensions in longershape must match current shape!")); + Object r = nDArray; + for (int i = this.shape.length; i < longershape.length; i++) { + r = new Object[] {r}; + } + return new NumericalArrayImpl(r); + } + + private ArrayList checkShape(Object nDArray, boolean deepCheck) { + ArrayList dims = new ArrayList<>(); + int l = Array.getLength(nDArray); + dims.add(l); + if (l == 0) return dims; + Object o = Array.get(nDArray, 0); + if (o.getClass().isArray()) { + ArrayList subshape = checkShape(o, deepCheck); + if (deepCheck) { + for (int i = 1; i < Array.getLength(nDArray); i++) { + if (!subshape.equals(checkShape(Array.get(nDArray, i), deepCheck))) { + throw (new IllegalActionException("not a proper array")); + } + } + } + dims.addAll(subshape); + } + return dims; + } + + private void setNumbers(Object old_with_primitives, Object new_with_numbers, int[] dimensions) { + for (int i = 0; i < dimensions[0]; i++) { + if (dimensions.length == 1) { + ((Number[]) new_with_numbers)[i] = (Number) Array.get(old_with_primitives, i); + } else { + setNumbers( + Array.get(old_with_primitives, i), + Array.get(new_with_numbers, i), + Arrays.stream(dimensions).skip(1).toArray()); + } + } + } + + private Object unPrimitive(Object o) { + if (!o.getClass().isArray()) + throw (new IllegalArgumentException("unPrimitive must work on an array")); + // int[] dimensions = checkShape(o, false).stream().flatMapToInt(IntStream::of).toArray(); + Object new_array = Array.newInstance(Number.class, this.shape); + setNumbers(o, new_array, this.shape); + return new_array; + } + + @Override + public Object asObject() { + return nDArray; + } + @Override public Number[] as1DArray() { - return (Number[]) nDArray; + if (Number.class.isAssignableFrom(nDArray.getClass().getComponentType())) { + return (Number[]) nDArray; + } + return (Number[]) unPrimitive(nDArray); } @Override public Number[][] as2DArray() { - return (Number[][]) nDArray; + if (Number.class.isAssignableFrom(nDArray.getClass().getComponentType().getComponentType())) { + return (Number[][]) nDArray; + } + return (Number[][]) unPrimitive(nDArray); } @Override public Number[][][] as3DArray() { - return (Number[][][]) nDArray; + if (Number.class.isAssignableFrom( + nDArray.getClass().getComponentType().getComponentType().getComponentType())) { + return (Number[][][]) nDArray; + } + return (Number[][][]) unPrimitive(nDArray); } @Override public Number[][][][] as4DArray() { - return (Number[][][][]) nDArray; + if (Number.class.isAssignableFrom( + nDArray + .getClass() + .getComponentType() + .getComponentType() + .getComponentType() + .getComponentType())) { + return (Number[][][][]) nDArray; + } + return (Number[][][][]) unPrimitive(nDArray); } @Override public Number[][][][][] as5DArray() { - return (Number[][][][][]) nDArray; + if (Number.class.isAssignableFrom( + nDArray + .getClass() + .getComponentType() + .getComponentType() + .getComponentType() + .getComponentType() + .getComponentType())) { + return (Number[][][][][]) nDArray; + } + return (Number[][][][][]) unPrimitive(nDArray); } } diff --git a/api/src/main/java/org/fairdatapipeline/objects/StandardArrayDataReader.java b/api/src/main/java/org/fairdatapipeline/objects/StandardArrayDataReader.java deleted file mode 100644 index 8e2db20..0000000 --- a/api/src/main/java/org/fairdatapipeline/objects/StandardArrayDataReader.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.fairdatapipeline.objects; - -import org.fairdatapipeline.file.CleanableFileChannel; - -public interface StandardArrayDataReader { - NumericalArray read(CleanableFileChannel fileChannel, String component); -} diff --git a/api/src/main/java/org/fairdatapipeline/objects/StandardTable.java b/api/src/main/java/org/fairdatapipeline/objects/StandardTable.java deleted file mode 100644 index 16c3cd9..0000000 --- a/api/src/main/java/org/fairdatapipeline/objects/StandardTable.java +++ /dev/null @@ -1,3 +0,0 @@ -package org.fairdatapipeline.objects; - -public interface StandardTable {} diff --git a/api/src/main/java/org/fairdatapipeline/objects/StandardTableDataReader.java b/api/src/main/java/org/fairdatapipeline/objects/StandardTableDataReader.java deleted file mode 100644 index c174eac..0000000 --- a/api/src/main/java/org/fairdatapipeline/objects/StandardTableDataReader.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.fairdatapipeline.objects; - -import org.fairdatapipeline.file.CleanableFileChannel; - -public interface StandardTableDataReader { - StandardTable readTable(CleanableFileChannel fileChannel, String component); -} diff --git a/api/src/main/java/org/fairdatapipeline/objects/TableDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/TableDefinition.java new file mode 100644 index 0000000..5309618 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/TableDefinition.java @@ -0,0 +1,56 @@ +package org.fairdatapipeline.objects; + +import java.util.Map; +import javax.annotation.Nonnull; +import org.fairdatapipeline.netcdf.NetcdfGroupName; +import org.fairdatapipeline.netcdf.VariableName; + +/** + * table is a bit of an odd one: it behaves like any other data component: it has a name + * (groupname), description and other attributes. but instead of directly containing a variable + * (array with dimensions) or a 'CoordinateVariable', it contains an array of + * LocalVariableDefinitions. These LocalVariableDefinitions must all only have local names (no + * group). they will share a single dimension for its length. + */ +public class TableDefinition extends NetcdfDataComponentDefinition { + @Nonnull LocalVariableDefinition[] columns; + @Nonnull NetcdfGroupName groupName; + + int size; + + public TableDefinition( + @Nonnull NetcdfGroupName groupName, + int size, + @Nonnull String description, + @Nonnull String long_name, + @Nonnull Map optional_attribs, + @Nonnull LocalVariableDefinition[] columns) { + super(description, long_name, optional_attribs); + this.groupName = groupName; + this.columns = columns; + this.size = size; + } + + public @Nonnull NetcdfGroupName getGroupName() { + return groupName; + } + + public boolean isUnlimited() { + return this.size == CoordinateVariableDefinition.UNLIMITED; + } + + public VariableName getVariableName(int i) { + if (i >= this.columns.length) + throw (new IllegalArgumentException( + "this table doesn't have that many columns " + i + " >= " + this.columns.length)); + return new VariableName(this.columns[i].getLocalName(), this.getGroupName()); + } + + public LocalVariableDefinition[] getColumns() { + return this.columns; // TODO make a copy + } + + public int getSize() { + return this.size; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/VariableDefinition.java b/api/src/main/java/org/fairdatapipeline/objects/VariableDefinition.java new file mode 100644 index 0000000..63b6757 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/objects/VariableDefinition.java @@ -0,0 +1,54 @@ +package org.fairdatapipeline.objects; + +import java.util.Map; +import javax.annotation.Nonnull; +import org.fairdatapipeline.netcdf.NetcdfDataType; + +/** + * VariableDefinition is the parent class of CoordinateVariableDefinition and + * DimensionalVariableDefinition. It has the full definition of a variable, only lacking + * 'dimensions' or 'size'. It is used in TableDefinition to define the columns of the table. + */ +public abstract class VariableDefinition extends NetcdfDataComponentDefinition { + final @Nonnull NetcdfDataType dataType; + final @Nonnull String units; + + final Object missingValue; + + VariableDefinition( + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs) { + super(description, long_name, optional_attribs); + this.dataType = dataType; + this.units = units; + this.missingValue = null; + } + + VariableDefinition( + @Nonnull NetcdfDataType dataType, + @Nonnull String description, + @Nonnull String units, + @Nonnull String long_name, + @Nonnull Map optional_attribs, + Object missingValue) { + super(description, long_name, optional_attribs); + this.dataType = dataType; + this.units = units; + this.missingValue = missingValue; + } + + public @Nonnull NetcdfDataType getDataType() { + return dataType; + } + + public Object getMissingValue() { + return missingValue; + } + + public @Nonnull String getUnits() { + return units; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/objects/package-info.java b/api/src/main/java/org/fairdatapipeline/objects/package-info.java index 08718ac..64b8bcc 100644 --- a/api/src/main/java/org/fairdatapipeline/objects/package-info.java +++ b/api/src/main/java/org/fairdatapipeline/objects/package-info.java @@ -1,2 +1,9 @@ -/** Arrays and tables for H5 files; not quite implemented yet. */ +/** + * Arrays, tables, variables, dimensions for netcdf files. + * + *

*Definition classes are to define the structure and metadata of netcdf data. + * + *

The only other thing here is the NumericalArray which is there to contain multidimensional + * numerical data that can be written to a Dimensional Variable. + */ package org.fairdatapipeline.objects; diff --git a/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java b/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java index 6c9bc30..84a4761 100644 --- a/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java +++ b/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; +import java.io.EOFException; import java.io.IOException; import java.io.PrintWriter; import java.net.URISyntaxException; @@ -26,16 +27,22 @@ import org.fairdatapipeline.distribution.ImmutableMinMax; import org.fairdatapipeline.distribution.MinMax; import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.netcdf.*; +import org.fairdatapipeline.objects.*; import org.fairdatapipeline.samples.ImmutableSamples; import org.fairdatapipeline.samples.Samples; import org.javatuples.Triplet; import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @EnabledIfEnvironmentVariable(named = "LOCALREG", matches = "FRESHASADAISY") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class CoderunIntegrationTest { + private static final Logger LOGGER = LoggerFactory.getLogger(CoderunIntegrationTest.class); + private final Table mockTable = ImmutableTable.builder() .put(0, "colA", 5) @@ -164,12 +171,18 @@ public void setUp() throws Exception { // rng = new RandomDataGenerator().getRandomGenerator(); setup_paths(); setup_data(); + create_namespace(this.ns); + create_namespace(this.altNamespace); this.CSV_hash = "eb7e7a49816c8a6a784260e4596e88bf7a96a6a5"; if (System.getProperty("os.name").contains("Windows")) this.CSV_hash = "d1713dcc0c6b28337d14f3693882aebca3e96f17"; cleanup_datastore(); } + void create_namespace(String ns) { + restClient.post(new RegistryNamespace(ns)); + } + void create_author() { RegistryAuthor author = (RegistryAuthor) restClient.getFirst(RegistryAuthor.class, Collections.emptyMap()); @@ -190,11 +203,12 @@ void create_author() { @AfterAll public void final_cleanup() throws IOException { - delete_directories(); + // delete_directories(); } public void cleanup_datastore() throws IOException { - delete_directories(); + // delete_directories(); + FileUtils.deleteDirectory(coderunTSPath.toFile()); Files.createDirectories(coderunTSPath); Files.copy(ori_configPath, configPath); Files.copy(ori_scriptPath, scriptPath); @@ -248,6 +262,10 @@ void check_issue(String issue, String... components) { }); } + /** + * @param inputs + * @param outputs + */ void check_last_coderun( List> inputs, List> outputs) { // quite a random set of checks to see if a coderun did what I expect it to do. @@ -259,6 +277,8 @@ void check_last_coderun( return; } String last_coderun = coderuns.get(coderuns.size() - 1); + LOGGER.trace("coderun: {}", last_coderun); + RegistryCode_run cr = (RegistryCode_run) restClient.getFirst( @@ -266,12 +286,15 @@ void check_last_coderun( RegistryObject script = (RegistryObject) restClient.get(RegistryObject.class, cr.getSubmission_script()); assertNotNull(script); + LOGGER.trace("script: {}", script.getDescription()); RegistryObject config = (RegistryObject) restClient.get(RegistryObject.class, cr.getModel_config()); assertNotNull(config); + LOGGER.trace("config: {}", config.getData_products()); RegistryObject code_repo = (RegistryObject) restClient.get(RegistryObject.class, cr.getCode_repo()); assertNotNull(code_repo); + LOGGER.trace("code_repo: {}", code_repo.getDescription()); assertThat(code_repo.getAuthors()) .containsExactly(restClient.makeAPIURL(RegistryAuthor.class, 1)); @@ -301,6 +324,8 @@ void check_last_coderun( if (outputs == null) assertThat(cr.getOutputs()).isEmpty(); else { + LOGGER.trace("outputs: {}", cr.getOutputs().size()); + cr.getOutputs().stream().forEach(c -> LOGGER.trace(c.toString())); assertThat(cr.getOutputs()).hasSameSizeAs(outputs); cr.getOutputs() .forEach( @@ -329,8 +354,8 @@ void testWriteEstimate() { String dataProduct = "human/population"; String component = "estimate-component"; try (Coderun coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeEstimate(estimate); } String hash = "a4f9d47dac45639e69a758a8b2d49bf11bbeb262"; @@ -343,8 +368,8 @@ void testReadEstimate() { String dataProduct = "human/population"; String component = "estimate-component"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dp.getComponent(component); + Data_product_read_toml dp = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dp.getComponent(component); assertThat(oc.readEstimate()).isEqualTo(estimate); } String hash = "a4f9d47dac45639e69a758a8b2d49bf11bbeb262"; @@ -357,8 +382,8 @@ void testWriteDistribution() { String dataProduct = "human/distribution"; String component = "distribution-component"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeDistribution(distribution); } String hash = "8e767aea46ac67c4546dacc23d302729c461fddd"; @@ -371,8 +396,8 @@ void testReadDistribution() { String dataProduct = "human/distribution"; String component = "distribution-component"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dc.getComponent(component); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dc.getComponent(component); assertThat(oc.readDistribution()).isEqualTo(distribution); } String hash = "8e767aea46ac67c4546dacc23d302729c461fddd"; @@ -385,8 +410,8 @@ void testWriteCategoricalDistribution() { String dataProduct = "human/cdistribution"; String component = "cdistribution-component"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeDistribution(categoricalDistribution); } String hash = "a30b735ba6ce0340dbf264518d8a3ca1918397b8"; @@ -399,8 +424,8 @@ void testReadCategoricalDistribution() { String dataProduct = "human/cdistribution"; String component = "cdistribution-component"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dc.getComponent(component); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dc.getComponent(component); assertThat(oc.readDistribution()).isEqualTo(categoricalDistribution); } String hash = "a30b735ba6ce0340dbf264518d8a3ca1918397b8"; @@ -413,8 +438,8 @@ void testWriteSamples() { String dataProduct = "human/samples"; String component = "example-samples-w"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeSamples(samples); } String hash = "83d034652197abd2f456f286c9ee7ac04500309b"; @@ -427,8 +452,8 @@ void testReadSamples() { String dataProduct = "human/samples"; String component = "example-samples-w"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dc.getComponent(component); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dc.getComponent(component); assertThat(oc.readSamples()).containsExactly(1, 2, 3); } String hash = "83d034652197abd2f456f286c9ee7ac04500309b"; @@ -442,12 +467,12 @@ void testWriteSamplesMultipleComponents() { String component1 = "example-samples-w1"; String component2 = "example-samples-w2"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); - Object_component_write oc1 = dp.getComponent(component1); + Object_component_write_filechannel_toml oc1 = dp.getComponent(component1); oc1.writeSamples(samples); - Object_component_write oc2 = dp.getComponent(component2); + Object_component_write_filechannel_toml oc2 = dp.getComponent(component2); oc2.writeSamples(samples); } String hash = "d9501d26df34a851591b1ef718564ab9f8f44c5d"; @@ -465,9 +490,9 @@ void testReadSamplesMultipleComponents() { String component1 = "example-samples-w1"; String component2 = "example-samples-w2"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc1 = dc.getComponent(component1); - Object_component_read oc2 = dc.getComponent(component2); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc1 = dc.getComponent(component1); + Object_component_read_filechannel_toml oc2 = dc.getComponent(component2); assertThat(oc1.readSamples()).containsExactly(1, 2, 3); assertThat(oc2.readSamples()).containsExactly(1, 2, 3); } @@ -485,8 +510,8 @@ void testWriteGlobDP() { String dataProduct = "animal/dog"; String component = "example-samples"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeSamples(samples2); } String hash = "a12e5337b4c6dd4c8ff119da2ae47996561b5a32"; @@ -499,8 +524,8 @@ void testReadNoGlob() { String dataProduct = "animal/dog"; String component = "example-samples"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dc.getComponent(component); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dc.getComponent(component); assertThat(oc.readSamples()).containsExactly(4, 5, 6); } String hash = "a12e5337b4c6dd4c8ff119da2ae47996561b5a32"; @@ -514,9 +539,9 @@ void testWriteGlobMultiDP() { String dataProduct2 = "animal/mouse"; String component = "example-samples"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp1 = coderun.get_dp_for_write(dataProduct1, "toml"); - Data_product_write dp2 = coderun.get_dp_for_write(dataProduct2, "toml"); - Object_component_write oc = dp1.getComponent(component); + Data_product_write_toml dp1 = coderun.get_dp_for_write_toml(dataProduct1); + Data_product_write_toml dp2 = coderun.get_dp_for_write_toml(dataProduct2); + Object_component_write_filechannel_toml oc = dp1.getComponent(component); oc.writeSamples(samples3); oc = dp2.getComponent(component); oc.writeSamples(samples4); @@ -577,14 +602,14 @@ void testWriteSamplesMultipleComponentsAndIssues() { String issue2 = "very bad component"; String issue3 = "this one is not so bad"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); - Object_component_write oc1 = dp.getComponent(component1); + Object_component_write_filechannel_toml oc1 = dp.getComponent(component1); oc1.raise_issue(issue1, 10); oc1.writeSamples(samples); oc1.raise_issue(issue2, 1); - Object_component_write oc2 = dp.getComponent(component2); + Object_component_write_filechannel_toml oc2 = dp.getComponent(component2); oc2.writeSamples(samples); oc2.raise_issue(issue3, 1); } @@ -608,10 +633,10 @@ void testReadSamplesMultipleComponentsAndIssues() { String issue1 = "upon re-reading this component we found even more problems"; String issue2 = "upon re-reading this component we found it's actually OK"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc1 = dc.getComponent(component1); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc1 = dc.getComponent(component1); oc1.raise_issue(issue1, 10); - Object_component_read oc2 = dc.getComponent(component2); + Object_component_read_filechannel_toml oc2 = dc.getComponent(component2); oc1.raise_issue(issue2, 0); assertThat(oc1.readSamples()).containsExactly(1, 2, 3); assertThat(oc2.readSamples()).containsExactly(1, 2, 3); @@ -635,11 +660,11 @@ void testRead_oneIssueToMultipleComp_and_script() { String issue = "one issue for 2 comps"; String issue2 = "this issue attached itself to the comps"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dc = coderun.get_dp_for_read(dataProduct); - Object_component_read oc1 = dc.getComponent(component1); + Data_product_read_toml dc = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc1 = dc.getComponent(component1); Issue i = coderun.raise_issue(issue, 2); i.add_components(oc1); - Object_component_read oc2 = dc.getComponent(component2); + Object_component_read_filechannel_toml oc2 = dc.getComponent(component2); i.add_components(oc2); assertThat(oc1.readSamples()).containsExactly(1, 2, 3); assertThat(oc2.readSamples()).containsExactly(1, 2, 3); @@ -662,8 +687,8 @@ void testRead_oneIssueToMultipleComp_and_script() { void testCSV_writeLink() throws IOException { String dataProduct = "animal/ant"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "csv"); - Object_component_write oc = dp.getComponent(); + Data_product_write_link dp = coderun.get_dp_for_write_link(dataProduct, "csv"); + Object_component_write_filechannel_link oc = dp.getComponent(); Path p = oc.writeLink(); assertNotNull(p); try (PrintWriter pw = new PrintWriter(p.toFile())) { @@ -692,8 +717,8 @@ private List getRecordFromLine(String line) { void testCSV_readLink() throws IOException { String dataProduct = "animal/ant"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dp.getComponent(); + Data_product_read_link dp = coderun.get_dp_for_read_link(dataProduct); + Object_component_read_filechannel_link oc = dp.getComponent(); Path p = oc.readLink(); int i = 0; try (Scanner scanner = new Scanner(p.toFile())) { @@ -712,8 +737,8 @@ void testCSV_writeLink_withIssue() throws IOException { String dataProduct = "animal/monkey"; String issue = "this does not seem to contain anything monkey-related"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "csv"); - Object_component_write oc = dp.getComponent(); + Data_product_write_link dp = coderun.get_dp_for_write_link(dataProduct, "csv"); + Object_component_write_filechannel_link oc = dp.getComponent(); Path p = oc.writeLink(); oc.raise_issue(issue, 9); assertNotNull(p); @@ -731,8 +756,8 @@ void testCSV_readLink_withIssue() throws IOException { String dataProduct = "animal/ant"; String issue = "not enough orange"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dp.getComponent(); + Data_product_read_link dp = coderun.get_dp_for_read_link(dataProduct); + Object_component_read_filechannel_link oc = dp.getComponent(); oc.raise_issue(issue, 10); Path p = oc.readLink(); int i = 0; @@ -753,8 +778,8 @@ void testRewriteDPname() { String dataProduct = "animal/canine"; String component = "NumberOfLegs"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeSamples(samples); } String rewrittenDP = "animal/doggy"; @@ -769,8 +794,8 @@ void testReadRewrittenDPname() { String dataProduct = "animal/canine"; String component = "NumberOfLegs"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dp.getComponent(component); + Data_product_read_toml dp = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dp.getComponent(component); oc.readSamples(); } String rewrittenDP = "animal/doggy"; @@ -784,8 +809,8 @@ void testAltNS() { String dataProduct = "test/altns"; String component = "altNScompo"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); - Object_component_write oc = dp.getComponent(component); + Data_product_write_toml dp = coderun.get_dp_for_write_toml(dataProduct); + Object_component_write_filechannel_toml oc = dp.getComponent(component); oc.writeSamples(samples); } String hash = "b8e68425f66bfb033dbbfeac3b48b922a2121ca0"; @@ -799,8 +824,8 @@ void testAltNSread() { String dataProduct = "test/altns"; String component = "altNScompo"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); - Object_component_read oc = dp.getComponent(component); + Data_product_read_toml dp = coderun.get_dp_for_read_toml(dataProduct); + Object_component_read_filechannel_toml oc = dp.getComponent(component); oc.readSamples(); } String hash = "b8e68425f66bfb033dbbfeac3b48b922a2121ca0"; @@ -812,8 +837,8 @@ void testAltNSread() { void testConfigFiletype() { String dataProduct = "animal/chicken"; try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_write dp = coderun.get_dp_for_write(dataProduct); - Object_component_write oc = dp.getComponent(); + Data_product_write_link dp = coderun.get_dp_for_write_link(dataProduct); + Object_component_write_filechannel_link oc = dp.getComponent(); try (CleanableFileChannel f = oc.getFileChannel()) { ByteBuffer bb = ByteBuffer.wrap(chickenTestText.getBytes(StandardCharsets.UTF_8)); f.write(bb); @@ -825,12 +850,12 @@ void testConfigFiletype() { check_last_coderun(null, List.of(new Triplet<>(dataProduct, "", hash))); try (var coderun = new Coderun(configPath, scriptPath, token)) { - Data_product_read dp = coderun.get_dp_for_read(dataProduct); + Data_product_read_link dp = coderun.get_dp_for_read_link(dataProduct); RegistryFile_type ft = (RegistryFile_type) restClient.get(RegistryFile_type.class, dp.registryObject.getFile_type()); assertThat(ft.getExtension()).isEqualTo("txt"); - Object_component_read oc = dp.getComponent(); + Object_component_read_filechannel_link oc = dp.getComponent(); try (CleanableFileChannel f = oc.getFileChannel()) { ByteBuffer bb = ByteBuffer.allocate(chickenTestText.length()); f.read(bb); @@ -852,4 +877,1376 @@ void emptyCoderun() { } check_last_coderun(null, null); } + + @Test + @Order(27) + void testWriteArray() throws IOException { + String dataProduct = "test/array1"; + String component_path = "component1/with/a/path"; + VariableName latname = new VariableName("lat", component_path); + VariableName lonname = new VariableName("lon", component_path); + VariableName nadefname = new VariableName("array1", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + CoordinateVariableDefinition latdim = + new CoordinateVariableDefinition( + latname, + new double[] {-75, -60, -45, -30, -15, 0, 15, 30, 45, 60, 75}, + "", + "degrees north", + "latitude"); + + CoordinateVariableDefinition londim = + new CoordinateVariableDefinition( + lonname, + new double[] {-180, -150, -120, -90, -60, -30, 0, 30, 60, 90, 120, 150}, + "", + "degrees east", + "longitude"); + Dimension[] dims = + new Dimension[] {new Dimension(latname.getName()), new Dimension(lonname.getName())}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + nadefname, + NetcdfDataType.DOUBLE, + dims, + "a test dataset of temperatures in space", + "C", + "surface temperature"); + Object_component_write_dimension oc_lon = dp.getComponent(londim); + Object_component_write_dimension oc_lat = dp.getComponent(latdim); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][] temperatures = new double[11][12]; + for (int lati = 0; lati < 11; lati++) + for (int loni = 0; loni < 12; loni++) + temperatures[lati][loni] = lati + (double) loni / 12.0; + NumericalArray nadat = new NumericalArrayImpl(temperatures); + try { + oc1.writeArrayData(nadat); + } catch (EOFException e) { + // + } + } + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + + check_last_coderun( + null, + Arrays.asList( + new Triplet<>(dataProduct, lonname.getFullPath(), hash), + new Triplet<>(dataProduct, latname.getFullPath(), hash), + new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + @Test + @Order(28) + void testReadArray() throws IOException { + String dataProduct = "test/array1"; + String component_name = "component1/with/a/path/array1"; + String oc_lon_name = "component1/with/a/path/lon"; + String oc_lat_name = "component1/with/a/path/lat"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc_lon = dc.getComponent(oc_lon_name); + VariableDefinition oc_lonVardef = oc_lon.getVardef(); + assertThat(oc_lonVardef.getClass()).isEqualTo(CoordinateVariableDefinition.class); + assertThat(oc_lonVardef.getDescription()).isEmpty(); + assertThat(oc_lonVardef.getUnits()).isEqualTo("degrees east"); + + Number[] lons = oc_lon.readArray().as1DArray(); + assertThat(lons) + .containsExactly( + -180.0, -150.0, -120.0, -90.0, -60.0, -30.0, 0.0, 30.0, 60.0, 90.0, 120.0, 150.0); + Object_component_read_nc oc_lat = dc.getComponent(oc_lat_name); + VariableDefinition oc_latVardef = oc_lat.getVardef(); + assertThat(oc_latVardef.getDescription()).isEmpty(); + assertThat(oc_latVardef.getUnits()).isEqualTo("degrees north"); + + Number[] lats = oc_lat.readArray().as1DArray(); + assertThat(lats) + .containsExactly(-75.0, -60.0, -45.0, -30.0, -15.0, 0.0, 15.0, 30.0, 45.0, 60.0, 75.0); + Object_component_read_nc oc1 = dc.getComponent(component_name); + VariableDefinition oc_vardef = oc1.getVardef(); + assertThat(oc_vardef.getClass()).isEqualTo(DimensionalVariableDefinition.class); + assertThat(oc_vardef.getUnits()).isEqualTo("C"); + assertThat(oc_vardef.getDescription()).isEqualTo("a test dataset of temperatures in space"); + assertThat(oc_vardef.getLong_name()).isEqualTo("surface temperature"); + DimensionalVariableDefinition oc_dimvarDef = (DimensionalVariableDefinition) oc_vardef; + Dimension[] dims = oc_dimvarDef.getDimensions(); + assertThat(dims).hasSize(2); + assertThat(dims[0].name().getName()).isEqualTo("lat"); + Number[][] temps = oc1.readArray().as2DArray(); + Number[][] expected_temps = new Number[11][12]; + for (int lati = 0; lati < 11; lati++) + for (int loni = 0; loni < 12; loni++) + expected_temps[lati][loni] = lati + (double) loni / 12.0; + assertThat(temps).isDeepEqualTo(expected_temps); + } + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + check_last_coderun( + Arrays.asList( + new Triplet<>(dataProduct, component_name, hash), + new Triplet<>(dataProduct, oc_lon_name, hash), + new Triplet<>(dataProduct, oc_lat_name, hash)), + null); + } + + @Test + @Order(29) + void testReadArray_byrow() throws IOException { + String dataProduct = "test/array1"; + String component_name = "component1/with/a/path/array1"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc1 = dc.getComponent(component_name); + Number[][] expected_temps = new Number[1][12]; + int[] shape = new int[] {1, 12}; + for (int lati = 0; lati < 11; lati++) { + for (int loni = 0; loni < 12; loni++) { + expected_temps[0][loni] = lati + (double) loni / 12.0; + } + NumericalArray na = oc1.readArray(shape); + + assertThat(na.as2DArray()).isDeepEqualTo(expected_temps); + System.out.println(Arrays.toString(na.as2DArray()[0])); + } + } + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_name, hash)), null); + } + + @Test + @Order(30) + void testReadArray_byhalfrow() throws IOException { + String dataProduct = "test/array1"; + String component_name = "component1/with/a/path/array1"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc1 = dc.getComponent(component_name); + Number[][] expected_temps1 = new Number[1][6]; + Number[][] expected_temps2 = new Number[1][6]; + int[] shape = new int[] {1, 6}; + for (int lati = 0; lati < 11; lati++) { + for (int loni = 0; loni < 6; loni++) { + expected_temps1[0][loni] = lati + (double) loni / 12.0; + expected_temps2[0][loni] = lati + (double) (6 + loni) / 12.0; + } + NumericalArray na = oc1.readArray(shape); + System.out.println(Arrays.toString(na.as2DArray()[0])); + assertThat(na.as2DArray()).isDeepEqualTo(expected_temps1); + na = oc1.readArray(shape); + System.out.println(Arrays.toString(na.as2DArray()[0])); + assertThat(na.as2DArray()).isDeepEqualTo(expected_temps2); + } + } + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_name, hash)), null); + } + + @Test + @Order(31) + void testReadArray_by_single_number() throws IOException { + String dataProduct = "test/array1"; + String component_name = "component1/with/a/path/array1"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc1 = dc.getComponent(component_name); + int[] shape = new int[] {1, 1}; + double expected_temp; + for (int lati = 0; lati < 11; lati++) { + for (int loni = 0; loni < 12; loni++) { + expected_temp = lati + (double) loni / 12.0; + NumericalArray na = oc1.readArray(shape); + assertThat(na.as2DArray()[0][0]).isEqualTo(expected_temp); + } + } + } + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_name, hash)), null); + } + + @Test + @Order(32) + void testReadArray_bydoublerow() throws IOException { + String dataProduct = "test/array1"; + String component_name = "component1/with/a/path/array1"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc1 = dc.getComponent(component_name); + Number[][] expected_temps = new Number[2][12]; + int[] shape = new int[] {2, 12}; + for (int lati = 0; lati < 10; lati += 2) { + for (int loni = 0; loni < 12; loni++) { + expected_temps[0][loni] = lati + (double) loni / 12.0; + expected_temps[1][loni] = lati + 1 + (double) loni / 12.0; + } + NumericalArray na = oc1.readArray(shape); + + assertThat(na.as2DArray()).isDeepEqualTo(expected_temps); + } + // trying to read the 11th row fails because there are not 2 rows to read; + // we can only read in double rows if there is an even number of rows. + assertThrows( + IllegalArgumentException.class, + () -> { + oc1.readArray(shape); + }); + } + + String hash = "5f09c2c852d631c9d5966f65e110263e7a5fcf44"; + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_name, hash)), null); + } + + @Test + @Order(33) + void testWrite3dArray() throws IOException { + String dataProduct = "test/array3d"; + String component_path = ""; + VariableName xname = new VariableName("x", component_path); + VariableName yname = new VariableName("y", component_path); + VariableName zname = new VariableName("z", component_path); + VariableName nadefname = new VariableName("array3d", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + CoordinateVariableDefinition xdim = + new CoordinateVariableDefinition(xname, new double[] {1, 2, 3, 4}, "", "", ""); + + CoordinateVariableDefinition ydim = + new CoordinateVariableDefinition(yname, new double[] {1, 2, 3}, "", "", ""); + + CoordinateVariableDefinition zdim = + new CoordinateVariableDefinition(zname, new double[] {1, 2, 3, 4, 5}, "", "", ""); + + Dimension[] dims = + new Dimension[] { + new Dimension(xname.getName()), + new Dimension(yname.getName()), + new Dimension(zname.getName()) + }; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + nadefname, NetcdfDataType.DOUBLE, dims, "a little 3d array", "", ""); + Object_component_write_dimension oc_x = dp.getComponent(xdim); + Object_component_write_dimension oc_y = dp.getComponent(ydim); + Object_component_write_dimension oc_z = dp.getComponent(zdim); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][][] values = new double[4][3][5]; + for (int x = 0; x < 4; x++) + for (int y = 0; y < 3; y++) + for (int z = 0; z < 5; z++) values[x][y][z] = x * 10.0 + (double) y + z / 12.0; + + NumericalArray nadat = new NumericalArrayImpl(values); + try { + oc1.writeArrayData(nadat); + } catch (EOFException e) { + // + } + } + String hash = "1dbe7c11162abc87cc9ba4ae1ceb7fa0843a3011"; + + check_last_coderun( + null, + Arrays.asList( + new Triplet<>(dataProduct, xname.getFullPath(), hash), + new Triplet<>(dataProduct, yname.getFullPath(), hash), + new Triplet<>(dataProduct, zname.getFullPath(), hash), + new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + @Test + @Order(34) + void testRead3dArray() throws IOException { + String dataProduct = "test/array3d"; + String oc_x_name = "x"; + String oc_y_name = "y"; + String oc_z_name = "z"; + String oc_values_name = "array3d"; + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc_x = dc.getComponent(oc_x_name); + int[] x_vals = + Arrays.stream(oc_x.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(x_vals).containsExactly(1, 2, 3, 4); + Object_component_read_nc oc_y = dc.getComponent(oc_y_name); + int[] y_vals = + Arrays.stream(oc_y.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(y_vals).containsExactly(1, 2, 3); + Object_component_read_nc oc_z = dc.getComponent(oc_z_name); + int[] z_vals = + Arrays.stream(oc_z.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(z_vals).containsExactly(1, 2, 3, 4, 5); + Object_component_read_nc oc_values = dc.getComponent(oc_values_name); + + Number[][][] values = oc_values.readArray().as3DArray(); + Number[][][] expected_values = new Number[4][3][5]; + for (int x = 0; x < 4; x++) + for (int y = 0; y < 3; y++) + for (int z = 0; z < 5; z++) expected_values[x][y][z] = x * 10.0 + (double) y + z / 12.0; + assertThat(values).isDeepEqualTo(expected_values); + } + String hash = "1dbe7c11162abc87cc9ba4ae1ceb7fa0843a3011"; + check_last_coderun( + Arrays.asList( + new Triplet<>(dataProduct, oc_values_name, hash), + new Triplet<>(dataProduct, oc_x_name, hash), + new Triplet<>(dataProduct, oc_y_name, hash), + new Triplet<>(dataProduct, oc_z_name, hash)), + null); + } + + @Test + @Order(34) + void testRead3dArray_in_2d_slices() throws IOException { + String dataProduct = "test/array3d"; + String oc_x_name = "x"; + String oc_y_name = "y"; + String oc_z_name = "z"; + String oc_values_name = "array3d"; + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc_x = dc.getComponent(oc_x_name); + int[] x_vals = + Arrays.stream(oc_x.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(x_vals).containsExactly(1, 2, 3, 4); + Object_component_read_nc oc_y = dc.getComponent(oc_y_name); + int[] y_vals = + Arrays.stream(oc_y.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(y_vals).containsExactly(1, 2, 3); + Object_component_read_nc oc_z = dc.getComponent(oc_z_name); + int[] z_vals = + Arrays.stream(oc_z.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(z_vals).containsExactly(1, 2, 3, 4, 5); + Object_component_read_nc oc_values = dc.getComponent(oc_values_name); + + Number[][][] expected_values = new Number[1][3][5]; + int[] shape = new int[] {1, 3, 5}; + for (int x = 0; x < 4; x++) { + for (int y = 0; y < 3; y++) + for (int z = 0; z < 5; z++) expected_values[0][y][z] = x * 10.0 + (double) y + z / 12.0; + Number[][][] values = oc_values.readArray(shape).as3DArray(); + assertThat(values).isDeepEqualTo(expected_values); + } + } + String hash = "1dbe7c11162abc87cc9ba4ae1ceb7fa0843a3011"; + check_last_coderun( + Arrays.asList( + new Triplet<>(dataProduct, oc_values_name, hash), + new Triplet<>(dataProduct, oc_x_name, hash), + new Triplet<>(dataProduct, oc_y_name, hash), + new Triplet<>(dataProduct, oc_z_name, hash)), + null); + } + + @Test + @Order(34) + void testRead3dArray_in_1d_slices() throws IOException { + String dataProduct = "test/array3d"; + String oc_x_name = "x"; + String oc_y_name = "y"; + String oc_z_name = "z"; + String oc_values_name = "array3d"; + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dc = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_nc oc_x = dc.getComponent(oc_x_name); + int[] x_vals = + Arrays.stream(oc_x.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(x_vals).containsExactly(1, 2, 3, 4); + Object_component_read_nc oc_y = dc.getComponent(oc_y_name); + int[] y_vals = + Arrays.stream(oc_y.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(y_vals).containsExactly(1, 2, 3); + Object_component_read_nc oc_z = dc.getComponent(oc_z_name); + int[] z_vals = + Arrays.stream(oc_z.readArray().as1DArray()).mapToInt(Number::intValue).toArray(); + assertThat(z_vals).containsExactly(1, 2, 3, 4, 5); + Object_component_read_nc oc_values = dc.getComponent(oc_values_name); + + Number[][][] expected_values = new Number[1][1][5]; + int[] shape = new int[] {1, 1, 5}; + for (int x = 0; x < 4; x++) + for (int y = 0; y < 3; y++) { + for (int z = 0; z < 5; z++) expected_values[0][0][z] = x * 10.0 + (double) y + z / 12.0; + Number[][][] values = oc_values.readArray(shape).as3DArray(); + assertThat(values).isDeepEqualTo(expected_values); + } + } + String hash = "1dbe7c11162abc87cc9ba4ae1ceb7fa0843a3011"; + check_last_coderun( + Arrays.asList( + new Triplet<>(dataProduct, oc_values_name, hash), + new Triplet<>(dataProduct, oc_x_name, hash), + new Triplet<>(dataProduct, oc_y_name, hash), + new Triplet<>(dataProduct, oc_z_name, hash)), + null); + } + + /** + * write a 2d (4,4) array, in array[1][4] slices + * + * @throws IOException + */ + @Test + @Order(35) + void testWriteArray_in_slices() throws IOException { + String dataProduct = "test/array2a"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = new Dimension[] {new Dimension(4), new Dimension(4)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][] values = new double[1][4]; + for (int xi = 0; xi < 4; xi++) { + for (int yi = 0; yi < 4; yi++) values[0][yi] = xi + (double) yi / 4.0; + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "9cd6b8fbe9c2dfc1ec38375bad70b333fc90cef9"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * same as previous; but write the slices as array[4] instead of array[1][4] + * + * @throws IOException + */ + @Test + @Order(36) + void testWriteArray_in_slices_1d() throws IOException { + String dataProduct = "test/array2b"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = new Dimension[] {new Dimension(4), new Dimension(4)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[] values = new double[4]; + for (int xi = 0; xi < 4; xi++) { + for (int yi = 0; yi < 4; yi++) values[yi] = xi + (double) yi / 4.0; + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "9cd6b8fbe9c2dfc1ec38375bad70b333fc90cef9"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * same as previous but write the slices in double rows: array[2][4] + * + * @throws IOException + */ + @Test + @Order(37) + void testWriteArray_in_slices2() throws IOException { + String dataProduct = "test/array2c"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = new Dimension[] {new Dimension(4), new Dimension(4)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][] values = new double[2][4]; + for (int xi = 0; xi < 4; xi += 2) { + for (int yi = 0; yi < 4; yi++) { + values[0][yi] = xi + (double) yi / 4.0; + values[1][yi] = xi + 1 + (double) yi / 4.0; + } + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "9cd6b8fbe9c2dfc1ec38375bad70b333fc90cef9"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * same as previous but write the slices in single values: array[1] + * + * @throws IOException + */ + @Test + @Order(38) + void testWriteArray_in_single_values() throws IOException { + String dataProduct = "test/array2d"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = new Dimension[] {new Dimension(4), new Dimension(4)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[] value = new double[1]; + for (int xi = 0; xi < 4; xi++) + for (int yi = 0; yi < 4; yi++) + try { + value[0] = xi + (double) yi / 4.0; + oc1.writeArrayData(new NumericalArrayImpl(value)); + } catch (EOFException e) { + // + } + } + String hash = "9cd6b8fbe9c2dfc1ec38375bad70b333fc90cef9"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * same as previous but write the slices in double values: array[2] + * + * @throws IOException + */ + @Test + @Order(39) + void testWriteArray_in_double_values() throws IOException { + String dataProduct = "test/array2e"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = new Dimension[] {new Dimension(4), new Dimension(4)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[] value = new double[2]; + for (int xi = 0; xi < 4; xi++) + for (int yi = 0; yi < 4; yi += 2) + try { + value[0] = xi + (double) yi / 4.0; + value[1] = xi + (double) (yi + 1) / 4.0; + + oc1.writeArrayData(new NumericalArrayImpl(value)); + } catch (EOFException e) { + // + } + } + String hash = "9cd6b8fbe9c2dfc1ec38375bad70b333fc90cef9"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing one big multi-dim object. + * + * @throws IOException + */ + @Test + @Order(40) + void testWrite_4d_array_a() throws IOException { + String dataProduct = "test/array4d_a"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][][][] values = new double[3][4][5][6]; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi++) + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) + values[xi][yi][zi][ci] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing three big array[4][5][6] objects. + * + * @throws IOException + */ + @Test + @Order(41) + void testWrite_4d_array_b() throws IOException { + String dataProduct = "test/array4d_b"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][][] values = new double[4][5][6]; + for (int xi = 0; xi < 3; xi++) { + for (int yi = 0; yi < 4; yi++) + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) + values[yi][zi][ci] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + LOGGER.trace("x: (" + xi + ") = " + Arrays.toString(values)); + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing 12 array[5][6] objects. + * + * @throws IOException + */ + @Test + @Order(42) + void testWrite_4d_array_c() throws IOException { + String dataProduct = "test/array4d_c"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][] values = new double[5][6]; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi++) { + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) values[zi][ci] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + LOGGER.trace("x,y: (" + xi + "," + yi + ") = " + Arrays.toString(values)); + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing 6 array[2][5][6] objects. + * + * @throws IOException + */ + @Test + @Order(43) + void testWrite_4d_array_d() throws IOException { + String dataProduct = "test/array4d_d"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[][][] values = new double[2][5][6]; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi += 2) { + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) { + values[0][zi][ci] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + values[1][zi][ci] = 100.0 * xi + 10.0 * (yi + 1) + zi + ci / 6.0; + } + LOGGER.trace("x,y: (" + xi + "," + yi + ") = " + Arrays.toString(values)); + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing 60 array[6] objects. + * + * @throws IOException + */ + @Test + @Order(44) + void testWrite_4d_array_e() throws IOException { + String dataProduct = "test/array4d_e"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[] values = new double[6]; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi++) + for (int zi = 0; zi < 5; zi++) { + for (int ci = 0; ci < 6; ci++) values[ci] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + LOGGER.trace("x,y,z: (" + xi + "," + yi + "," + zi + ") = " + Arrays.toString(values)); + + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * write a 4d array array[3][4][5][6] by writing 360 array[1] objects. + * + * @throws IOException + */ + @Test + @Order(45) + void testWrite_4d_array_f() throws IOException { + String dataProduct = "test/array4d_f"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + Dimension[] dims = + new Dimension[] {new Dimension(3), new Dimension(4), new Dimension(5), new Dimension(6)}; + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition(nadefname, NetcdfDataType.DOUBLE, dims, "", "", ""); + Object_component_write_array oc1 = dp.getComponent(nadef); + + double[] values = new double[1]; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi++) + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) { + values[0] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + LOGGER.trace("x,y,z,c: (" + xi + "," + yi + "," + zi + "," + ci + ") = " + values[0]); + + try { + oc1.writeArrayData(new NumericalArrayImpl(values)); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + null, Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash))); + } + + /** + * read a 4d array array[3][4][5][6] by reading 360 array[1] objects. + * + * @throws IOException + */ + @Test + @Order(45) + void testRead_4d_array_f() throws IOException { + String dataProduct = "test/array4d_f"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + + Object_component_read_nc oc1 = dp.getComponent(nadefname.getFullPath()); + + double[] values = new double[1]; + int[] shape = new int[] {1, 1, 1, 1}; + for (int xi = 0; xi < 3; xi++) + for (int yi = 0; yi < 4; yi++) + for (int zi = 0; zi < 5; zi++) + for (int ci = 0; ci < 6; ci++) { + values[0] = 100.0 * xi + 10.0 * yi + zi + ci / 6.0; + LOGGER.trace("x,y,z,c: (" + xi + "," + yi + "," + zi + "," + ci + ") = " + values[0]); + + try { + NumericalArray na = oc1.readArray(shape); + Number[][][][] n = na.as4DArray(); + Number nn = n[0][0][0][0]; + assertThat(nn).isEqualTo((Number) values[0]); + } catch (EOFException e) { + // + } + } + } + String hash = "75edff902a6cec8a27e5ca17c57be0f7e452d9b6"; + + check_last_coderun( + Arrays.asList(new Triplet<>(dataProduct, nadefname.getFullPath(), hash)), null); + } + + /** + * test that dimensions for non-metadata'd non-described dimensions get returned as just 'sized' + * dimensions. + * + * @throws IOException + */ + @Test + @Order(46) + void getVarDef() throws IOException { + String dataProduct = "test/array4d_f"; + String component_path = ""; + VariableName nadefname = new VariableName("array", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + + Object_component_read_nc oc1 = dp.getComponent(nadefname.getFullPath()); + VariableDefinition vDef = oc1.getVardef(); + assertThat(vDef.getDescription()).isEmpty(); + assertThat(vDef.getDataType()).isEqualTo(NetcdfDataType.DOUBLE); + assertThat(vDef.getUnits()).isEmpty(); + assertThat(vDef.getLong_name()).isEmpty(); + assertThat(vDef.getClass()).isEqualTo(DimensionalVariableDefinition.class); + DimensionalVariableDefinition dvDef = (DimensionalVariableDefinition) vDef; + Dimension[] dims = dvDef.getDimensions(); + + for (int i = 0; i < 4; i++) assertThat(dims[i].size()).isEqualTo(i + 3); + } + } + + /** + * test write table with 1 column only; with 3 ints. + * + * @throws IOException + */ + @Test + @Order(47) + void writeTable() throws IOException { + String dataProduct = "my_lovely_little_table"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("itemIndex"), + NetcdfDataType.INT, + "the index number of the item", + "", + "indexNr") + }; + TableDefinition tabledef = + new TableDefinition(tablename, 3, "", "", Collections.emptyMap(), columns); + Object_component_write_table oc1 = dp.getComponent(tabledef); + oc1.writeData(0, new int[] {2, 4, 6}); + } + String hash = "cfaeec9980ac37a8527f934ea3fba957d6d7fc9e"; + + check_last_coderun(null, Arrays.asList(new Triplet<>(dataProduct, component_path, hash))); + } + + /** + * test write table with a String and a double column only + * + * @throws IOException + */ + @Test + @Order(48) + void writeTable2() throws IOException { + String dataProduct = "my_lovely_little_table2"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("item_name"), + NetcdfDataType.STRING, + "the name of the item", + "", + "Item name"), + new LocalVariableDefinition( + new NetcdfName("item_price"), + NetcdfDataType.DOUBLE, + "the price of the item", + "", + "Item price") + }; + TableDefinition tabledef = + new TableDefinition(tablename, 3, "", "", Collections.emptyMap(), columns); + Object_component_write_table oc1 = dp.getComponent(tabledef); + oc1.writeData(0, new String[] {"Apples", "Pears", "Oranges"}); + oc1.writeData(1, new double[] {1.1, 2.2, 3.3}); + } + String hash = "2fdfc1c1dc88937c94f0c6172661ef4808b40257"; + + check_last_coderun(null, Arrays.asList(new Triplet<>(dataProduct, component_path, hash))); + } + + /** + * test write a table with unlimited size. + * + * @throws IOException + */ + @Test + @Order(49) + void writeTable3() throws IOException { + String dataProduct = "my_lovely_little_table3"; + String component_path = "tablex"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("item_name"), + NetcdfDataType.STRING, + "the name of the items", + "", + "Item name"), + new LocalVariableDefinition( + new NetcdfName("item_price"), + NetcdfDataType.DOUBLE, + "the price of the items", + "", + "Item price") + }; + TableDefinition tabledef = + new TableDefinition(tablename, 0, "", "", Collections.emptyMap(), columns); + Object_component_write_table oc1 = dp.getComponent(tabledef); + oc1.writeData(0, new String[] {"Apples", "Pears", "Oranges"}); + oc1.writeData(1, new double[] {1.1, 2.2, 3.3}); + } + String hash = "837a7f07ca0dbcdda0e76485a2f7fc16d5428481"; + + check_last_coderun(null, Arrays.asList(new Triplet<>(dataProduct, component_path, hash))); + } + + /** + * write table, unlimited size. write each column 3 items at a time. + * + * @throws IOException + */ + @Test + @Order(50) + void writeTable4() throws IOException { + String dataProduct = "my_lovely_little_table4"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("item_name"), + NetcdfDataType.STRING, + "the name of the item", + "", + "Item name"), + new LocalVariableDefinition( + new NetcdfName("item_price"), + NetcdfDataType.DOUBLE, + "the price of one item", + "GBP", + "Item price"), + new LocalVariableDefinition( + new NetcdfName("item_quantity"), + NetcdfDataType.INT, + "the number of items", + "", + "quantity") + }; + TableDefinition tabledef = + new TableDefinition(tablename, 0, "", "", Collections.emptyMap(), columns); + Object_component_write_table oc1 = dp.getComponent(tabledef); + oc1.writeData(0, new String[] {"Apples", "Pears", "Oranges"}); + oc1.writeData(1, new double[] {1.1, 2.2, 3.3}); + oc1.writeData(2, new int[] {3, 5, 7}); + + oc1.writeData(0, new String[] {"Bananas", "Kiwis", "Grapes"}); + oc1.writeData(1, new double[] {2.5, 1.5, 0.2}); + oc1.writeData(2, new int[] {9, 11, 13}); + } + String hash = "e7542ca7f48109246407c314ef8979650bb9efb5"; + + check_last_coderun(null, Arrays.asList(new Triplet<>(dataProduct, component_path, hash))); + } + + /** + * write table, with missing values + * + * @throws IOException + */ + @Test + @Order(51) + void writeTable5() throws IOException { + String dataProduct = "my_lovely_little_table5"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write_nc dp = coderun.get_dp_for_write_nc(dataProduct); + + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("item_name"), + NetcdfDataType.STRING, + "the name of the item", + "", + "Item name", + Collections.emptyMap(), + "_"), + new LocalVariableDefinition( + new NetcdfName("item_price"), + NetcdfDataType.DOUBLE, + "the price of one item", + "GBP", + "Item price", + Collections.emptyMap(), + Double.NaN), + new LocalVariableDefinition( + new NetcdfName("item_quantity"), + NetcdfDataType.INT, + "the number of items", + "", + "quantity", + Collections.emptyMap(), + Integer.MIN_VALUE) + }; + TableDefinition tabledef = + new TableDefinition(tablename, 0, "", "", Collections.emptyMap(), columns); + Object_component_write_table oc1 = dp.getComponent(tabledef); + oc1.writeData(0, new String[] {"Apples", "_", "Oranges"}); + oc1.writeData(1, new double[] {1.1, Double.NaN, 3.3}); + oc1.writeData(2, new int[] {3, Integer.MIN_VALUE, 7}); + } + String hash = "094efd5e8557d20d4842fe45dad01ca43ed718f2"; + + check_last_coderun(null, Arrays.asList(new Triplet<>(dataProduct, component_path, hash))); + } + + /** + * test read table with 1 column only; with 3 ints. + * + * @throws IOException + */ + @Test + @Order(52) + void readTable() throws IOException { + String dataProduct = "my_lovely_little_table"; + String component_path = "table"; + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + + Object_component_read_table oc1 = dp.getTable(component_path); + TableDefinition tabledef = oc1.getTabledef(); + Assertions.assertEquals("", tabledef.getDescription()); + Assertions.assertEquals("", tabledef.getLong_name()); + Assertions.assertEquals(3, tabledef.getSize()); + LocalVariableDefinition[] c = tabledef.getColumns(); + Assertions.assertEquals(1, c.length); + Assertions.assertEquals(NetcdfDataType.INT, c[0].getDataType()); + Assertions.assertEquals("the index number of the item", c[0].getDescription()); + Assertions.assertEquals("", c[0].getUnits()); + Assertions.assertEquals("indexNr", c[0].getLong_name()); + + Object o = oc1.readData(0); + int[] i = (int[]) o; + Assertions.assertArrayEquals(new int[] {2, 4, 6}, i); + } + String hash = "cfaeec9980ac37a8527f934ea3fba957d6d7fc9e"; + + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_path, hash)), null); + } + + /** + * test - when we are trying to open a component as Array, when in reality it is a table the + * dp.getComponent() will throw IllegalArgumentException. + * + * @throws IOException + */ + @Test + @Order(53) + void readTable_as_array() throws IOException { + String dataProduct = "my_lovely_little_table"; + String component_path = "table"; + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + Assertions.assertThrows( + NetcdfComponentWrongTypeException.class, + () -> { + dp.getComponent(component_path); + }); + } + } + + /** + * test - when we are trying to open a component as Array, when in reality it is a table the + * dp.getComponent() will throw IllegalArgumentException. + * + * @throws IOException + */ + @Test + @Order(54) + void readArray_as_table() throws IOException { + String dataProduct = "test/array1"; + String component_path = "component1/with/a/path"; + VariableName latname = new VariableName("lat", component_path); + VariableName arrayname = new VariableName("array1", component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + String latnames = latname.getFullPath(); + Assertions.assertThrows( + NetcdfComponentWrongTypeException.class, + () -> { + dp.getTable(latnames); + }); + String arraynames = arrayname.getFullPath(); + Assertions.assertThrows( + NetcdfComponentWrongTypeException.class, + () -> { + dp.getTable(arraynames); + }); + } + } + + /** + * test write table with a String and a double column only + * + * @throws IOException + */ + @Test + @Order(55) + void readTable2() throws IOException { + String dataProduct = "my_lovely_little_table2"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_table oc1 = dp.getTable(tablename.getGroupName()); + TableDefinition tabledef = oc1.getTabledef(); + Assertions.assertEquals(3, tabledef.getSize()); + Assertions.assertEquals("", tabledef.getDescription()); + Assertions.assertEquals("", tabledef.getLong_name()); + LocalVariableDefinition[] columns = tabledef.getColumns(); + Assertions.assertEquals(2, columns.length); + Assertions.assertEquals(NetcdfDataType.STRING, columns[0].getDataType()); + Assertions.assertEquals("item_name", columns[0].getLocalName().getName()); + Assertions.assertEquals("the name of the item", columns[0].getDescription()); + Assertions.assertEquals("", columns[0].getUnits()); + Assertions.assertEquals(NetcdfDataType.DOUBLE, columns[1].getDataType()); + Assertions.assertEquals("item_price", columns[1].getLocalName().getName()); + Assertions.assertEquals("the price of the item", columns[1].getDescription()); + Assertions.assertEquals("", columns[1].getUnits()); + String[] s = (String[]) oc1.readData(0); + Assertions.assertArrayEquals(new String[] {"Apples", "Pears", "Oranges"}, s); + double[] n = (double[]) oc1.readData(1); + Assertions.assertArrayEquals(new double[] {1.1, 2.2, 3.3}, n); + } + String hash = "2fdfc1c1dc88937c94f0c6172661ef4808b40257"; + + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_path, hash)), null); + } + + @Test + @Order(56) + void readTable3() throws IOException { + String dataProduct = "my_lovely_little_table3"; + String component_path = "tablex"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + Object_component_read_table oc1 = dp.getTable(tablename.getGroupName()); + TableDefinition tableDef = oc1.getTabledef(); + Assertions.assertEquals("", tableDef.getDescription()); + Assertions.assertEquals("", tableDef.getLong_name()); + Assertions.assertEquals(3, tableDef.getSize()); + LocalVariableDefinition[] columns = tableDef.getColumns(); + Assertions.assertEquals(2, columns.length); + Assertions.assertEquals(NetcdfDataType.STRING, columns[0].getDataType()); + Assertions.assertEquals("item_name", columns[0].getLocalName().getName()); + Assertions.assertEquals("the name of the items", columns[0].getDescription()); + Assertions.assertEquals("", columns[0].getUnits()); + Assertions.assertEquals(NetcdfDataType.DOUBLE, columns[1].getDataType()); + Assertions.assertEquals("item_price", columns[1].getLocalName().getName()); + Assertions.assertEquals("the price of the items", columns[1].getDescription()); + Assertions.assertEquals("", columns[1].getUnits()); + String[] s = (String[]) oc1.readData(0); + Assertions.assertArrayEquals(new String[] {"Apples", "Pears", "Oranges"}, s); + double[] n = (double[]) oc1.readData(1); + Assertions.assertArrayEquals(new double[] {1.1, 2.2, 3.3}, n); + } + String hash = "837a7f07ca0dbcdda0e76485a2f7fc16d5428481"; + + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_path, hash)), null); + } + + /** + * write table, unlimited size. write each column 3 items at a time. + * + * @throws IOException + */ + @Test + @Order(57) + void readTable4() throws IOException { + String dataProduct = "my_lovely_little_table4"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + + Object_component_read_table oc1 = dp.getTable(tablename.getGroupName()); + Assertions.assertNull(oc1.getTabledef().getColumns()[0].getMissingValue()); + Assertions.assertArrayEquals( + new String[] {"Apples", "Pears", "Oranges"}, (String[]) oc1.readData(0, 3)); + Assertions.assertArrayEquals(new double[] {1.1, 2.2, 3.3}, (double[]) oc1.readData(1, 3)); + Assertions.assertArrayEquals(new int[] {3, 5, 7}, (int[]) oc1.readData(2, 3)); + + Assertions.assertArrayEquals( + new String[] {"Bananas", "Kiwis", "Grapes"}, (String[]) oc1.readData(0, 3)); + Assertions.assertArrayEquals(new double[] {2.5, 1.5, 0.2}, (double[]) oc1.readData(1, 3)); + Assertions.assertArrayEquals(new int[] {9, 11, 13}, (int[]) oc1.readData(2, 3)); + } + String hash = "e7542ca7f48109246407c314ef8979650bb9efb5"; + + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_path, hash)), null); + } + + /** + * write table, with missing values + * + * @throws IOException + */ + @Test + @Order(58) + void readTable5() throws IOException { + String dataProduct = "my_lovely_little_table5"; + String component_path = "table"; + NetcdfGroupName tablename = new NetcdfGroupName(component_path); + + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read_nc dp = coderun.get_dp_for_read_nc(dataProduct); + + Object_component_read_table oc1 = dp.getTable(tablename.getGroupName()); + LocalVariableDefinition[] c = oc1.getTabledef().getColumns(); + Assertions.assertEquals("_", c[0].getMissingValue()); + Assertions.assertEquals(Double.NaN, c[1].getMissingValue()); + Assertions.assertEquals(Integer.MIN_VALUE, c[2].getMissingValue()); + + Assertions.assertArrayEquals( + new String[] {"Apples", "_", "Oranges"}, (String[]) oc1.readData(0)); + Assertions.assertArrayEquals(new double[] {1.1, Double.NaN, 3.3}, (double[]) oc1.readData(1)); + Assertions.assertArrayEquals(new int[] {3, Integer.MIN_VALUE, 7}, (int[]) oc1.readData(2)); + } + String hash = "094efd5e8557d20d4842fe45dad01ca43ed718f2"; + + check_last_coderun(Arrays.asList(new Triplet<>(dataProduct, component_path, hash)), null); + } } diff --git a/api/src/test/java/org/fairdatapipeline/config/ConfigTester.java b/api/src/test/java/org/fairdatapipeline/config/ConfigTest.java similarity index 97% rename from api/src/test/java/org/fairdatapipeline/config/ConfigTester.java rename to api/src/test/java/org/fairdatapipeline/config/ConfigTest.java index 21bfbb3..752b1fc 100644 --- a/api/src/test/java/org/fairdatapipeline/config/ConfigTester.java +++ b/api/src/test/java/org/fairdatapipeline/config/ConfigTest.java @@ -10,7 +10,7 @@ import org.junit.jupiter.params.provider.ValueSource; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class ConfigTester { +class ConfigTest { YamlReader yamlReader; @BeforeAll diff --git a/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfBuilderTest.java b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfBuilderTest.java new file mode 100644 index 0000000..a8d9868 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfBuilderTest.java @@ -0,0 +1,465 @@ +package org.fairdatapipeline.netcdf; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Objects; +import org.apache.commons.io.FileUtils; +import org.fairdatapipeline.objects.CoordinateVariableDefinition; +import org.fairdatapipeline.objects.DimensionalVariableDefinition; +import org.fairdatapipeline.objects.LocalVariableDefinition; +import org.fairdatapipeline.objects.TableDefinition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import ucar.nc2.*; +import ucar.nc2.write.Nc4Chunking; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class NetcdfBuilderTest { + final Runnable onClose = this::myClose; + + private void myClose() { + // do nothing + } + + /** + * testing creation of a single group in netcdf using NetcdfBuilder.getGroup() + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_create_group_with_dim() throws IOException, URISyntaxException { + String filename = "test_create_group_with_dim"; + String resourceName = "/netcdf/test_create_group_with_dim.nc"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Group.Builder g = b.getGroup(null, "/aap/noot/mies"); + g.addDimension(new Dimension("bla", 3)); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * testing creation of a group and its subgroup (each containing a dimension), using + * NetcdfBuilder.getGroup() + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_create_2groups_with_dims() throws IOException, URISyntaxException { + String filename = "test_create_2groups_with_dims"; + String resourceName = "/netcdf/test_create_2groups_with_dims.nc"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Group.Builder g1 = b.getGroup(null, "/aap/noot/mies"); + Group.Builder g2 = b.getGroup(null, "/aap/noot/mies/pis"); + g1.addDimension(new Dimension("miesdim", 3)); + g2.addDimension(new Dimension("pisdim", 2)); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * this creates two tables: /my/little/table and /my/little/othertable + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_create_2table_groups() throws IOException, URISyntaxException { + String filename = "test_create_table_group"; + String resourceName = "/netcdf/test_create_table_group.nc"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Group.Builder g1 = b.getGroup(null, "/my/little/table", true); + g1.addAttribute(new Attribute("group_type", "table")); + Group.Builder g2 = b.getGroup(null, "/my/little/othertable", true); + g2.addAttribute(new Attribute("group_type", "table")); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** show that it is not possible to create a table within another table. */ + @Test + void test_create_table_in_table() throws IOException { + String filename = "test_create_table_in_table"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Group.Builder g1 = b.getGroup(null, "/my/little/table", true); + g1.addAttribute(new Attribute("group_type", "table")); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> b.getGroup(null, "/my/little/table/subtable", true)); + } + } + + /** show that it is not possible to 'get' a group within another table. */ + @Test + void test_create_something_in_table() throws IOException { + String filename = "test_create_something_in_table"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Group.Builder g1 = b.getGroup(null, "/my/little/table", true); + g1.addAttribute(new Attribute("group_type", "table")); + Assertions.assertThrows( + IllegalArgumentException.class, () -> b.getGroup(null, "/my/little/table/subgroup")); + } + } + + /** + * this shows that the 'mustBeFresh' argument prevents 'get'-ting a group that already exists. + * this is in order to make sure a table always creates its own unique group to exist in. + */ + @Test + void test_create_table_in_existing_group() throws IOException { + String filename = "test_create_table_in_existing_group"; + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.getGroup(null, "/my/little/group"); + Assertions.assertThrows( + IllegalArgumentException.class, () -> b.getGroup(null, "/my/little/group", true)); + } + } + + /** + * creating a coordinate variable. the int[] {1, 2, 3} determines its datatype and its size. it is + * only prepared here, so the 1, 2, 3 values are not stored. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_prepare_coordinatevar() throws IOException, URISyntaxException { + String filename = "test_prepare_coordinatevar"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + CoordinateVariableDefinition cvdef = + new CoordinateVariableDefinition( + new VariableName("coordinatevariable", ""), + new int[] {1, 2, 3}, + "my first coordinate", + "cm", + "very long name of my coordinate variable"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + + b.prepare(cvdef); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * creating two coordinate variables with the same name. then a dimensional variable referencing + * the name of the two coordinate variables. the variable is now 4 long, probably because the + * 4-long dimension is 'nearer' the variable. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_prepare_coordinatevars() throws IOException, URISyntaxException { + String filename = "test_prepare_coordinatevars"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + CoordinateVariableDefinition cvdef1 = + new CoordinateVariableDefinition( + new VariableName("dim", ""), new int[] {1, 2, 3}, "", "", ""); + CoordinateVariableDefinition cvdef2 = + new CoordinateVariableDefinition( + new VariableName("dim", "one"), new int[] {1, 2, 3, 4}, "", "", ""); + DimensionalVariableDefinition dimdef = + new DimensionalVariableDefinition( + new VariableName("var", "one/two"), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] { + new org.fairdatapipeline.objects.Dimension("dim") + }, + "", + "", + ""); + + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + + b.prepare(cvdef1); + b.prepare(cvdef2); + b.prepare(dimdef); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * using a dimensionalvar with zero length dimensions; results in a scalar variable. not sure if I + * had meant this to be so, and if it should be so. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_prepare_scalar_dimensionalvar() throws IOException, URISyntaxException { + String filename = "test_prepare_scalar_dimensionalvar"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + DimensionalVariableDefinition dimdef = + new DimensionalVariableDefinition( + new VariableName("dimensionalvariable", ""), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] {}, + "my first dimensional variable", + "cm", + "very long name of my coordinate variable"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + + b.prepare(dimdef); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * when trying to create a dimensionalvariable the referenced dimensions in the dimensions[] must + * exist. + * + * @throws IOException + */ + @Test + void test_prepare_dimensionalvar_missing_dim() throws IOException { + String filename = "test_prepare_dimensionalvar"; + String extension = ".nc"; + Path filePath = Files.createTempFile(filename, extension); + DimensionalVariableDefinition dimdef = + new DimensionalVariableDefinition( + new VariableName("dimensionalvariable", ""), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] { + new org.fairdatapipeline.objects.Dimension("dimension1") + }, + "my first dimensional variable", + "cm", + "very long name of my coordinate variable"); + + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + Assertions.assertThrows(IllegalArgumentException.class, () -> b.prepare(dimdef)); + } + } + + /** + * creating a coordinatevariable and a 1-d dimensional variable referencing the coordinate + * variable. we have to make sure we first prepare the coordinate variable; this has to exist when + * we create the dimensional variable that references it. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_prepare_dimensionalvar() throws IOException, URISyntaxException { + String filename = "test_prepare_dimensionalvar"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + + DimensionalVariableDefinition dimdef = + new DimensionalVariableDefinition( + new VariableName("dimensionalvariable", ""), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] { + new org.fairdatapipeline.objects.Dimension("dimension1") + }, + "my first dimensional variable", + "cm", + "very long name of my coordinate variable"); + CoordinateVariableDefinition cvdef = + new CoordinateVariableDefinition( + new VariableName("dimension1", ""), + new int[] {1, 2, 3}, + "my first coordinate", + "cm", + "very long name of my coordinate variable"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(cvdef); + b.prepare(dimdef); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * creating a coordinatevariable and a 1-d dimensional variable referencing the coordinate + * variable. we have to make sure we first prepare the coordinate variable; this has to exist when + * we create the dimensional variable that references it. + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_prepare_dimensionalvar_remotedim() throws IOException, URISyntaxException { + String filename = "test_prepare_dimensionalvar_remotedim"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + + DimensionalVariableDefinition dimdef = + new DimensionalVariableDefinition( + new VariableName("dimensionalvariable", "mygroup/mysubgroup"), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] { + new org.fairdatapipeline.objects.Dimension("dimension1") + }, + "my first dimensional variable", + "cm", + "very long name of my coordinate variable"); + CoordinateVariableDefinition cvdef = + new CoordinateVariableDefinition( + new VariableName("dimension1", "mygroup"), + new int[] {1, 2, 3}, + "my first coordinate", + "cm", + "very long name of my coordinate variable"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(cvdef); + b.prepare(dimdef); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** just testing the string array optional attributes. */ + @Test + void test_linked_from_multi_attribute() throws IOException, URISyntaxException { + String filename = "test_linked_from_multi_attribute"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + + TableDefinition td = + new TableDefinition( + new NetcdfGroupName("table"), + 1, + "", + "", + Collections.singletonMap("linked_from", new String[] {"apples", "pears"}), + new LocalVariableDefinition[] { + new LocalVariableDefinition(new NetcdfName("column1"), NetcdfDataType.INT, "", "", "") + }); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(td); + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void test_add_dimension_after_dimensionalvariable() throws IOException, URISyntaxException { + String filename = "test_add_dimension_after_dimensionalvariable"; + String extension = ".nc"; + String resourceName = "/netcdf/" + filename + extension; + Path filePath = Files.createTempFile(filename, extension); + + CoordinateVariableDefinition cvar_in_group = + new CoordinateVariableDefinition( + new VariableName("dim", "group"), NetcdfDataType.INT, 6, "", "", ""); + + CoordinateVariableDefinition cvar_in_root = + new CoordinateVariableDefinition( + new VariableName("dim", ""), NetcdfDataType.INT, 3, "", "", ""); + + DimensionalVariableDefinition td = + new DimensionalVariableDefinition( + new VariableName("array", "group"), + NetcdfDataType.INT, + new org.fairdatapipeline.objects.Dimension[] { + new org.fairdatapipeline.objects.Dimension("dim") + }, + "", + "", + ""); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(cvar_in_root); + b.prepare(td); + // array is now 3 long - it gets its length from /dim + b.prepare(cvar_in_group); + // and now the array is 6 long - it gets its length from /group/dim + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfDataTypeTest.java b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfDataTypeTest.java new file mode 100644 index 0000000..33ef262 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfDataTypeTest.java @@ -0,0 +1,102 @@ +package org.fairdatapipeline.netcdf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.Index; + +class NetcdfDataTypeTest { + @Test + void test_translate_datatype_integer() { + Object o = new Integer[] {1, 2, 3}; + DataType dt = NetcdfDataType.translate(NetcdfDataType.translateDatatype(o)); + Assertions.assertEquals(DataType.INT, dt); + } + + @Test + void test_translate_datatype_int() { + Object o = new int[] {1, 2, 3}; + DataType dt = NetcdfDataType.translate(NetcdfDataType.translateDatatype(o)); + Assertions.assertEquals(DataType.INT, dt); + } + + @Test + void test_translate_datatype_double() { + Object o = new double[] {1.1, 2.2, 3.3}; + DataType dt = NetcdfDataType.translate(NetcdfDataType.translateDatatype(o)); + Assertions.assertEquals(DataType.DOUBLE, dt); + } + + @Test + void test_translate_datatype_string() { + Object o = new String[] {"a", "s", "ee"}; + DataType dt = NetcdfDataType.translate(NetcdfDataType.translateDatatype(o)); + Assertions.assertEquals(DataType.STRING, dt); + } + + @Test + void translate_array_test_int() { + Object o = new int[] {1, 2, 3, 4}; + ucar.ma2.Array a = NetcdfDataType.translateArray(o); + Assertions.assertEquals(2, a.getInt(1)); + } + + @Test + void translate_array_test_double() { + Object o = new double[] {1.1, 2.2, 3.3, 4.4}; + ucar.ma2.Array a = NetcdfDataType.translateArray(o); + Assertions.assertEquals(2.2, a.getDouble(1)); + } + + @Test + void translate_array_test_string() { + Object o = new String[] {"aap", "noot", "mies"}; + ucar.ma2.Array a = NetcdfDataType.translateArray(o); + Assertions.assertEquals("noot", a.getObject(1)); + } + + @Test + void translate_array_test_multidim() { + Object o = new int[] {1, 2, 3, 11, 12, 13}; + ucar.ma2.Array a = NetcdfDataType.translateArray(NetcdfDataType.INT, new int[] {2, 3}, o); + Assertions.assertEquals(13, a.getInt(Index.factory(a.getShape()).set0(1).set1(2))); + } + + @Test + void makeArrayFromJavaArrayint() { + Object o = new int[] {1, 2, 3}; + Array a = Array.makeFromJavaArray(o); + Assertions.assertEquals(2, a.getInt(1)); + } + + @Test + void makeArrayFromJavaArraydouble() { + Object o = new double[] {1.1, 2.2, 3.3}; + Array a = Array.makeFromJavaArray(o); + Assertions.assertEquals(2.2, a.getDouble(1)); + } + + @Test + void makeArrayFromJavaInt2d() { + Object o = new int[][] {{1, 2, 3}, {11, 22, 22}}; + Array a = Array.makeFromJavaArray(o); + Assertions.assertEquals(22, a.getInt(Index.factory(a.getShape()).set0(1).set1(1))); + } + + @Test + void makeArrayFromJava_addedTopDimension() { + int[] i = new int[] {1, 2, 3}; + Array a = Array.makeArrayRankPlusOne(Array.makeFromJavaArray(i)); + Assertions.assertEquals(2, a.getShape().length); + } + + @Test + @Disabled("this one failed because makeFromJavaArray only works on an Array of primitives") + void makeArrayFromJavaArrayString() { + Object o = new String[] {"aap", "noot", "mies"}; + Array a = Array.makeFromJavaArray(o); + Assertions.assertEquals("noot", a.getObject(1)); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfReaderTest.java b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfReaderTest.java new file mode 100644 index 0000000..355b295 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfReaderTest.java @@ -0,0 +1,93 @@ +package org.fairdatapipeline.netcdf; + +import java.io.IOException; +import java.util.Objects; +import org.fairdatapipeline.objects.Dimension; +import org.fairdatapipeline.objects.DimensionalVariableDefinition; +import org.fairdatapipeline.objects.NumericalArray; +import org.fairdatapipeline.objects.VariableDefinition; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import ucar.nc2.Variable; + +class NetcdfReaderTest { + private static final String read_int_resource = "/netcdf/test_build_prepare_write_INT.nc"; + + /** + * test only the reading of metadata (of the 'temperature' variable). does not test reading of + * data. + * + * @throws IOException + */ + @Test + void test_read_int_ardef() throws IOException { + String filename = Objects.requireNonNull(getClass().getResource(read_int_resource)).toString(); + NetcdfReader reader = new NetcdfReader(filename); + VariableDefinition ardef = reader.getArray("aap/noot/mies/temperature"); + Assertions.assertEquals(NetcdfDataType.INT, ardef.getDataType()); + Assertions.assertEquals("a test dataset with temperatures in 2d space", ardef.getDescription()); + Assertions.assertEquals("C", ardef.getUnits()); + Assertions.assertEquals("surface temperature", ardef.getLong_name()); + Assertions.assertEquals( + org.fairdatapipeline.objects.DimensionalVariableDefinition.class, ardef.getClass()); + DimensionalVariableDefinition dimvar = (DimensionalVariableDefinition) ardef; + Assertions.assertEquals(new Dimension("X"), dimvar.getDimensions()[0]); + Assertions.assertEquals(new Dimension("Y"), dimvar.getDimensions()[1]); + } + + /** + * test only the reading of metadata (of the 'X' variable). does not test reading of data. + * + * @throws IOException + */ + @Test + void test_read_int_Xdimdef() throws IOException { + String filename = Objects.requireNonNull(getClass().getResource(read_int_resource)).toString(); + NetcdfReader reader = new NetcdfReader(filename); + VariableDefinition ardef = reader.getArray("aap/noot/mies/X"); + Assertions.assertEquals(NetcdfDataType.INT, ardef.getDataType()); + Assertions.assertEquals( + org.fairdatapipeline.objects.CoordinateVariableDefinition.class, ardef.getClass()); + Assertions.assertEquals( + "the x-axis is measured in along the length of my football pitch; (0,0) is the southwest corner.", + ardef.getDescription()); + Assertions.assertEquals("m", ardef.getUnits()); + Assertions.assertEquals("", ardef.getLong_name()); + } + + /** + * test reading the whole array + * + * @throws IOException + */ + @Test + void test_read_int_data() throws IOException { + String filename = Objects.requireNonNull(getClass().getResource(read_int_resource)).toString(); + NetcdfReader reader = new NetcdfReader(filename); + Variable v = reader.getVariable("aap/noot/mies/temperature"); + NumericalArray na = reader.read(v); + Assertions.assertEquals(1, na.as2DArray()[0][0]); + Assertions.assertArrayEquals(new Number[] {1, 2, 3}, na.as2DArray()[0]); + Assertions.assertArrayEquals(new Number[] {11, 12, 13}, na.as2DArray()[1]); + } + + /** + * test reading the array row by row + * + * @throws IOException + */ + @Test + void test_read_int_data_by_row() throws IOException { + String filename = Objects.requireNonNull(getClass().getResource(read_int_resource)).toString(); + NetcdfReader reader = new NetcdfReader(filename); + Variable v = reader.getVariable("aap/noot/mies/temperature"); + int[] ori = new int[] {0, 0}; + int[] shape = new int[] {1, 3}; + for (int i = 0; i <= 1; i++) { + ori[0] = i; + NumericalArray na = reader.read(v, ori, shape); + if (i == 0) Assertions.assertArrayEquals(new Number[] {1, 2, 3}, na.as2DArray()[0]); + else Assertions.assertArrayEquals(new Number[] {11, 12, 13}, na.as2DArray()[0]); + } + } +} diff --git a/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfWriterTest.java b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfWriterTest.java new file mode 100644 index 0000000..c106728 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/netcdf/NetcdfWriterTest.java @@ -0,0 +1,1067 @@ +package org.fairdatapipeline.netcdf; + +import java.io.*; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.io.FileUtils; +import org.fairdatapipeline.objects.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; +import ucar.nc2.write.Nc4Chunking; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class NetcdfWriterTest { + private static final Logger logger = LoggerFactory.getLogger(NetcdfWriterTest.class); + private static final String COMMA_DELIMITER = ","; + final Runnable onClose = this::myClose; + + private void myClose() { + // do nothing + } + + /** + * test a simple INT array stored using NetcdfBuilder.prepareArray and .writeDimensionVariables + * and .writeArrayData + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_build_prepare_write_INT() throws IOException, URISyntaxException { + String filename = "test_build_prepare_write_INT"; + String resourceName = "/netcdf/test_build_prepare_write_INT.nc"; + String group = "aap/noot/mies"; + VariableName xName = new VariableName("X", group); + VariableName yName = new VariableName("Y", group); + VariableName tempName = new VariableName("temperature", group); + + CoordinateVariableDefinition xdim = + new CoordinateVariableDefinition( + xName, + new int[] {2, 4}, + "the x-axis is measured in along the length of my football pitch; (0,0) is the southwest corner.", + "m", + ""); + CoordinateVariableDefinition ydim = + new CoordinateVariableDefinition( + yName, + new int[] {3, 6, 9}, + "the y-axis is measured in along the width of my football pitch; (0,0) is the southwest corner.", + "m", + ""); + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + tempName, + NetcdfDataType.INT, + new Dimension[] {new Dimension(xName.getName()), new Dimension(yName.getName())}, + "a test dataset with temperatures in 2d space", + "C", + "surface temperature"); + NumericalArray nadat = new NumericalArrayImpl(new int[][] {{1, 2, 3}, {11, 12, 13}}); + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(xdim); + b.prepare(ydim); + b.prepare(nadef); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(xdim); + w.writeDimensionVariable(ydim); + w.writeArrayData(w.getVariable(nadef.getVariableName()), nadat); + } catch (InvalidRangeException e) { + // + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void test_build_prepare_write_with_fillvalue() throws IOException, URISyntaxException { + String filename = "test_build_prepare_write_with_fillvalue"; + String resourceName = "/netcdf/test_build_prepare_write_with_fillvalue.nc"; + String group = "mytestgroup"; + VariableName dimName = new VariableName("dim", group); + VariableName dataName = new VariableName("data", group); + + CoordinateVariableDefinition dim = + new CoordinateVariableDefinition( + dimName, new int[] {1, 2, 3, 4, 5, 6, 7, 8}, "as simple as ABC", "", ""); + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + dataName, + NetcdfDataType.INT, + new Dimension[] {new Dimension(dimName.getName())}, + "a test dataset with missing values", + "", + "", + Collections.emptyMap(), + -1); + NumericalArray nadat = new NumericalArrayImpl(new int[] {1, 2, 3, -1, 5, 6}); + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(dim); + b.prepare(nadef); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(dim); + w.writeArrayData(w.getVariable(nadef.getVariableName()), nadat); + } catch (InvalidRangeException e) { + // + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void test_table() throws IOException, URISyntaxException { + String filename = "test_table"; + String resourceName = "/netcdf/test_table.nc"; + + LocalVariableDefinition[] columns; + + NetcdfGroupName tableName = new NetcdfGroupName("myTestTable"); + + columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition(new NetcdfName("id"), NetcdfDataType.INT, "", "", ""), + new LocalVariableDefinition( + new NetcdfName("itemName"), NetcdfDataType.STRING, "", "", "item name"), + new LocalVariableDefinition( + new NetcdfName("value"), NetcdfDataType.DOUBLE, "", "", "value of the item") + }; + + TableDefinition table = + new TableDefinition( + tableName, + 0, + "This is a simple test table to see what it will look like in a netCDF file.", + "My little simple initial test table", + Collections.emptyMap(), + columns); + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + // prescription variables: + b.prepare(table); + + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + int[] ids = new int[] {1, 2, 3}; + String[] names = new String[] {"apples", "bananas", "pears"}; + double[] doubles = new double[] {1.1, 2.2, 3.3}; + + w.writeArrayData( + w.getVariable(table.getVariableName(0)), NetcdfDataType.translateArray(ids)); + w.writeArrayData( + w.getVariable(table.getVariableName(1)), NetcdfDataType.translateArray(names)); + w.writeArrayData( + w.getVariable(table.getVariableName(2)), NetcdfDataType.translateArray(doubles)); + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void test_table_with_missing() throws IOException, URISyntaxException { + String filename = "test_table_with_missing"; + String resourceName = "/netcdf/test_table_with_missing.nc"; + + LocalVariableDefinition[] columns; + + NetcdfGroupName tableName = new NetcdfGroupName("myTestTable"); + + columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("id"), NetcdfDataType.INT, "", "", "", Collections.emptyMap(), -1), + new LocalVariableDefinition( + new NetcdfName("itemName"), + NetcdfDataType.STRING, + "", + "", + "item name", + Collections.emptyMap(), + ""), + new LocalVariableDefinition( + new NetcdfName("value"), + NetcdfDataType.DOUBLE, + "", + "", + "value of the item", + Collections.emptyMap(), + Double.NaN) + }; + + TableDefinition table = + new TableDefinition( + tableName, + 0, + "This is a simple test table to see what it will look like in a netCDF file.", + "My little simple test table with some missing values", + Collections.singletonMap( + "random_attribute", new String[] {"just testing a random attribute"}), + columns); + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + // prescription variables: + b.prepare(table); + + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + int[] ids = new int[] {1, -1, 3, 4, 5, 6}; + String[] names = new String[] {"apples", "bananas", "pears", "", "kiwis", "mangos"}; + double[] doubles = new double[] {1.1, 2.2, 3.3, 4.4, Double.NaN, 6.6}; + + w.writeArrayData( + w.getVariable(table.getVariableName(0)), NetcdfDataType.translateArray(ids)); + w.writeArrayData( + w.getVariable(table.getVariableName(1)), NetcdfDataType.translateArray(names)); + w.writeArrayData( + w.getVariable(table.getVariableName(2)), NetcdfDataType.translateArray(doubles)); + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void test_prescribing_data() throws IOException, URISyntaxException { + String filename = "test_prescribing_data"; + String resourceName = "/netcdf/test_prescribing_data.nc.bz2"; + String prescribingCsvResourceName = "/prescribing_june2016.csv.bz2"; + String hbtResourceName = "/health boards.csv"; + String countrycodeName = "/country code.csv"; + + LocalVariableDefinition[] columns; + + NetcdfGroupName prescribingGroupName = + new NetcdfGroupName( + "themes/health_and_care/prescriptions_in_the_community/prescribing_data_june_2016"); + NetcdfGroupName healthboardGroupName = + new NetcdfGroupName( + "themes/health_and_care/geography_codes_and_labels/health_board_2014_-_health_board_2019"); + NetcdfGroupName countriesGroupName = + new NetcdfGroupName("themes/health_and_care/geography_codes_and_labels/country"); + int num_healthboards = 18; + int num_countries = 1; + + // COUNTRY VARIABLES: + + columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition(new NetcdfName("id"), NetcdfDataType.INT, "", "", ""), + new LocalVariableDefinition( + new NetcdfName("Country"), NetcdfDataType.STRING, "", "", "Country Code"), + new LocalVariableDefinition( + new NetcdfName("CountryName"), NetcdfDataType.STRING, "", "", "Name of the Country") + }; + TableDefinition countries = + new TableDefinition( + countriesGroupName, + 0, + "9 digit standard geography codes (S92) and matching labels for Country", + "", + Stream.of( + new String[][] { + { + "source", + "https://www.opendata.nhs.scot/dataset/geography-codes-and-labels/resource/9c6e6c56-2697-4184-92c6-60d69c2b6792" + }, + { + "linked_from", + new VariableName(new NetcdfName("Country"), healthboardGroupName).toString() + } + }) + .collect(Collectors.toMap(data -> data[0], data -> new String[] {data[1]})), + columns); + + // HEALTH BOARDS VARIABLES: + + columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition(new NetcdfName("id"), NetcdfDataType.INT, "", "", ""), + new LocalVariableDefinition( + new NetcdfName("HB"), + NetcdfDataType.STRING, + "", + "", + "Health Board 2014 Code (revised in 2018 & 2019)"), + new LocalVariableDefinition( + new NetcdfName("HBName"), + NetcdfDataType.STRING, + "", + "", + "Name of the Health Board 2014 (revised in 2018 & 2019)"), + new LocalVariableDefinition( + new NetcdfName("HBDateEnacted"), + NetcdfDataType.INT, + "", + "", + "Date Health Board Code was enacted."), + new LocalVariableDefinition( + new NetcdfName("HBDateArchived"), + NetcdfDataType.INT, + "", + "", + "Date Health Board Code was archived.", + Collections.emptyMap(), + 0), + new LocalVariableDefinition( + new NetcdfName("Country"), + NetcdfDataType.STRING, + "", + "", + "Country Code for Scotland", + Collections.singletonMap("linked_to", new String[] {countriesGroupName.toString()})) + }; + + TableDefinition healthboards = + new TableDefinition( + healthboardGroupName, + 0, + "9 digit standard geography codes (S08) and matching labels Health Board 2014 (revised in 2018 & 2019) in the health sector.\n" + + "\n" + + "All 14 Health Boards are listed with their corresponding name and the country code for Scotland.\n" + + "\n" + + "From 02-Feb 2018 there has been a minor boundary change to Keltybridge and Fife Environmental Energy Park at Westfield. From 01-Apr 2019 " + + "there has been a boundary change to Cardowan by Stepps. On both occasions, the affected Council Area, Health and Social Care Partnership " + + "and Health Board codes have been archived, and new codes introduced reflecting the new boundaries.", + "Health Board 2014 - Health Board 2019", + Stream.of( + new String[][] { + { + "source", + "https://www.opendata.nhs.scot/dataset/geography-codes-and-labels/resource/652ff726-e676-4a20-abda-435b98dd7bdc" + }, + { + "linked_from", + new VariableName(new NetcdfName("HBT"), prescribingGroupName).toString() + } + }) + .collect(Collectors.toMap(data -> data[0], data -> new String[] {data[1]})), + columns); + + // ACTUAL PRESCRIPTION VARIABLES: + + columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition(new NetcdfName("id"), NetcdfDataType.INT, "", "", ""), + new LocalVariableDefinition( + new NetcdfName("HBT"), + NetcdfDataType.STRING, + "", + "", + "Each NHS health board has a unique nine digit code identifying the NHS board where prescribing of an item took place, based on boundaries as at 1st April 2014", + Collections.singletonMap( + "linked_to", new String[] {healthboardGroupName.toString()})), + new LocalVariableDefinition( + new NetcdfName("GPPractice"), + NetcdfDataType.INT, + "", + "", + "Unique five digit numeric GP practice code identifying where the prescribing of an item took place. If it is not possible to determine the exact location from which a prescription originates it is assigned to an unallocated practice code. __Unallocated__ practice codes have been assigned the 99997. Prescriptions that originated from a __dentist surgery__ have been assigned the code 99999. Prescriptions that originated from a community __pharmacy__ have been assigned the code 99996. Prescriptions that originated from a __hospital__ have been assigned the code 99998.", + Collections.singletonMap( + "linked_to", + new String[] { + "https://www.opendata.nhs.scot/dataset/gp-practice-contact-details-and-list-sizes" + })), + new LocalVariableDefinition( + new NetcdfName("BNFItemCode"), + NetcdfDataType.STRING, + "", + "", + "A 15 digit British National Formulary (BNF) Item code in which the first seven digits are allocated according to the categories in the BNF and the last 8 digits represent the medicinal product, form, strength and the link to the generic equivalent product. The BNF Item Code takes the following form: *Characters 1 & 2 show BNF chapter; *3 & 4 show the BNF section; *5 & 6 show the BNF paragraph; *7 shows the BNF sub-paragraph; *8 & 9 show the chemical substance; *10 & 11 show the product; *12 & 13 show the strength and formulation; *14 & 15 show the link to the generic equivalent product: Where the product is a generic, the 14th and 15th characters will be the same as the 12th and 13th character; Where the product is a brand, the 14th and 15th characters will be the same as the generic equivalent (if this exists); Where the product is a brand and a generic equivalent does not exist, the 14th and 15th characters will be \"A0\". There are items within the prescribing dataset (mainly from the additional chapters that are included as appendices within the BNF or not listed in the BNF) that have a shorter BNF item code which has been manually input to allow users of the data to identify the BNF chapter, BNF section, BNF paragraph, and BNF sub-paragraph an item aligns to. These items follow the numbering convention above for the initial six digits."), + new LocalVariableDefinition( + new NetcdfName("BNFItemDescription"), + NetcdfDataType.STRING, + "", + "", + "The drug item description as it appears in the latest edition of the British National Formulary (BNF), detailing the product name, formulation and strength."), + new LocalVariableDefinition( + new NetcdfName("NumberOfPaidItems"), + NetcdfDataType.INT, + "", + "", + "The number of paid items relates to the number of prescription items dispensed and for which the dispenser has been reimbursed. An item is an individual product dispensed, e.g. 100 aspirin tablets of 300mg. There should be a maximum of three line items on a prescription; this should be three individual products defined by active ingredient, formulation type and strength for medicines, with appropriate parallel measures for appliances. A compounded product with a known formula will count as one item despite the number of ingredients."), + new LocalVariableDefinition( + new NetcdfName("PaidQuantity"), + NetcdfDataType.FLOAT, + "", + "", + "Paid quantity of an individual item for which the dispenser has been reimbursed, e.g. 100 tablets."), + new LocalVariableDefinition( + new NetcdfName("GrossIngredientCost"), + NetcdfDataType.FLOAT, + "", + "", + "Paid Gross Ingredient Cost (excluding Broken Bulk) is the cost of drugs and appliances reimbursed before deduction of any dispenser discount, i.e. the basic price of a drug as listed in the Scottish Drug Tariff or price lists. Note that this definition differs from other parts of the UK. The figures are in £s and pence. The Gross Ingredient Cost measure excludes broken bulk, that allows a contractor to claim a complete pack where a prescription is received for a product which comes in a larger pack and there is a risk of no further prescriptions for the product before the stock expires."), + new LocalVariableDefinition( + new NetcdfName("PaidDateMonth"), + NetcdfDataType.INT, + "", + "", + "The date (YYYYMM) in which the prescription item was processed for payment.") + }; + + TableDefinition prescribing = + new TableDefinition( + prescribingGroupName, + 0, + "Information on community pharmacy activity and direct pharmaceutical care services, covering June 2016 for all NHS health boards. Publication Date 13 September 2016", + "Prescribing Data June 2016", + Collections.singletonMap( + "source", + new String[] { + "https://www.opendata.nhs.scot/dataset/prescriptions-in-the-community/resource/a636862a-77e0-4c97-ba97-268578534a8e" + }), + columns); + + Path filePath = Files.createTempFile(filename, ".nc"); + Instant start = Instant.now(); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + // prescription variables: + b.prepare(countries); + b.prepare(healthboards); + b.prepare(prescribing); + + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + try (BufferedReader csvFile = + new BufferedReader( + new FileReader( + Path.of(Objects.requireNonNull(getClass().getResource(hbtResourceName)).toURI()) + .toFile()))) { + // read the health board CSV file + String line; + int[] idbuffer = new int[num_healthboards]; + String[] hbbuffer = new String[num_healthboards]; + String[] hbnamebuffer = new String[num_healthboards]; + int[] hbenactedbuffer = new int[num_healthboards]; + int[] hbarchivedbuffer = new int[num_healthboards]; + String[] countrybuffer = new String[num_healthboards]; + int i = 0; + csvFile.readLine(); // skip the header line + while ((line = csvFile.readLine()) != null) { + String[] values = line.split(COMMA_DELIMITER); + idbuffer[i] = Integer.parseInt(values[0]); + hbbuffer[i] = values[1]; + hbnamebuffer[i] = values[2]; + hbenactedbuffer[i] = Integer.parseInt(values[3]); + try { + if (values[4].length() > 0) hbarchivedbuffer[i] = Integer.parseInt(values[4]); + else hbarchivedbuffer[i] = (int) healthboards.getColumns()[4].getMissingValue(); + } catch (NumberFormatException e) { + logger.error("numberformatexception: '" + values[4] + "'"); + hbarchivedbuffer[i] = (int) healthboards.getColumns()[4].getMissingValue(); + } + countrybuffer[i] = values[5]; + i += 1; + } + w.writeArrayData( + w.getVariable(healthboards.getVariableName(0)), + NetcdfDataType.translateArray(idbuffer)); + w.writeArrayData( + w.getVariable(healthboards.getVariableName(1)), + NetcdfDataType.translateArray(hbbuffer)); + w.writeArrayData( + w.getVariable(healthboards.getVariableName(2)), + NetcdfDataType.translateArray(hbnamebuffer)); + w.writeArrayData( + w.getVariable(healthboards.getVariableName(3)), + NetcdfDataType.translateArray(hbenactedbuffer)); + w.writeArrayData( + w.getVariable(healthboards.getVariableName(4)), + NetcdfDataType.translateArray(hbarchivedbuffer)); + w.writeArrayData( + w.getVariable(healthboards.getVariableName(5)), + NetcdfDataType.translateArray(countrybuffer)); + } + try (BufferedReader csvFile = + new BufferedReader( + new FileReader( + Path.of(Objects.requireNonNull(getClass().getResource(countrycodeName)).toURI()) + .toFile()))) { + // read the country code CSV file; treat it as if it has more than 1 line. + String line; + int[] idbuffer = new int[num_countries]; + String[] countrybuffer = new String[num_countries]; + String[] countrynamebuffer = new String[num_countries]; + csvFile.readLine(); // skip the header line + int i = 0; + while ((line = csvFile.readLine()) != null) { + String[] values = line.split(COMMA_DELIMITER); + idbuffer[i] = Integer.parseInt(values[0]); + countrybuffer[i] = values[1]; + countrynamebuffer[i] = values[2]; + i += 1; + } + w.writeArrayData( + w.getVariable(countries.getVariableName(0)), NetcdfDataType.translateArray(idbuffer)); + w.writeArrayData( + w.getVariable(countries.getVariableName(1)), + NetcdfDataType.translateArray(countrybuffer)); + w.writeArrayData( + w.getVariable(countries.getVariableName(2)), + NetcdfDataType.translateArray(countrynamebuffer)); + } + try (BufferedReader csvFile = + new BufferedReader( + new InputStreamReader( + new BZip2CompressorInputStream( + new FileInputStream( + Path.of( + Objects.requireNonNull( + getClass().getResource(prescribingCsvResourceName)) + .toURI()) + .toFile()), + true)))) { + + String line; + + Variable idVar = w.getVariable(prescribing.getVariableName(0)); + Variable hbtVar = w.getVariable(prescribing.getVariableName(1)); + Variable gpVar = w.getVariable(prescribing.getVariableName(2)); + Variable bnfcodeVar = w.getVariable(prescribing.getVariableName(3)); + Variable bnfdescVar = w.getVariable(prescribing.getVariableName(4)); + Variable numitemsVar = w.getVariable(prescribing.getVariableName(5)); + Variable paidqVar = w.getVariable(prescribing.getVariableName(6)); + Variable grossVar = w.getVariable(prescribing.getVariableName(7)); + Variable paiddatemonthVar = w.getVariable(prescribing.getVariableName(8)); + + int[] origin = new int[] {0}; + int linenum = 0; + int bufindex = 0; + final int bufsize = 50000; + int[] idbuffer = new int[bufsize]; + String[] hbtwritebuffer = new String[bufsize]; + int[] gppracticebuffer = new int[bufsize]; + String[] bnfcodebuffer = new String[bufsize]; + String[] bnfdescbuffer = new String[bufsize]; + int[] numitemsbuffer = new int[bufsize]; + float[] paidquantitybuffer = new float[bufsize]; + float[] grossingredientcostbuffer = new float[bufsize]; + int[] paiddatemonthbuffer = new int[bufsize]; + + csvFile.readLine(); // skip the header line.. + + while ((line = csvFile.readLine()) != null) { + String[] values = line.split(COMMA_DELIMITER); + idbuffer[bufindex] = Integer.parseInt(values[0]); + hbtwritebuffer[bufindex] = values[1]; + gppracticebuffer[bufindex] = Integer.parseInt(values[2]); + bnfcodebuffer[bufindex] = values[3]; + bnfdescbuffer[bufindex] = values[4]; + numitemsbuffer[bufindex] = Integer.parseInt(values[5]); + paidquantitybuffer[bufindex] = Float.parseFloat(values[6]); + grossingredientcostbuffer[bufindex] = Float.parseFloat(values[7]); + paiddatemonthbuffer[bufindex] = Integer.parseInt(values[8]); + bufindex += 1; + if (bufindex == bufsize) { + w.writeArrayData(idVar, NetcdfDataType.translateArray(idbuffer), origin); + w.writeArrayData(hbtVar, NetcdfDataType.translateArray(hbtwritebuffer), origin); + w.writeArrayData(gpVar, NetcdfDataType.translateArray(gppracticebuffer), origin); + w.writeArrayData(bnfcodeVar, NetcdfDataType.translateArray(bnfcodebuffer), origin); + w.writeArrayData(bnfdescVar, NetcdfDataType.translateArray(bnfdescbuffer), origin); + w.writeArrayData(numitemsVar, NetcdfDataType.translateArray(numitemsbuffer), origin); + w.writeArrayData(paidqVar, NetcdfDataType.translateArray(paidquantitybuffer), origin); + w.writeArrayData( + grossVar, NetcdfDataType.translateArray(grossingredientcostbuffer), origin); + w.writeArrayData( + paiddatemonthVar, NetcdfDataType.translateArray(paiddatemonthbuffer), origin); + origin[0] = origin[0] + bufsize; + bufindex = 0; + } + linenum += 1; + if (linenum == 100000 * Math.floor((float) linenum / 100000)) { + logger.trace(String.valueOf(linenum)); + } + } + w.writeArrayData( + idVar, NetcdfDataType.translateArray(Arrays.copyOf(idbuffer, bufindex)), origin); + w.writeArrayData( + hbtVar, + NetcdfDataType.translateArray(Arrays.copyOf(hbtwritebuffer, bufindex)), + origin); + w.writeArrayData( + gpVar, + NetcdfDataType.translateArray(Arrays.copyOf(gppracticebuffer, bufindex)), + origin); + w.writeArrayData( + bnfcodeVar, + NetcdfDataType.translateArray(Arrays.copyOf(bnfcodebuffer, bufindex)), + origin); + w.writeArrayData( + bnfdescVar, + NetcdfDataType.translateArray(Arrays.copyOf(bnfdescbuffer, bufindex)), + origin); + w.writeArrayData( + numitemsVar, + NetcdfDataType.translateArray(Arrays.copyOf(numitemsbuffer, bufindex)), + origin); + w.writeArrayData( + paidqVar, + NetcdfDataType.translateArray(Arrays.copyOf(paidquantitybuffer, bufindex)), + origin); + w.writeArrayData( + grossVar, + NetcdfDataType.translateArray(Arrays.copyOf(grossingredientcostbuffer, bufindex)), + origin); + w.writeArrayData( + paiddatemonthVar, + NetcdfDataType.translateArray(Arrays.copyOf(paiddatemonthbuffer, bufindex)), + origin); + } + + } catch (InvalidRangeException e) { + // + } + } + Instant end = Instant.now(); + logger.trace(Duration.between(start, end).toString()); + start = Instant.now(); + Path bz2file = Path.of(filePath + ".bz2"); + { + InputStream in = Files.newInputStream(filePath); + OutputStream fout = Files.newOutputStream(bz2file); + BufferedOutputStream out = new BufferedOutputStream(fout); + BZip2CompressorOutputStream bzOut = new BZip2CompressorOutputStream(out); + final byte[] buffer = new byte[65536]; + int i = 0; + while (-1 != (i = in.read(buffer))) { + bzOut.write(buffer, 0, i); + } + bzOut.close(); + in.close(); + } + end = Instant.now(); + logger.trace(Duration.between(start, end).toString()); + + Assertions.assertTrue( + FileUtils.contentEquals( + bz2file.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + FileUtils.delete(bz2file.toFile()); + } + + /** testing the prepare/write sequence with 2 arrays. */ + @Test + void test_build_write_two_arrays() throws IOException, URISyntaxException { + String filename = "test_build_write_two_arrays"; + String resourceName = "/netcdf/test_build_write_two_arrays.nc"; + String group1 = "my/group/temps"; + String group2 = "my/othergroup/heights"; + VariableName xName = new VariableName("X", group1); + VariableName yName = new VariableName("Y", group1); + VariableName tempName = new VariableName("temp", group1); + + CoordinateVariableDefinition xdim = + new CoordinateVariableDefinition( + xName, + new int[] {2, 4}, + "the x-axis runs east-west with 0 = south-east corner of my garden", + "cm", + ""); + CoordinateVariableDefinition ydim = + new CoordinateVariableDefinition( + yName, + new int[] {3, 6, 9}, + "the y-axis runs south-north with 0 = south-east corner of my garden", + "cm", + ""); + DimensionalVariableDefinition temperature = + new DimensionalVariableDefinition( + tempName, + NetcdfDataType.INT, + new Dimension[] {new Dimension(xName.getName()), new Dimension(yName.getName())}, + "a test dataset with int temperatures in 2d space, measure in a 2cm grid", + "C", + "surface temperature"); + + VariableName personName = new VariableName("person", group2); + VariableName dateName = new VariableName("date", group2); + VariableName personheightName = new VariableName("personheight", group2); + CoordinateVariableDefinition persondim = + new CoordinateVariableDefinition( + personName, + new String[] {"Bram Boskamp", "Rosalie Boskamp"}, + "the person's name is good enough an identifier for me", + "", + "named person"); + CoordinateVariableDefinition datedim = + new CoordinateVariableDefinition( + dateName, + new int[] {1640995200, 1643673600, 1646092800}, + "the date the measurement was taken", + "seconds since 01-01-1970 00:00:00", + ""); + + DimensionalVariableDefinition heights = + new DimensionalVariableDefinition( + personheightName, + NetcdfDataType.DOUBLE, + new Dimension[] { + new Dimension(personName.getName()), new Dimension(dateName.getName()) + }, + "a test dataset with real height in 2d space, with measurements for each person on a number of dates", + "m", + ""); + + NumericalArray temp_data = new NumericalArrayImpl(new int[][] {{1, 2, 3}, {11, 12, 13}}); + NumericalArray height_data = + new NumericalArrayImpl(new double[][] {{1.832, 1.828, 1.823}, {1.229, 1.232, 1.239}}); + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(xdim); + b.prepare(ydim); + b.prepare(temperature); + b.prepare(persondim); + b.prepare(datedim); + b.prepare(heights); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(xdim); + w.writeDimensionVariable(ydim); + w.writeDimensionVariable(persondim); + w.writeDimensionVariable(datedim); + w.writeArrayData(w.getVariable(tempName), temp_data); + w.writeArrayData(w.getVariable(personheightName), height_data); + } catch (InvalidRangeException e) { + // + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * test using NetcdfWritehandle to write a 3d array in timeslices: an XY 2d set for 1 write each t + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_build_write_in_parts1() throws IOException, URISyntaxException { + String filename = "test_build_write_in_parts1"; + String resourceName = "/netcdf/test_build_write_in_parts.nc"; + + String group = "three/d/intime"; + + VariableName timeName = new VariableName("time", group); + VariableName xName = new VariableName("X", group); + VariableName yName = new VariableName("Y", group); + + CoordinateVariableDefinition timedim = + new CoordinateVariableDefinition( + timeName, + new int[] {1640995200, 1640995201, 1640995202, 1640995203, 1640995204}, + "", + "seconds since 01-01-1970", + ""); + CoordinateVariableDefinition xdim = + new CoordinateVariableDefinition(xName, new int[] {2, 4}, "my x axis", "cm", ""); + CoordinateVariableDefinition ydim = + new CoordinateVariableDefinition(yName, new int[] {3, 6, 9}, "my y axis", "cm", ""); + + VariableName temperatureName = new VariableName("temperature", group); + + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + temperatureName, + NetcdfDataType.INT, + new Dimension[] { + new Dimension(timeName.getName()), + new Dimension(xName.getName()), + new Dimension(yName.getName()) + }, + "a test dataset with temperatures in time and space", + "C", + ""); + + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(timedim); + b.prepare(xdim); + b.prepare(ydim); + b.prepare(nadef); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(timedim); + w.writeDimensionVariable(xdim); + w.writeDimensionVariable(ydim); + Variable v = w.getVariable(temperatureName); + int[][] xyMeasurements = new int[2][3]; + int i = 0; + int[] origin = new int[] {0, 0, 0}; + for (int time = 0; time < 5; time++) { + for (int x = 0; x < 2; x++) for (int y = 0; y < 3; y++) xyMeasurements[x][y] = i++; + Array data = ucar.ma2.Array.makeFromJavaArray(xyMeasurements); + origin[0] = time; + w.writeArrayData(v, Array.makeArrayRankPlusOne(data), origin); + } + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + /** + * test using NetcdfWritehandle to write a TXY 3d array in 1d vectors: write a Y 1d vector set for + * 1 write each TX + * + * @throws IOException + * @throws URISyntaxException + */ + @Test + void test_build_write_in_parts2() throws IOException, URISyntaxException { + String filename = "test_build_write_in_parts2"; + String resourceName = "/netcdf/test_build_write_in_parts.nc"; + String group = "three/d/intime"; + VariableName timeName = new VariableName("time", group); + VariableName xName = new VariableName("X", group); + VariableName yName = new VariableName("Y", group); + + VariableName tempName = new VariableName("temperature", group); + + CoordinateVariableDefinition timedim = + new CoordinateVariableDefinition( + timeName, + new int[] {1640995200, 1640995201, 1640995202, 1640995203, 1640995204}, + "", + "seconds since 01-01-1970", + ""); + CoordinateVariableDefinition xdim = + new CoordinateVariableDefinition(xName, new int[] {2, 4}, "my x axis", "cm", ""); + CoordinateVariableDefinition ydim = + new CoordinateVariableDefinition(yName, new int[] {3, 6, 9}, "my y axis", "cm", ""); + DimensionalVariableDefinition nadef = + new DimensionalVariableDefinition( + tempName, + NetcdfDataType.INT, + new Dimension[] { + new Dimension(timeName.getName()), + new Dimension(xName.getName()), + new Dimension(yName.getName()) + }, + "a test dataset with temperatures in time and space", + "C", + ""); + + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(timedim); + b.prepare(xdim); + b.prepare(ydim); + b.prepare(nadef); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(timedim); + w.writeDimensionVariable(xdim); + w.writeDimensionVariable(ydim); + Variable v = w.getVariable(nadef.getVariableName()); + int[] origin = new int[] {0, 0, 0}; + int[] yMeasurements = new int[3]; + int i = 0; + for (int time = 0; time < 5; time++) + for (int x = 0; x < 2; x++) { + origin[0] = time; + origin[1] = x; + for (int y = 0; y < 3; y++) yMeasurements[y] = i++; + Array data = ucar.ma2.Array.makeFromJavaArray(yMeasurements); + w.writeArrayData( + v, Array.makeArrayRankPlusOne(Array.makeArrayRankPlusOne(data)), origin); + } + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void sharedDimension() throws IOException, URISyntaxException { + String filename = "sharedDimension"; + String resourceName = "/netcdf/sharedDimension.nc"; + + String group_time = "time"; + String group_temp = group_time + "/temp"; + + VariableName timeName = new VariableName("time", group_time); + VariableName xName = new VariableName("X", group_temp); + VariableName yName = new VariableName("Y", group_temp); + VariableName tempName = new VariableName("temp", group_temp); + + CoordinateVariableDefinition time = + new CoordinateVariableDefinition( + timeName, + new int[] {12, 13, 14, 15, 16}, + "this is the time dimension that other arrays should link to", + "seconds (since 01-01-1970)", + ""); + + CoordinateVariableDefinition dim_x = + new CoordinateVariableDefinition(xName, new int[] {2, 4}, "my x axis", "cm", ""); + CoordinateVariableDefinition dim_y = + new CoordinateVariableDefinition(yName, new int[] {3, 6, 9}, "my y axis", "cm", ""); + + DimensionalVariableDefinition nadef_temp = + new DimensionalVariableDefinition( + tempName, + NetcdfDataType.INT, + new Dimension[] { + new Dimension(timeName.getName()), + new Dimension(xName.getName()), + new Dimension(yName.getName()) + }, + "a test dataset with temperatures in time and space", + "C", + "surface temperature"); + + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(time); + b.prepare(dim_x); + b.prepare(dim_y); + b.prepare(nadef_temp); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(time); + w.writeDimensionVariable(dim_x); + w.writeDimensionVariable(dim_y); + Variable v = w.getVariable(nadef_temp.getVariableName()); + int[][] xyMeasurements = new int[2][3]; + int[] origin = new int[] {0, 0, 0}; + int i = 0; + for (int t = 0; t < 5; t++) { + origin[0] = t; + for (int x = 0; x < 2; x++) for (int y = 0; y < 3; y++) xyMeasurements[x][y] = i++; + Array data = ucar.ma2.Array.makeFromJavaArray(xyMeasurements); + w.writeArrayData(v, Array.makeArrayRankPlusOne(data), origin); + } + + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } + + @Test + void inRoot() throws IOException, URISyntaxException { + String filename = "inRoot"; + String resourceName = "/netcdf/inRoot.nc"; + + String root_group = ""; + + VariableName timeName = new VariableName("time", root_group); + VariableName tempName = new VariableName("temp", root_group); + + CoordinateVariableDefinition time = + new CoordinateVariableDefinition( + timeName, + new int[] {12, 13, 14, 15, 16}, + "this is the time dimension that other arrays should link to", + "seconds (since 01-01-1970)", + ""); + + DimensionalVariableDefinition nadef_temp = + new DimensionalVariableDefinition( + tempName, + NetcdfDataType.INT, + new Dimension[] {new Dimension(timeName.getName())}, + "a test dataset with temperatures in time", + "C", + ""); + + Path filePath = Files.createTempFile(filename, ".nc"); + try (NetcdfBuilder b = + new NetcdfBuilder( + filePath.toString(), Nc4Chunking.Strategy.standard, 3, true, this.onClose)) { + b.prepare(time); + b.prepare(nadef_temp); + try (NetcdfWriter w = new NetcdfWriter(b, this.onClose)) { + w.writeDimensionVariable(time); + Variable v = w.getVariable(nadef_temp.getVariableName()); + int[] temp = new int[] {10, 14, 12, 19, 11}; + w.writeArrayData(v, new NumericalArrayImpl(temp)); + + } catch (InvalidRangeException e) { + + } + } + Assertions.assertTrue( + FileUtils.contentEquals( + filePath.toFile(), + Path.of(Objects.requireNonNull(getClass().getResource(resourceName)).toURI()) + .toFile())); + FileUtils.delete(filePath.toFile()); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/netcdf/VariableNameTest.java b/api/src/test/java/org/fairdatapipeline/netcdf/VariableNameTest.java new file mode 100644 index 0000000..5c7ea3f --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/netcdf/VariableNameTest.java @@ -0,0 +1,76 @@ +package org.fairdatapipeline.netcdf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class VariableNameTest { + + @ParameterizedTest + @ValueSource(strings = {"", "/", " ", "_bla", "bla//bla", "bla/", "*bla", "bla\nbla"}) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_bad_full_paths(String fullpath) { + Assertions.assertThrows(IllegalArgumentException.class, () -> new VariableName(fullpath)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "bla/bla", + "bla/191", + "191/191", + "bla/bla/bla", + "/bla/bla", + }) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_good_full_paths(String fullpath) { + Assertions.assertDoesNotThrow(() -> new VariableName(fullpath)); + } + + @ParameterizedTest + @ValueSource(strings = {"bla192", "bla", "123"}) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_good_names(String name) { + Assertions.assertDoesNotThrow(() -> new NetcdfName(name)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "/bla"}) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_bad_names(String name) { + Assertions.assertThrows(IllegalArgumentException.class, () -> new NetcdfName(name)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "bla", "/bla", "/bla/bla"}) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_good_groups(String group) { + Assertions.assertDoesNotThrow(() -> new NetcdfGroupName(group)); + } + + @ParameterizedTest + @ValueSource(strings = {"bla/", "/bla/"}) + /* "" is not a valid fullPath. a valid full path contains at least a variable name. */ + void test_bad_groups(String group) { + Assertions.assertThrows(IllegalArgumentException.class, () -> new NetcdfGroupName(group)); + } + + @Test + /* /bla/bla gets rid of the starting slash; group and name are both bla. */ + void test_a_specific_path1() { + VariableName vn = new VariableName("/bla/bla"); + Assertions.assertEquals("bla", vn.getName().toString()); + Assertions.assertEquals("bla", vn.getGroupName().toString()); + Assertions.assertEquals("bla/bla", vn.getFullPath()); + } + + @Test + /* /bla gets rid of the starting slash; group and name are both bla. */ + void test_a_specific_path2() { + VariableName vn = new VariableName("/bla"); + Assertions.assertEquals("bla", vn.getName().toString()); + Assertions.assertEquals("", vn.getGroupName().toString()); + Assertions.assertEquals("bla", vn.getFullPath()); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/objects/CoordinateVariableDefinitionTest.java b/api/src/test/java/org/fairdatapipeline/objects/CoordinateVariableDefinitionTest.java new file mode 100644 index 0000000..bebbba8 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/objects/CoordinateVariableDefinitionTest.java @@ -0,0 +1,34 @@ +package org.fairdatapipeline.objects; + +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.VariableName; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class CoordinateVariableDefinitionTest { + + @Test + void test_constructor() { + CoordinateVariableDefinition c = + new CoordinateVariableDefinition( + new VariableName("bla", ""), + NetcdfDataType.INT, + CoordinateVariableDefinition.UNLIMITED, + "a little description", + "cm", + "a very long name"); + Assertions.assertTrue(c.isUnlimited()); + } + + @Test + void test_constructor2() { + CoordinateVariableDefinition c = + new CoordinateVariableDefinition( + new VariableName("bla", ""), + new int[] {1, 2, 3}, + "a little description", + "cm", + "a very long name"); + Assertions.assertEquals(3, c.getSize()); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/objects/DimensionalVariableDefinitionTest.java b/api/src/test/java/org/fairdatapipeline/objects/DimensionalVariableDefinitionTest.java new file mode 100644 index 0000000..5af59cc --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/objects/DimensionalVariableDefinitionTest.java @@ -0,0 +1,31 @@ +package org.fairdatapipeline.objects; + +import java.util.Collections; +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.VariableName; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DimensionalVariableDefinitionTest { + + /** + * we can only reference a coordinatevariable/dimension existing in our current group or one of + * its parent groups. + */ + @Test + void test_prepare_dimensionalvar() { + String dimname = "name"; + String varname = "varname"; + DimensionalVariableDefinition dv = + new DimensionalVariableDefinition( + new VariableName(varname, ""), + NetcdfDataType.INT, + new Dimension[] {new Dimension(dimname)}, + "", + "", + "", + Collections.singletonMap("optional_attrib", new String[] {"value1", "value2"})); + Assertions.assertEquals(dimname, dv.getDimensions()[0].name().getName()); + Assertions.assertEquals("value1", dv.getOptional_attribs().get("optional_attrib")[0]); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/objects/NumericalArrayTest.java b/api/src/test/java/org/fairdatapipeline/objects/NumericalArrayTest.java new file mode 100644 index 0000000..2af1500 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/objects/NumericalArrayTest.java @@ -0,0 +1,119 @@ +package org.fairdatapipeline.objects; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import org.fairdatapipeline.api.IllegalActionException; +import org.junit.jupiter.api.*; + +class NumericalArrayTest { + + @Test + void t1dArray() { + Number[] n = new Number[] {1, 2, 3}; + NumericalArray na = new NumericalArrayImpl(n); + + Assertions.assertEquals(n, na.as1DArray()); + } + + @Test + void t2dArray() { + Number[][] n = new Number[][] {{1, 2, 3}, {4, 5, 6}}; + Number[][] r; + NumericalArray na = new NumericalArrayImpl(n); + r = na.as2DArray(); + Assertions.assertEquals(n, r); + } + + private ArrayList checkShape(Object nDArray, boolean deepCheck) { + ArrayList dims = new ArrayList<>(); + int l = Array.getLength(nDArray); + dims.add(l); + if (l == 0) return dims; + Object o = Array.get(nDArray, 0); + if (o.getClass().isArray()) { + ArrayList subshape = checkShape(o, deepCheck); + if (deepCheck) { + for (int i = 1; i < Array.getLength(nDArray); i++) { + if (!subshape.equals(checkShape(Array.get(nDArray, i), deepCheck))) { + throw (new IllegalActionException("not a proper array")); + } + } + } + dims.addAll(subshape); + } + return dims; + } + + void setNumbers(Object old_with_primitives, Object new_with_numbers, int[] dimensions) { + for (int i = 0; i < dimensions[0]; i++) { + if (dimensions.length == 1) { + ((Number[]) new_with_numbers)[i] = (Number) Array.get(old_with_primitives, i); + } else { + setNumbers( + Array.get(old_with_primitives, i), + Array.get(new_with_numbers, i), + Arrays.stream(dimensions).skip(1).toArray()); + } + } + } + + Object unprim(Object o) { + if (!o.getClass().isArray()) + throw (new IllegalArgumentException("unprim must work on an array")); + int[] dimensions = checkShape(o, false).stream().mapToInt(i -> i).toArray(); + Object new_array = Array.newInstance(Number.class, dimensions); + setNumbers(o, new_array, dimensions); + return new_array; + } + + @Test + void test_unprimitivize5() { + Object o = new int[][][] {{{1, 2}, {3, 4}}, {{4, 5}, {6, 7}}}; + System.out.println(Arrays.deepToString((int[][][]) o)); + Object oo = unprim(o); + System.out.println(Arrays.deepToString((Number[][][]) oo)); + } + + @Test + void check_type() { + int i = 1; + Object oo = (Object) i; + System.out.println(oo.getClass()); + Object o = new int[][] {{1, 2, 3}, {4, 5, 6}}; + + NumericalArray na = new NumericalArrayImpl(o); + na.as2DArray(); + + o = new int[] {1, 2, 3, 4, 5}; + new NumericalArrayImpl(o); + + o = new double[] {1.1, 2.2, 3.3, 4.4, 5.5}; + new NumericalArrayImpl(o); + + o = new Integer[] {1, 2, 3, 4, 5}; + new NumericalArrayImpl(o); + + Assertions.assertThrows( + IllegalActionException.class, + () -> { + new NumericalArrayImpl(new Integer[][] {{1, 2, 3}, {4, 5}}); + }); + + Assertions.assertThrows( + IllegalActionException.class, + () -> { + new NumericalArrayImpl(new Integer[][][] {{{1, 2}, {3, 4}}, {{5, 6}, {7, 8, 9}}}); + }); + + Assertions.assertThrows( + IllegalActionException.class, + () -> { + new NumericalArrayImpl(new Integer[][][] {{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}, {9, 10}}}); + }); + + // int[] o2 = new int[] {1,2,3,4,5,6}; + // check_type2(o2); + + } +} diff --git a/api/src/test/java/org/fairdatapipeline/objects/TableTest.java b/api/src/test/java/org/fairdatapipeline/objects/TableTest.java new file mode 100644 index 0000000..abdfdbd --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/objects/TableTest.java @@ -0,0 +1,36 @@ +package org.fairdatapipeline.objects; + +import java.util.Collections; +import org.fairdatapipeline.netcdf.NetcdfDataType; +import org.fairdatapipeline.netcdf.NetcdfGroupName; +import org.fairdatapipeline.netcdf.NetcdfName; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TableTest { + @Test + void table() { + LocalVariableDefinition[] columns = + new LocalVariableDefinition[] { + new LocalVariableDefinition( + new NetcdfName("bla"), NetcdfDataType.STRING, "item", "", "", Collections.emptyMap()), + new LocalVariableDefinition( + new NetcdfName("price"), + NetcdfDataType.FLOAT, + "price", + "GBP", + "the price per item", + Collections.emptyMap()) + }; + + TableDefinition td = + new TableDefinition( + new NetcdfGroupName("my/group"), + 0, + "A test table", + "Bram's beautiful test table", + Collections.emptyMap(), + columns); + Assertions.assertEquals("A test table", td.getDescription()); + } +} diff --git a/api/src/test/resources/config-stdapi.yaml b/api/src/test/resources/config-stdapi.yaml index bcddb5f..310bd98 100644 --- a/api/src/test/resources/config-stdapi.yaml +++ b/api/src/test/resources/config-stdapi.yaml @@ -48,6 +48,61 @@ read: - data_product: animal/chicken use: version: 0.0.1 + - data_product: test/array1 + use: + version: 0.0.1 + - data_product: test/array3d + use: + version: 0.0.1 + - data_product: test/array2a + use: + version: 0.0.1 + - data_product: test/array2b + use: + version: 0.0.1 + - data_product: test/array2c + use: + version: 0.0.1 + - data_product: test/array2d + use: + version: 0.0.1 + - data_product: test/array2e + use: + version: 0.0.1 + - data_product: test/array4d_a + use: + version: 0.0.1 + - data_product: test/array4d_b + use: + version: 0.0.1 + - data_product: test/array4d_c + use: + version: 0.0.1 + - data_product: test/array4d_d + use: + version: 0.0.1 + - data_product: test/array4d_e + use: + version: 0.0.1 + - data_product: test/array4d_f + use: + version: 0.0.1 + - data_product: my_lovely_little_table + use: + version: 0.0.1 + - data_product: my_lovely_little_table2 + use: + version: 0.0.1 + - data_product: my_lovely_little_table3 + use: + version: 0.0.1 + - data_product: my_lovely_little_table4 + use: + version: 0.0.1 + - data_product: my_lovely_little_table5 + use: + version: 0.0.1 + write: - data_product: human/population @@ -89,5 +144,93 @@ write: file_type: txt use: version: 0.0.1 - - + - data_product: test/array1 + description: the first NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array3d + description: the 3d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array2a + description: the second NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array2b + description: the second NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array2c + description: the second NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array2d + description: the second NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array2e + description: the second NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_a + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_b + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_c + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_d + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_e + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: test/array4d_f + description: the 4d NC array test + file_type: nc + use: + version: 0.0.1 + - data_product: my_lovely_little_table + description: table test + file_type: nc + use: + version: 0.0.1 + - data_product: my_lovely_little_table2 + description: table test + file_type: nc + use: + version: 0.0.1 + - data_product: my_lovely_little_table3 + description: table test + file_type: nc + use: + version: 0.0.1 + - data_product: my_lovely_little_table4 + description: table test + file_type: nc + use: + version: 0.0.1 + - data_product: my_lovely_little_table5 + description: table test + file_type: nc + use: + version: 0.0.1 diff --git a/api/src/test/resources/country code.csv b/api/src/test/resources/country code.csv new file mode 100644 index 0000000..88d473d --- /dev/null +++ b/api/src/test/resources/country code.csv @@ -0,0 +1,2 @@ +_id,Country,CountryName +1,S92000003,Scotland diff --git a/api/src/test/resources/health boards.csv b/api/src/test/resources/health boards.csv new file mode 100644 index 0000000..af366d4 --- /dev/null +++ b/api/src/test/resources/health boards.csv @@ -0,0 +1,19 @@ +_id,HB,HBName,HBDateEnacted,HBDateArchived,Country +1,S08000015,NHS Ayrshire and Arran,20140401,,S92000003 +2,S08000016,NHS Borders,20140401,,S92000003 +3,S08000017,NHS Dumfries and Galloway,20140401,,S92000003 +4,S08000018,NHS Fife,20140401,20180201,S92000003 +5,S08000019,NHS Forth Valley,20140401,,S92000003 +6,S08000020,NHS Grampian,20140401,,S92000003 +7,S08000021,NHS Greater Glasgow and Clyde,20140401,20190331,S92000003 +8,S08000022,NHS Highland,20140401,,S92000003 +9,S08000023,NHS Lanarkshire,20140401,20190331,S92000003 +10,S08000024,NHS Lothian,20140401,,S92000003 +11,S08000025,NHS Orkney,20140401,,S92000003 +12,S08000026,NHS Shetland,20140401,,S92000003 +13,S08000027,NHS Tayside,20140401,20180201,S92000003 +14,S08000028,NHS Western Isles,20140401,,S92000003 +15,S08000029,NHS Fife,20180202,,S92000003 +16,S08000030,NHS Tayside,20180202,,S92000003 +17,S08000031,NHS Greater Glasgow and Clyde,20190401,,S92000003 +18,S08000032,NHS Lanarkshire,20190401,,S92000003 diff --git a/api/src/test/resources/netcdf/inRoot.nc b/api/src/test/resources/netcdf/inRoot.nc new file mode 100644 index 0000000..69fa75d Binary files /dev/null and b/api/src/test/resources/netcdf/inRoot.nc differ diff --git a/api/src/test/resources/netcdf/sharedDimension.nc b/api/src/test/resources/netcdf/sharedDimension.nc new file mode 100644 index 0000000..9c8b129 Binary files /dev/null and b/api/src/test/resources/netcdf/sharedDimension.nc differ diff --git a/api/src/test/resources/netcdf/test_add_dimension_after_dimensionalvariable.nc b/api/src/test/resources/netcdf/test_add_dimension_after_dimensionalvariable.nc new file mode 100644 index 0000000..3cddad0 Binary files /dev/null and b/api/src/test/resources/netcdf/test_add_dimension_after_dimensionalvariable.nc differ diff --git a/api/src/test/resources/netcdf/test_build_prepare_write_INT.nc b/api/src/test/resources/netcdf/test_build_prepare_write_INT.nc new file mode 100644 index 0000000..7d2e633 Binary files /dev/null and b/api/src/test/resources/netcdf/test_build_prepare_write_INT.nc differ diff --git a/api/src/test/resources/netcdf/test_build_prepare_write_with_fillvalue.nc b/api/src/test/resources/netcdf/test_build_prepare_write_with_fillvalue.nc new file mode 100644 index 0000000..9c5c76f Binary files /dev/null and b/api/src/test/resources/netcdf/test_build_prepare_write_with_fillvalue.nc differ diff --git a/api/src/test/resources/netcdf/test_build_write_in_parts.nc b/api/src/test/resources/netcdf/test_build_write_in_parts.nc new file mode 100644 index 0000000..e4a7a5a Binary files /dev/null and b/api/src/test/resources/netcdf/test_build_write_in_parts.nc differ diff --git a/api/src/test/resources/netcdf/test_build_write_two_arrays.nc b/api/src/test/resources/netcdf/test_build_write_two_arrays.nc new file mode 100644 index 0000000..4fff0b8 Binary files /dev/null and b/api/src/test/resources/netcdf/test_build_write_two_arrays.nc differ diff --git a/api/src/test/resources/netcdf/test_create_2groups_with_dims.nc b/api/src/test/resources/netcdf/test_create_2groups_with_dims.nc new file mode 100644 index 0000000..fde17ba Binary files /dev/null and b/api/src/test/resources/netcdf/test_create_2groups_with_dims.nc differ diff --git a/api/src/test/resources/netcdf/test_create_group_with_dim.nc b/api/src/test/resources/netcdf/test_create_group_with_dim.nc new file mode 100644 index 0000000..8a45195 Binary files /dev/null and b/api/src/test/resources/netcdf/test_create_group_with_dim.nc differ diff --git a/api/src/test/resources/netcdf/test_create_table_group.nc b/api/src/test/resources/netcdf/test_create_table_group.nc new file mode 100644 index 0000000..53897ef Binary files /dev/null and b/api/src/test/resources/netcdf/test_create_table_group.nc differ diff --git a/api/src/test/resources/netcdf/test_linked_from_multi_attribute.nc b/api/src/test/resources/netcdf/test_linked_from_multi_attribute.nc new file mode 100644 index 0000000..db23427 Binary files /dev/null and b/api/src/test/resources/netcdf/test_linked_from_multi_attribute.nc differ diff --git a/api/src/test/resources/netcdf/test_prepare_coordinatevar.nc b/api/src/test/resources/netcdf/test_prepare_coordinatevar.nc new file mode 100644 index 0000000..7a5c135 Binary files /dev/null and b/api/src/test/resources/netcdf/test_prepare_coordinatevar.nc differ diff --git a/api/src/test/resources/netcdf/test_prepare_coordinatevars.nc b/api/src/test/resources/netcdf/test_prepare_coordinatevars.nc new file mode 100644 index 0000000..ee1654a Binary files /dev/null and b/api/src/test/resources/netcdf/test_prepare_coordinatevars.nc differ diff --git a/api/src/test/resources/netcdf/test_prepare_dimensionalvar.nc b/api/src/test/resources/netcdf/test_prepare_dimensionalvar.nc new file mode 100644 index 0000000..5644bce Binary files /dev/null and b/api/src/test/resources/netcdf/test_prepare_dimensionalvar.nc differ diff --git a/api/src/test/resources/netcdf/test_prepare_dimensionalvar_remotedim.nc b/api/src/test/resources/netcdf/test_prepare_dimensionalvar_remotedim.nc new file mode 100644 index 0000000..ae485d6 Binary files /dev/null and b/api/src/test/resources/netcdf/test_prepare_dimensionalvar_remotedim.nc differ diff --git a/api/src/test/resources/netcdf/test_prepare_scalar_dimensionalvar.nc b/api/src/test/resources/netcdf/test_prepare_scalar_dimensionalvar.nc new file mode 100644 index 0000000..d6e7137 Binary files /dev/null and b/api/src/test/resources/netcdf/test_prepare_scalar_dimensionalvar.nc differ diff --git a/api/src/test/resources/netcdf/test_prescribing_data.nc.bz2 b/api/src/test/resources/netcdf/test_prescribing_data.nc.bz2 new file mode 100644 index 0000000..c52f842 Binary files /dev/null and b/api/src/test/resources/netcdf/test_prescribing_data.nc.bz2 differ diff --git a/api/src/test/resources/netcdf/test_table.nc b/api/src/test/resources/netcdf/test_table.nc new file mode 100644 index 0000000..62e3816 Binary files /dev/null and b/api/src/test/resources/netcdf/test_table.nc differ diff --git a/api/src/test/resources/netcdf/test_table_with_missing.nc b/api/src/test/resources/netcdf/test_table_with_missing.nc new file mode 100644 index 0000000..306ef95 Binary files /dev/null and b/api/src/test/resources/netcdf/test_table_with_missing.nc differ diff --git a/api/src/test/resources/prescribing_june2016.csv.bz2 b/api/src/test/resources/prescribing_june2016.csv.bz2 new file mode 100644 index 0000000..5aa2bb1 Binary files /dev/null and b/api/src/test/resources/prescribing_june2016.csv.bz2 differ diff --git a/api/src/test/resources/simplelogger.properties b/api/src/test/resources/simplelogger.properties new file mode 100644 index 0000000..64fb3b6 --- /dev/null +++ b/api/src/test/resources/simplelogger.properties @@ -0,0 +1,34 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=trace + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +#org.slf4j.simpleLogger.showThreadName=true + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +#org.slf4j.simpleLogger.showShortLogName=false \ No newline at end of file diff --git a/dataregistry/build.gradle b/dataregistry/build.gradle index 7136769..bc20dd9 100644 --- a/dataregistry/build.gradle +++ b/dataregistry/build.gradle @@ -121,7 +121,7 @@ publishing { } signing { - if (project.hasProperty('signing.keyId') && project.hasProperty('signing.secretKeyRingFile')) { + if (project.hasProperty('signing.keyId') && project.hasProperty('signing.secretKeyRingFile') && project.hasProperty('signing.password')) { sign publishing.publications.mavenJava } } diff --git a/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/Registry_ObjectListReader.java b/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/Registry_ObjectListReader.java index 22a122c..dfa0efa 100644 --- a/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/Registry_ObjectListReader.java +++ b/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/Registry_ObjectListReader.java @@ -38,10 +38,10 @@ public Registry_ObjectList readFrom( try { ObjectMapper om = new ObjectMapper(); om.registerModule(new JavaTimeModule()); - Type MyTypeParameter = ((ParameterizedType) genericType).getActualTypeArguments()[0]; + Type myTypeParameter = ((ParameterizedType) genericType).getActualTypeArguments()[0]; JavaType ptype = om.getTypeFactory() - .constructParametricType(Registry_ObjectList.class, (Class) MyTypeParameter); + .constructParametricType(Registry_ObjectList.class, (Class) myTypeParameter); return om.readValue(entityStream, ptype); } catch (Exception e) { throw new ProcessingException("Error deserializing Registry_ObjectList", e); diff --git a/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/RestClient.java b/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/RestClient.java index ac3d407..e0bafc0 100644 --- a/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/RestClient.java +++ b/dataregistry/src/main/java/org/fairdatapipeline/dataregistry/restclient/RestClient.java @@ -27,7 +27,7 @@ /** Jakarta WS client implementation for interacting with the FAIR Data Registry */ public class RestClient { - private static final Logger logger = LoggerFactory.getLogger(RestClient.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RestClient.class); private WebTarget wt; private Client client; private final MediaType jsonWithVersion = @@ -83,7 +83,7 @@ public Registry_RootObject getFirst(Class c, Map< try { Registry_ObjectList o = wt2.request(this.getJsonMediaType()).get(new GenericType<>(p)); if (o.getCount() == 0) { - logger.trace("getFirst({}, {}) returned 0 items", c.getSimpleName(), m); + LOGGER.trace("getFirst({}, {}) returned 0 items", c.getSimpleName(), m); return null; } return o.getResults().get(0); @@ -159,7 +159,7 @@ public Registry_RootObject get(Class c, int i) try { return wt2.request(this.getJsonMediaType()).get(c); } catch (NotFoundException e) { - logger.warn("get(Class, Int)", e); + LOGGER.warn("get(Class, Int)", e); return null; } catch (Exception e) { deal_with_jakarta_http_exceptions(e); @@ -181,7 +181,7 @@ public Registry_RootObject get(Class c, APIURL ap try { return wt2.request(this.getJsonMediaType()).get(c); } catch (NotFoundException e) { - logger.warn("get(Class, APIURL)", e); + LOGGER.warn("get(Class, APIURL)", e); return null; } catch (Exception e) { deal_with_jakarta_http_exceptions(e); @@ -219,9 +219,9 @@ public Registry_Updateable post(Registry_Updateable o) InputStream i = (InputStream) r.getEntity(); try { String text = IOUtils.toString(i, StandardCharsets.UTF_8.name()); - logger.error("post(Registry_Updateable) -- error: {}", text); + LOGGER.error("post(Registry_Updateable) -- error: {}", text); } catch (IOException e) { - logger.error("post(Registry_Updateable) -- IOException trying to read response entity.", e); + LOGGER.error("post(Registry_Updateable) -- IOException trying to read response entity.", e); } return null; } else { @@ -276,9 +276,9 @@ public Registry_Updateable patch(Registry_Updateable o) InputStream i = (InputStream) r.getEntity(); try { String text = IOUtils.toString(i, StandardCharsets.UTF_8.name()); - logger.error("patch(Registry_Updateable) -- error: {}", text); + LOGGER.error("patch(Registry_Updateable) -- error: {}", text); } catch (IOException e) { - logger.error("patch(Registry_Updateable) -- IOException trying to read response entity.", e); + LOGGER.error("patch(Registry_Updateable) -- IOException trying to read response entity.", e); } return null; } @@ -322,9 +322,9 @@ public Registry_Updateable put(Registry_Updateable o) InputStream i = (InputStream) r.getEntity(); try { String text = IOUtils.toString(i, StandardCharsets.UTF_8.name()); - logger.error("put(Registry_Updateable) -- error: {}", text); + LOGGER.error("put(Registry_Updateable) -- error: {}", text); } catch (IOException e) { - logger.error("put(Registry_Updateable) -- IOException trying to read response entity.", e); + LOGGER.error("put(Registry_Updateable) -- IOException trying to read response entity.", e); } return null; } @@ -385,7 +385,7 @@ public APIURL makeAPIURL(Class c, int i) { try { return new APIURL(wt2.getUri() + "/"); } catch (URISyntaxException e) { - logger.error(e.toString()); + LOGGER.error(e.toString()); return null; } } diff --git a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/restClientTest.java b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/RestClientTest.java similarity index 99% rename from dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/restClientTest.java rename to dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/RestClientTest.java index 6b3764b..5613b4b 100644 --- a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/restClientTest.java +++ b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/RestClientTest.java @@ -24,7 +24,7 @@ @EnabledIfEnvironmentVariable(named = "LOCALREG", matches = "FRESHASADAISY") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class restClientTest { +class RestClientTest { String localReg = "http://localhost:8000/api/"; String badReg = "http://localhost:1234/nothere/"; RestClient lc; diff --git a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/testPatchAndDelete.java b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestPatchAndDelete.java similarity index 99% rename from dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/testPatchAndDelete.java rename to dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestPatchAndDelete.java index 46c807b..ff8c01f 100644 --- a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/testPatchAndDelete.java +++ b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestPatchAndDelete.java @@ -15,7 +15,7 @@ @EnabledIfEnvironmentVariable(named = "LOCALREG", matches = "FRESHASADAISY") @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class testPatchAndDelete { +class TestPatchAndDelete { String localReg = "http://localhost:8000/api/"; RestClient lc; RegistryCode_run cr; diff --git a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/test_Registry_RootObjectWriter.java b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestRegistry_RootObjectWriter.java similarity index 97% rename from dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/test_Registry_RootObjectWriter.java rename to dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestRegistry_RootObjectWriter.java index e18a179..f22f167 100644 --- a/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/test_Registry_RootObjectWriter.java +++ b/dataregistry/src/test/java/org/fairdatapipeline/dataregistry/restclient/TestRegistry_RootObjectWriter.java @@ -7,7 +7,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -class test_Registry_RootObjectWriter { +class TestRegistry_RootObjectWriter { public String write_json(Registry_RootObject o, Class type) { ByteArrayOutputStream ba = new ByteArrayOutputStream();