diff --git a/cadc-dali-parquet/build.gradle b/cadc-dali-parquet/build.gradle index 8436fecc..94bf1b12 100644 --- a/cadc-dali-parquet/build.gradle +++ b/cadc-dali-parquet/build.gradle @@ -15,18 +15,18 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '0.5.5' +version = '0.5.6' description = 'OpenCADC VO-parquet library' def git_url = 'https://github.com/opencadc/dal' dependencies { implementation 'org.opencadc:cadc-util:[1.12.10,)' - implementation 'org.opencadc:cadc-dali:[1.2.29,)' + implementation 'org.opencadc:cadc-dali:[1.2.30,)' - implementation 'org.apache.hadoop:hadoop-common:[3.3.6,)' + implementation 'org.apache.hadoop:hadoop-common:[3.3.6,3.5)' implementation 'org.apache.parquet:parquet-hadoop:[1.13.1,)' - implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:[3.3.6,)' + implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:[3.3.6,3.5)' // CVE-2025-52999 runtimeOnly 'com.fasterxml.jackson.core:jackson-core:[2.15,)' @@ -35,6 +35,21 @@ dependencies { testImplementation 'xerces:xercesImpl:[2.0,3.0)' } +// HACK: needed because of guava variants +configurations.all { + def envAttr = Attribute.of("org.gradle.jvm.environment", String) + + try { + // Try the Gradle 7/8 way (using the actual Class) + def targetJvmClass = Class.forName("org.gradle.api.attributes.java.TargetJvmEnvironment") + def jvmAttr = Attribute.of("org.gradle.jvm.environment", targetJvmClass) + attributes.attribute(jvmAttr, objects.named(targetJvmClass, "standard-jvm")) + } catch (ClassNotFoundException e) { + // Fallback for Gradle 6 (using the String) + attributes.attribute(envAttr, "standard-jvm") + } +} + // end applications could also add the following exclude // as all these libs are not needed configurations { diff --git a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java index 5cc1c6c6..6d59d816 100644 --- a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java +++ b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java @@ -69,8 +69,7 @@ package ca.nrc.cadc.dali.tables.parquet; -import static ca.nrc.cadc.dali.tables.parquet.ParquetWriter.IVOA_VOTABLE_PARQUET_CONTENT_KEY; - +import ca.nrc.cadc.dali.tables.TableReader; import ca.nrc.cadc.dali.tables.parquet.io.RandomSeekableInputFile; import ca.nrc.cadc.dali.tables.parquet.readerhelper.ParquetTableData; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; @@ -78,7 +77,6 @@ import ca.nrc.cadc.dali.tables.votable.VOTableReader; import ca.nrc.cadc.dali.tables.votable.VOTableResource; import ca.nrc.cadc.dali.tables.votable.VOTableTable; - import ca.nrc.cadc.dali.util.ByteArrayFormat; import ca.nrc.cadc.dali.util.ByteFormat; import ca.nrc.cadc.dali.util.DoubleArrayFormat; @@ -97,17 +95,14 @@ import ca.nrc.cadc.io.MultiBufferIO; import ca.nrc.cadc.io.RandomAccessFile; import ca.nrc.cadc.io.RandomAccessSource; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.apache.log4j.Logger; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -121,7 +116,7 @@ * Parquet Reader - Reads parquet content and produces a VOTableDocument representation. * */ -public class ParquetReader { +public class ParquetReader implements TableReader { private static final Logger log = Logger.getLogger(ParquetReader.class); @@ -202,7 +197,7 @@ private VOTableDocument readEmptyVOTable(RandomSeekableInputFile inputFile) thro ParquetMetadata metadata = reader.getFooter(); parquetSchema = metadata.getFileMetaData().getSchema(); - String votable = metadata.getFileMetaData().getKeyValueMetaData().get(IVOA_VOTABLE_PARQUET_CONTENT_KEY); + String votable = metadata.getFileMetaData().getKeyValueMetaData().get(ParquetWriter.IVOA_VOTABLE_PARQUET_CONTENT_KEY); voTableDocument = getVOTableDocument(votable, parquetSchema); votableFields = voTableDocument.getResourceByType("results").getTable().getFields(); diff --git a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java index 365f85a5..04ad8125 100644 --- a/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java +++ b/cadc-dali-parquet/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java @@ -89,7 +89,6 @@ import ca.nrc.cadc.dali.util.URIFormat; import ca.nrc.cadc.dali.util.UUIDFormat; import ca.nrc.cadc.util.HexUtil; - import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; @@ -105,7 +104,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; import org.apache.parquet.hadoop.ParquetFileWriter; diff --git a/cadc-dali/build.gradle b/cadc-dali/build.gradle index 675f02d7..e2b3b693 100644 --- a/cadc-dali/build.gradle +++ b/cadc-dali/build.gradle @@ -15,7 +15,7 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '1.2.29' +version = '1.2.30' description = 'OpenCADC DALI library' def git_url = 'https://github.com/opencadc/dal' diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java new file mode 100644 index 00000000..b84d3c2c --- /dev/null +++ b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/TableReader.java @@ -0,0 +1,90 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2025. (c) 2025. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package ca.nrc.cadc.dali.tables; + +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public interface TableReader { + + /** + * Read a table from an InputStream and build a VOTable object. + * + * @param istream InputStream to read from. + * @return a VOTable object. + * @throws IOException is problem reading the InputStream. + */ + VOTableDocument read(InputStream istream) throws IOException; +} diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java index 8ea8a495..9e0d81aa 100644 --- a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java +++ b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/votable/VOTableReader.java @@ -69,12 +69,11 @@ package ca.nrc.cadc.dali.tables.votable; -import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY; -import static ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType.BINARY2; - import ca.nrc.cadc.dali.tables.BinaryTableData; import ca.nrc.cadc.dali.tables.ListTableData; import ca.nrc.cadc.dali.tables.TableData; +import ca.nrc.cadc.dali.tables.TableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableWriter.SerializationType; import ca.nrc.cadc.dali.util.Format; import ca.nrc.cadc.dali.util.FormatFactory; import ca.nrc.cadc.util.StringUtil; @@ -106,7 +105,7 @@ * * @author pdowler */ -public class VOTableReader { +public class VOTableReader implements TableReader { private static final Logger log = Logger.getLogger(VOTableReader.class); @@ -206,6 +205,7 @@ public VOTableDocument read(String xml) * @return a VOTable object. * @throws IOException is problem reading the InputStream. */ + @Override public VOTableDocument read(InputStream istream) throws IOException { Reader reader = new BufferedReader(new InputStreamReader(istream, "UTF-8")); @@ -341,10 +341,11 @@ protected VOTableDocument readImpl(Reader reader) // TODO: check for href in which case encoding may be irrelevant? final String encoding = streamData.getAttributeValue("encoding", VOTableReader.DEFAULT_STREAM_ENCODING); - if (binaryData.getName().equals(BINARY.name()) || binaryData.getName().equals(BINARY2.name())) { + if (binaryData.getName().equals(SerializationType.BINARY.name()) + || binaryData.getName().equals(SerializationType.BINARY2.name())) { vot.setTableData(new BinaryTableData( new ByteArrayInputStream(streamData.getText().getBytes(StandardCharsets.UTF_8)), - vot.getFields(), encoding, formatFactory, binaryData.getName().equals(BINARY2.name()))); + vot.getFields(), encoding, formatFactory, binaryData.getName().equals(SerializationType.BINARY2.name()))); } else { throw new UnsupportedOperationException("Unsupported type: " + binaryData.getName()); } diff --git a/dap/build.gradle b/dap/build.gradle index 600502d9..3687b0d3 100644 --- a/dap/build.gradle +++ b/dap/build.gradle @@ -29,6 +29,7 @@ dependencies { implementation 'org.opencadc:cadc-util:[1.6.1,)' implementation 'org.opencadc:cadc-rest:[1.4.5,)' + implementation 'org.opencadc:cadc-registry:[1.9,)' implementation 'org.opencadc:cadc-cdp:[1.2.3,)' implementation 'org.opencadc:cadc-uws:[1.0.5,)' implementation 'org.opencadc:cadc-uws-server:[1.2.4,)'