/****************************************************************************** * * Copyright (C) 2010 by * * CGTD ICARE * Université des Sciences et Technologies de Lille 1 * Polytech'Lille - Bâtiment Eydoux - Bureau E207 * Avenue Paul Langevin * 59650 Villeneuve d'Ascq Cedex * France * * Description : * ------------- * * Implementation of HDFCompressor class * * Author : * -------- * * CGTD-ICARE/UDEV Nicolas PASCAL * * License : * --------- * * This file must be used under the terms of the CeCILL. * This source file is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at * http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt * *****************************************************************************/ #include "HDFCompressor.h" HDFCompressor::HDFCompressor ( const string & infile, const string & outfile, const int comp_level ) : infile ( infile ), outfile ( outfile ), comp_level ( comp_level ) { sd_id_src = FAIL; sd_id_dest = FAIL; // validate inputs if ( comp_level < 0 || comp_level > 9 ) { string msg ( "Invalid compression level " + to_string ( comp_level ) + ". Must be in range [0-9]" ) ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } } ostream &operator<< ( ostream &stream, const HDFCompressor &self ) { stream << "--- HDFCompressor ---" << endl; stream << "infile = " << self.infile << endl; stream << "outfile = " << self.outfile << endl; stream << "comp_level = " << self.comp_level << endl << endl; return stream; } void HDFCompressor::run() { // open input/output hdf files init_sd_interface(); // duplicate file attributes copy_file_attr ( sd_id_src, sd_id_dest ); // duplicate and compress variables data copy_vars_data ( sd_id_src, sd_id_dest ); // duplicate variables attributes copy_vars_attr ( sd_id_src, sd_id_dest ); // close input/output hdf files close_sd_interface(); } void HDFCompressor::init_sd_interface() { string msg (""); // open input file for reading sd_id_src = SDstart ( infile.c_str(), DFACC_READ ); if ( sd_id_src == FAIL ) { msg = "SDstart Error : fail to open source file " + infile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // open output file for writing sd_id_dest = SDstart ( outfile.c_str(), DFACC_CREATE ); if ( sd_id_dest == FAIL ) { msg = "SDstart Error : fail to open destination file " + outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } } void HDFCompressor::close_sd_interface() { string msg (""); int32 status = FAIL; bool error = false; // close input file for reading if ( sd_id_src != FAIL ) { status = SDend ( sd_id_src ); if ( status == FAIL ) { msg = "SDend Error : fail to close source file " + infile + "\n" ; error = true; } } sd_id_src = FAIL; // open output file for writing if ( sd_id_dest != FAIL ) { status = SDend ( sd_id_dest ); if ( sd_id_dest == FAIL ) { msg += "SDend Error : fail to close destination file " + outfile ; error = true; } } sd_id_dest = FAIL; // if an error occurs, throws an exception if ( error ) { g_exception e( __FILE__ , __LINE__ , msg ); throw e; } } void HDFCompressor::copy_file_attr ( const int32 sd_id_src, const int32 sd_id_dest ) { int32 n_datasets, n_file_attrs; intn status = FAIL; string msg ( "" ); status = SDfileinfo ( sd_id_src, &n_datasets, &n_file_attrs ); if ( status == FAIL ) { msg = "SDfileinfo Error : fail to read informations of file " + this->outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } copy_attrs ( sd_id_src, sd_id_dest, n_file_attrs ); } void HDFCompressor::copy_attrs ( const int32 obj_id_src, const int32 obj_id_dest, const int32 n_attrs ) { // --- crosses src attributes and copy them to dest for ( int32 i_attr = 0 ; i_attr < n_attrs ; ++i_attr ) { copy_attr ( obj_id_src, obj_id_dest, i_attr ); } } void HDFCompressor::copy_attr(const int32 obj_id_src, const int32 obj_id_dest, const int32 i_attr) { int32 status; int32 data_type; int32 n_values; char attr_name [ MAX_NC_NAME ]; VOIDP attr_data = NULL; string msg ( "" ); // read source attr name status = SDattrinfo ( obj_id_src, i_attr, attr_name, &data_type, &n_values ); if ( status == FAIL ) { msg = "SDattrinfo error : can't retrieve infos of attribute " + to_string( i_attr ) + "\n"; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // read source attr data int16 sz_type = get_data_type_size ( data_type ); if ( sz_type <= 0 ) { msg = "Invalid HDF Type : can't retrieve the size of data type with code " + to_string ( data_type ) + "\n"; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } attr_data = ( VOIDP ) ( malloc ( n_values * sz_type ) ); if ( attr_data == NULL ) { msg = "Memory Error : can't allocate buffer for attribute " + string ( attr_name ) + "\n"; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } status = SDreadattr ( obj_id_src, i_attr, attr_data ); if ( status == FAIL ) { msg = "SDreadattr error : can't read attribute " + string ( attr_name ) + "\n"; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // write destination attribute status = SDsetattr ( obj_id_dest, attr_name, data_type, n_values, attr_data ); if ( status == FAIL ) { msg = "SDsetattr error : can't write attribute " + string ( attr_name ) + "\n"; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } free ( attr_data ); attr_data = NULL; } HDFCompressor::~ HDFCompressor() { close_sd_interface(); } VOIDP HDFCompressor::read_var_data ( const int32 sd_id, const int32 i_var, char name[], int32 & rank, int32 dim_sizes[], int32 & data_type ) { int32 sds_id; intn status; int32 n_attrs; int32 nvals = 1; VOIDP data = NULL; int32 * start = NULL; string msg ( "" ); int16 sz_type = -1; sds_id = SDselect ( sd_id, i_var ); if ( sds_id == FAIL ) { msg = "SDselect Error : fail to read select sds " + to_string ( i_var ) + " in file " + infile ; goto error; } status = SDgetinfo ( sds_id, name, &rank, dim_sizes, &data_type, &n_attrs ); if ( status == FAIL ) { msg = "SDgetinfo Error : fail to read infos of sds with id " + to_string ( sds_id ) + " in file " + infile ; goto error; } /* allocate read buffer */ // number of values in datasets for ( int32 i = 0 ; i < rank ; ++i ) nvals *= dim_sizes [i]; sz_type = get_data_type_size ( data_type ); if ( sz_type <= 0 ) { msg = "Invalid HDF Type : can't retrieve the size of data type with code " + to_string ( data_type ) + "\n"; goto error; } data = ( VOIDP ) malloc ( nvals * sz_type ); if ( data == NULL ) { msg = "Memory Error : fail to allocate data buffer for reading SDS " + string ( name ) + " in file " + infile ; goto error; } // read the whole dataset start = ( int32 * ) malloc ( rank * sizeof ( int32 ) ); if ( start == NULL ) { msg = "Memory Error : fail to allocate start buffer for reading SDS " + string ( name ) + " in file " + infile ; goto error; } fill_n ( start, rank, 0 ); status = SDreaddata ( sds_id, start, NULL, dim_sizes, (VOIDP) data ); free ( start ), start = NULL; if ( status == FAIL ) { msg = "SDreaddata Error : fail to read data of sds with id " + to_string ( sds_id ) + " in file " + infile ; goto error; } status = SDendaccess (sds_id); if ( status == FAIL ) { msg = "SDendaccess Error : fail to close access to sds with id " + to_string ( sds_id ) + " in file " + infile ; goto error; } return data; error : // free eventual previous allocations free ( data ), data = NULL; free ( start ), start = NULL; g_exception e( __FILE__ , __LINE__ , msg ); throw e; return NULL; } void HDFCompressor::copy_vars_data ( const int32 sd_id_src, const int32 sd_id_dest ) { int32 n_datasets, n_file_attrs; intn status = FAIL; string msg ( "" ); status = SDfileinfo ( sd_id_src, &n_datasets, &n_file_attrs ); if ( status == FAIL ) { msg = "SDfileinfo Error : fail to read informations of file " + this->outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // --- crosses src datasets and copy them to dest for ( int32 i_var = 0 ; i_var < n_datasets ; ++i_var ) { copy_var_data ( sd_id_src, sd_id_dest, i_var ); } } int HDFCompressor::copy_var_data ( const int32 sd_id_src, const int32 sd_id_dest, const int32 i_var ) { int32 dim_sizes [ MAX_VAR_DIMS ]; int32 rank, data_type; char name [ MAX_NC_NAME ]; string msg ( "" ); comp_info c_info; int32 * start = NULL; intn status = FAIL; /* read source data */ VOIDP data = read_var_data ( sd_id_src, i_var, name, rank, dim_sizes, data_type ); /* init out SDS */ int32 sds_id = SDcreate ( sd_id_dest, name, data_type, rank, dim_sizes ); if ( sds_id == FAIL ) { msg = "SDcreate Error : fail to create SDS " + string ( name ) + " in file " + outfile; goto exit_failure; } /* set compression */ c_info.deflate.level = comp_level; status = SDsetcompress ( sds_id, COMP_CODE_DEFLATE, &c_info ); if ( status == FAIL ) { msg = "SDsetcompress Error : fail to set compression for SDS " + string ( name ) + " in file " + outfile; goto exit_failure; } /* write data */ start = ( int32 * ) malloc ( rank * sizeof ( int32 ) ); if ( start == NULL ) { msg = "Memory Error : fail to allocate start buffer for reading SDS " + string ( name ) + " in file " + infile ; goto exit_failure; } fill_n ( start, rank, 0 ); status = SDwritedata ( sds_id, start, NULL, dim_sizes, data ); if ( status == FAIL ) { msg = "SDwritedata Error : fail to write SDS " + string ( name ) + " in file " + outfile; goto exit_failure; } status = SDendaccess (sds_id); if ( status == FAIL ) { msg = "SDendaccess Error : fail to close SDS " + string ( name ) + " in file " + outfile; goto exit_failure; } free ( start ); free ( data ); return 1; exit_failure : free ( data ); free ( start ); g_exception e( __FILE__ , __LINE__ , msg ); throw e; return FAIL; } void HDFCompressor::copy_vars_attr (const int32 sd_id_src, const int32 sd_id_dest) { int32 n_datasets, n_file_attrs; intn status = FAIL; int32 sds_id_src, sds_id_dest; int32 n_var_attrs; int32 dim_sizes [ MAX_VAR_DIMS ]; int32 rank, data_type; char name [ MAX_NC_NAME ]; string msg ( "" ); status = SDfileinfo ( sd_id_src, &n_datasets, &n_file_attrs ); if ( status == FAIL ) { msg = "SDfileinfo Error : fail to read informations of file " + outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // --- crosses src datasets and copy them to dest for ( int32 i_var = 0 ; i_var < n_datasets ; ++i_var ) { // open source and dest SDS sds_id_src = SDselect ( sd_id_src, i_var ); if ( sds_id_src == FAIL ) { msg = "SDselect Error : fail to select source sds " + to_string ( i_var ) + " in file " + infile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } sds_id_dest = SDselect ( sd_id_dest, i_var ); if ( sds_id_dest == FAIL ) { msg = "SDselect Error : fail to select destination sds " + to_string ( i_var ) + " in file " + outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // read number of attributes to copy status = SDgetinfo ( sds_id_src, name, &rank, dim_sizes, &data_type, &n_var_attrs ); if ( status == FAIL ) { msg = "SDgetinfo Error : fail to read infos of sds with id " + to_string ( sds_id_src ) + " in file " + infile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } // copy them copy_attrs ( sds_id_src, sds_id_dest, n_var_attrs ); // close source and dest SDS status = SDendaccess ( sds_id_src ); if ( status == FAIL ) { msg = "SDendaccess Error : fail to close access to sds with id " + to_string ( sds_id_src ) + " in file " + infile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } status = SDendaccess ( sds_id_dest ); if ( status == FAIL ) { msg = "SDendaccess Error : fail to close access to sds with id " + to_string ( sds_id_dest ) + " in file " + outfile ; g_exception e( __FILE__ , __LINE__ , msg ); throw e; } } }