diff --git a/CHANGES.md b/CHANGES.md index bc9a71147..88f1d9378 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,15 @@ # NCBI External Developer Release: +## NCBI VDB 3.1.0 +**March 5, 2024** + + **blast**: fixed VdbBlast2naReaderData() for ReferenceSet fills entire output array + **cloud, kns, ngs-tools, sra-tools**: don't change global network timeouts when checking cloud location + **kdb, kdbmeta, sra-tools**: removed confusing warning in Windows debug build + **kns, ngs-tools, sra-tools**: fixed hanging on Mac and BSD when data access is unreliable + + ## NCBI VDB 3.0.10 **December 19, 2023** diff --git a/build/Makefile.vers b/build/Makefile.vers index 2a089606c..1aa826851 100644 --- a/build/Makefile.vers +++ b/build/Makefile.vers @@ -23,4 +23,4 @@ # =========================================================================== # NCBI-VDB and library version -VERSION = 3.0.10 +VERSION = 3.1.0 diff --git a/interfaces/kdb/consistency-check.h b/interfaces/kdb/consistency-check.h index f77fa98cc..c794c8bda 100644 --- a/interfaces/kdb/consistency-check.h +++ b/interfaces/kdb/consistency-check.h @@ -89,7 +89,7 @@ struct CCReportInfoBlock { uint64_t start; uint64_t count; - + bool missingChecksum; } blob; struct ccrb_index_s @@ -99,7 +99,6 @@ struct CCReportInfoBlock uint64_t num_keys; uint64_t num_rows; uint64_t num_holes; - } index; struct ccrb_visit_s { diff --git a/interfaces/kdb/manager.h b/interfaces/kdb/manager.h index c924376d4..ed5a525e3 100644 --- a/interfaces/kdb/manager.h +++ b/interfaces/kdb/manager.h @@ -51,7 +51,7 @@ struct KDirectory; */ enum { - /* must be handled carefully and can not go back to KDirectory + /* must be handled carefully and can not go back to KDirectory * as this value has a different meaning in KDirectory */ kptAny = 0, kptDatabase = kptLastDefined, @@ -83,15 +83,16 @@ KDB_EXTERN rc_t CC KDBManagerRelease ( const KDBManager *self ); /* MakeRead * MakeUpdate + * MakeText (reads data from a Json string) + * * create library handle for specific use - * NB - only one of the functions will be implemented * * "wd" [ IN, NULL OKAY ] - optional working directory for * accessing the file system. mgr will attach its own reference. */ KDB_EXTERN rc_t CC KDBManagerMakeRead ( const KDBManager **mgr, struct KDirectory const *wd ); KDB_EXTERN rc_t CC KDBManagerMakeUpdate ( KDBManager **mgr, struct KDirectory *wd ); - +KDB_EXTERN rc_t CC KDBManagerMakeText ( const KDBManager **mgr, const char * input, char * error, size_t error_size ); /* Version * returns the library version diff --git a/interfaces/klib/vdb_release_version.h b/interfaces/klib/vdb_release_version.h index 65ed64c3d..831e84a83 100644 --- a/interfaces/klib/vdb_release_version.h +++ b/interfaces/klib/vdb_release_version.h @@ -26,6 +26,6 @@ * ============================================================================== */ -#define VDB_RELEASE_VERSION 0x0300000A +#define VDB_RELEASE_VERSION 0x03010000 #endif /* _h_klib_vdb_release_version_ */ diff --git a/interfaces/kns/http.h b/interfaces/kns/http.h index 91e581088..ee9189ed9 100644 --- a/interfaces/kns/http.h +++ b/interfaces/kns/http.h @@ -209,17 +209,29 @@ typedef struct KClientHttpRequest KClientHttpRequest, KHttpRequest; * * "vers" [ IN ] - http version * + * "connectMillis", "readMillis, writeMillis" [ IN ] - connect/read/write + * timeouts to supply to sockets - when negative, infinite timeout; + * when 0, return immediately; positive gives maximum wait time in sec/mS + * for reads and writes respectively. + * * "conn" [ IN, NULL OKAY ] - previously opened stream for communications. * * "url" [ IN ] - full resource identifier. if "conn" is NULL, * the url is parsed for remote endpoint and is opened by mgr. */ +/* Use timeouts from KNSManager */ KNS_EXTERN rc_t CC KClientHttpMakeRequest ( const KClientHttp *self, KClientHttpRequest **req, const char *url, ... ); +/* Use timeouts from KNSManager */ KNS_EXTERN rc_t CC KNSManagerMakeClientRequest ( struct KNSManager const *self, KClientHttpRequest **req, ver_t version, struct KStream *conn, const char *url, ... ); +/* Timeouts are specified */ +KNS_EXTERN rc_t CC KNSManagerMakeTimedClientRequest ( + struct KNSManager const *self, KClientHttpRequest **req, + ver_t version, int32_t connMillis, int32_t readMillis, + int32_t writeMillis, struct KStream *conn, const char *url, ... ); /* AddRef * Release diff --git a/interfaces/kns/socket.h b/interfaces/kns/socket.h index 1baee5e87..65148441f 100644 --- a/interfaces/kns/socket.h +++ b/interfaces/kns/socket.h @@ -68,8 +68,9 @@ typedef struct KSocket KSocket; * "retryTimeout" [ IN ] - the connect request should be repeated upon failure * until this timeout expires. * - * "readMillis" [ IN ] and "writeMillis" - when negative, infinite timeout - * when 0, return immediately, positive gives maximum wait time in mS + * "connectMillis", "readMillis" and "writeMillis" [ IN ] - when negative, + * infinite timeout; + * when 0, return immediately; positive gives maximum wait time in mS * for reads and writes respectively. * * "from" [ IN ] - client endpoint @@ -78,12 +79,17 @@ typedef struct KSocket KSocket; * * both endpoints have to be of type epIP; creates a TCP connection */ + /* Use connectMillis from KNSManager */ KNS_EXTERN rc_t CC KNSManagerMakeConnection ( struct KNSManager const * self, struct KSocket ** conn, struct KEndPoint const * from, struct KEndPoint const * to ); - KNS_EXTERN rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * self, struct KSocket ** conn, int32_t readMillis, int32_t writeMillis, struct KEndPoint const * from, struct KEndPoint const * to ); +/* connectMillis is specified */ +KNS_EXTERN rc_t CC KNSManagerMakeTimedConnectionExt ( + struct KNSManager const * self, struct KSocket ** conn, + int32_t connectMillis, int32_t readMillis, int32_t writeMillis, + struct KEndPoint const * from, struct KEndPoint const * to ); KNS_EXTERN rc_t CC KNSManagerMakeRetryConnection ( struct KNSManager const * self, struct KSocket ** conn, struct timeout_t * retryTimeout, diff --git a/interfaces/vfs/services-priv.h b/interfaces/vfs/services-priv.h index 222907f95..914a7c129 100644 --- a/interfaces/vfs/services-priv.h +++ b/interfaces/vfs/services-priv.h @@ -66,7 +66,9 @@ rc_t KServiceTestNamesQueryExt ( KService * self, VRemoteProtocols protocols, const char * dir, const char * file, const char * expected ); +/* Get/Set quality type in service request */ rc_t KServiceGetQuality(const KService * self, const char ** quality); +rc_t KServiceSetQuality(KService * self, const char * quality); rc_t KService1Search ( const struct KNSManager * mgr, const char * cgi, diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index a5acba545..f3af4e5a1 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -33,6 +33,7 @@ add_subdirectory(cloud) add_subdirectory(ext) add_subdirectory(kapp) add_subdirectory(kdb) +add_subdirectory(kdbtext) add_subdirectory(kfc) add_subdirectory(kfg) add_subdirectory(kfs) diff --git a/libs/blast/blast-mgr.c b/libs/blast/blast-mgr.c index bcdb755a7..d22583d4d 100644 --- a/libs/blast/blast-mgr.c +++ b/libs/blast/blast-mgr.c @@ -50,7 +50,7 @@ #include /* fprintf */ #include /* memset */ -#define TOOLKIT "sratoolkit3_0_10" +#define TOOLKIT "sratoolkit3_1_0" /******************************************************************************/ diff --git a/libs/blast/reference.c b/libs/blast/reference.c index 830c0efd3..066b01941 100644 --- a/libs/blast/reference.c +++ b/libs/blast/reference.c @@ -928,7 +928,7 @@ static uint32_t _ReferencesData2na(References *self, self->eos = true; *status = eVdbBlastNoErr; /* end of set */ S - return 0; + return num_read; } rfd1 = rfd; rfd = &self->refs->rfd[self->rfdi]; @@ -1193,7 +1193,9 @@ static uint32_t _ReferencesData2na(References *self, ++self->read_id; } ++num_read; - break; + + if (*status == eVdbBlastNoErr || *status == eVdbBlastCircularSequence) + break; } return num_read; } @@ -1272,6 +1274,9 @@ uint32_t _Core2naDataRef(struct Core2na *self, self->eos = true; } + if (r->eos) + self->eos = true; + if (self->eos) { return 0; } diff --git a/libs/cloud/cloud-cmn.c b/libs/cloud/cloud-cmn.c index dac9d1b2f..4882ab490 100644 --- a/libs/cloud/cloud-cmn.c +++ b/libs/cloud/cloud-cmn.c @@ -48,22 +48,11 @@ rc_t KNSManager_Read(const struct KNSManager *cself, char *buffer, size_t bsize, KClientHttpRequest *req = NULL; - int32_t cmsec = 0; - int32_t rmsec = 0; - int32_t wmsec = 0; - assert(self); - /* save existing timeouts */ - cmsec = self->conn_timeout; - rmsec = self->http_read_timeout; - wmsec = self->http_write_timeout; - /* minimize timeouts to check cloudy URLs */ - self -> conn_timeout - = self -> http_read_timeout = self->http_write_timeout = 500; - - rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL, url); + rc = KNSManagerMakeTimedClientRequest( + self, &req, 0x01010000, 500, 500, 500, NULL, url); if (rc == 0 && hdrName != NULL) { @@ -108,11 +97,6 @@ rc_t KNSManager_Read(const struct KNSManager *cself, char *buffer, size_t bsize, RELEASE(KClientHttpResult, rslt); } - /* restore timeouts in KNSManager */ - self->conn_timeout = cmsec; - self->http_read_timeout = rmsec; - self->http_write_timeout = wmsec; - RELEASE(KClientHttpRequest, req); return rc; diff --git a/libs/kdb/column-base.c b/libs/kdb/column-base.c index 1998f2c52..9f513a865 100644 --- a/libs/kdb/column-base.c +++ b/libs/kdb/column-base.c @@ -105,7 +105,7 @@ rc_t KColumnSever ( const KColumn *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcColumn, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcColumn, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/column-cc.c b/libs/kdb/column-cc.c index d440e431d..bc45648d8 100644 --- a/libs/kdb/column-cc.c +++ b/libs/kdb/column-cc.c @@ -145,6 +145,7 @@ rc_t KColumnCheckBlobs(const KColumn *self, uint64_t row; uint64_t rows; rc_t rc; + uint64_t nMissingChecksum = 0; rc = KColumnIdRange(self, &start, &rows); if (rc) { @@ -157,6 +158,7 @@ rc_t KColumnCheckBlobs(const KColumn *self, const KColumnBlob *blob; int64_t first; uint32_t count; + bool missingChecksum = false; rc = KColumnOpenBlobRead(self, &blob, row + start); if (rc) { @@ -181,20 +183,27 @@ rc_t KColumnCheckBlobs(const KColumn *self, rc = KColumnBlobValidate(blob); KColumnBlobRelease(blob); if (rc) { - nfo->info.done.rc = rc; - nfo->info.done.mesg = "contains bad data"; - nfo->type = ccrpt_Done; - return report(nfo, ctx); + if (GetRCState(rc) == (int)rcNotFound && GetRCObject(rc) == (int)rcChecksum) { + missingChecksum = true; + nMissingChecksum += 1; + } + else { + nfo->type = ccrpt_Done; + nfo->info.done.rc = rc; + nfo->info.done.mesg = "contains bad data"; + return report(nfo, ctx); + } } nfo->type = ccrpt_Blob; nfo->info.blob.start = first; nfo->info.blob.count = count; + nfo->info.blob.missingChecksum = missingChecksum; rc = report(nfo, ctx); row += count; } nfo->info.done.rc = 0; - nfo->info.done.mesg = "checksums ok"; + nfo->info.done.mesg = nMissingChecksum == 0 ? "checksums ok" : "checksums missing"; nfo->type = ccrpt_Done; rc = report(nfo, ctx); return rc; diff --git a/libs/kdb/columnblob-base.c b/libs/kdb/columnblob-base.c index 81f8d6871..12d4e9f0a 100644 --- a/libs/kdb/columnblob-base.c +++ b/libs/kdb/columnblob-base.c @@ -71,7 +71,7 @@ rc_t CC KColumnBlobBaseRelease ( const KColumnBlob *cself ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcCursor, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcCursor, rcAccessing, rcSelf, rcNull ); LIB_EXPORT rc_t CC KColumnBlobAddRef ( const KColumnBlob *self ) { diff --git a/libs/kdb/database-base.c b/libs/kdb/database-base.c index 4373bd49b..cb6702eb3 100644 --- a/libs/kdb/database-base.c +++ b/libs/kdb/database-base.c @@ -108,7 +108,7 @@ rc_t KDatabaseSever ( const KDatabase *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcDatabase, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcDatabase, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ @@ -202,7 +202,7 @@ LIB_EXPORT rc_t CC KDatabaseOpenTableRead ( const KDatabase *self, const struct KTable **tbl, const char *name, ... ) { if ( self == NULL && self -> vt == NULL ) - return RC ( rcVDB, rcDatabase, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcDatabase, rcAccessing, rcSelf, rcNull ); rc_t rc; va_list args; diff --git a/libs/kdb/database-base.h b/libs/kdb/database-base.h index 469cb9745..a883bfea3 100644 --- a/libs/kdb/database-base.h +++ b/libs/kdb/database-base.h @@ -80,7 +80,7 @@ struct KDatabase_vt struct KDatabaseBase { - KDatabase_vt * vt; + const KDatabase_vt * vt; KRefcount refcount; }; diff --git a/libs/kdb/database-cmn.h b/libs/kdb/database-cmn.h index 30e92afd0..5f4ebc16e 100644 --- a/libs/kdb/database-cmn.h +++ b/libs/kdb/database-cmn.h @@ -37,8 +37,8 @@ #define KDATABASE_IMPL KDatabase #include "database-base.h" -#ifdef __cplusplus +#ifdef __cplusplus extern "C" { #endif diff --git a/libs/kdb/index-base.c b/libs/kdb/index-base.c index ce995bd58..3b1f89f13 100644 --- a/libs/kdb/index-base.c +++ b/libs/kdb/index-base.c @@ -72,7 +72,7 @@ rc_t CC KIndexBaseRelease ( const KIndex *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/index-base.h b/libs/kdb/index-base.h index 38558f18c..3abe481b5 100644 --- a/libs/kdb/index-base.h +++ b/libs/kdb/index-base.h @@ -74,7 +74,7 @@ struct KIndex_vt struct KIndex { - KIndex_vt * vt; + const KIndex_vt * vt; KRefcount refcount; }; diff --git a/libs/kdb/libkdb.vers.h b/libs/kdb/libkdb.vers.h index 9e8c421a8..e8a839f04 100644 --- a/libs/kdb/libkdb.vers.h +++ b/libs/kdb/libkdb.vers.h @@ -26,4 +26,4 @@ #pragma once -#define LIBKDB_VERS 0x0300000A +#define LIBKDB_VERS 0x03010000 diff --git a/libs/kdb/manager-base.c b/libs/kdb/manager-base.c index 1184af740..c0a6bfc7d 100644 --- a/libs/kdb/manager-base.c +++ b/libs/kdb/manager-base.c @@ -109,7 +109,7 @@ rc_t KDBManagerSever ( const KDBManager *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/manager-base.h b/libs/kdb/manager-base.h index 67f4f5575..13a66ee71 100644 --- a/libs/kdb/manager-base.h +++ b/libs/kdb/manager-base.h @@ -76,20 +76,9 @@ struct KDBManager_vt rc_t ( CC * vPathOpenRemoteDBRead ) ( struct KDBMGR_IMPL const * self, struct KDatabase const ** p_db, struct VPath const * remote, struct VPath const * cache ); }; -// Public write side-only API - -// KDB_EXTERN rc_t CC KDBManagerLock ( KDBManager *self, const char *path, ... ); -// KDB_EXTERN rc_t CC KDBManagerVLock ( KDBManager *self, const char *path, va_list args ); - -// KDB_EXTERN rc_t CC KDBManagerUnlock ( KDBManager *self, const char *path, ... ); -// KDB_EXTERN rc_t CC KDBManagerVUnlock ( KDBManager *self, const char *path, va_list args ); - -// KDB_EXTERN rc_t CC KDBManagerDrop ( KDBManager *self, uint32_t obj_type, const char *path, ... ); -// KDB_EXTERN rc_t CC KDBManagerVDrop ( KDBManager *self, uint32_t obj_type, const char *path, va_list args ); - struct KDBManagerBase { - KDBManager_vt * vt; + const KDBManager_vt * vt; KRefcount refcount; }; @@ -107,9 +96,6 @@ extern rc_t CC KDBManagerBaseRelease ( const KDBMGR_IMPL *self ); KDBMGR_IMPL *KDBManagerAttach ( const KDBMGR_IMPL *self ); rc_t KDBManagerSever ( const KDBMGR_IMPL *self ); -// write side only public API -// ... - #ifdef __cplusplus } #endif diff --git a/libs/kdb/meta-base.c b/libs/kdb/meta-base.c index a1fe88f7c..68056a255 100644 --- a/libs/kdb/meta-base.c +++ b/libs/kdb/meta-base.c @@ -104,7 +104,7 @@ rc_t KMetadataSever ( const KMetadata *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/meta-base.h b/libs/kdb/meta-base.h index e9feaf03f..251cb4fe3 100644 --- a/libs/kdb/meta-base.h +++ b/libs/kdb/meta-base.h @@ -65,7 +65,7 @@ struct KMetadata_vt struct KMetadata { - KMetadata_vt * vt; + const KMetadata_vt * vt; KRefcount refcount; }; diff --git a/libs/kdb/metanode-base.c b/libs/kdb/metanode-base.c index 4d4738fe8..6d90f0f28 100644 --- a/libs/kdb/metanode-base.c +++ b/libs/kdb/metanode-base.c @@ -93,7 +93,7 @@ rc_t KMDataNodeSever ( const KMDataNode *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/metanode-base.h b/libs/kdb/metanode-base.h index 9df6de278..036403bec 100644 --- a/libs/kdb/metanode-base.h +++ b/libs/kdb/metanode-base.h @@ -74,7 +74,7 @@ struct KMDataNode { BSTNode n; - KMDataNode_vt * vt; + const KMDataNode_vt * vt; KRefcount refcount; }; diff --git a/libs/kdb/rcolumnblob.c b/libs/kdb/rcolumnblob.c index 038fa19eb..34b74eb46 100644 --- a/libs/kdb/rcolumnblob.c +++ b/libs/kdb/rcolumnblob.c @@ -177,7 +177,7 @@ KRColumnBlobIdRange ( const KColumnBlob *bself, int64_t *first, uint32_t *count * runs checksum validation on unmodified blob */ static -rc_t KColumnBlobValidateCRC32 ( const KRColumnBlob *self ) +rc_t validateCRC32 ( const KRColumnBlob *self ) { rc_t rc; const KRColumn *col = self -> col; @@ -222,7 +222,7 @@ rc_t KColumnBlobValidateCRC32 ( const KRColumnBlob *self ) } static -rc_t KColumnBlobValidateMD5 ( const KRColumnBlob *self ) +rc_t validateMD5 ( const KRColumnBlob *self ) { rc_t rc; const KRColumn *col = self -> col; @@ -274,15 +274,21 @@ rc_t CC KRColumnBlobValidate ( const KColumnBlob *bself ) { CAST(); - if ( self -> loc . u . blob . size != 0 ) switch ( self -> col -> checksum ) + + if ( self -> loc . u . blob . size == 0 ) + return 0; + + switch ( self -> col -> checksum ) { case kcsCRC32: - return KColumnBlobValidateCRC32 ( self ); + return validateCRC32 ( self ); case kcsMD5: - return KColumnBlobValidateMD5 ( self ); + return validateMD5 ( self ); + case kcsNone: + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcChecksum, rcNotFound ); + default: + return RC ( rcDB, rcBlob, rcValidating, rcType, rcUnexpected ); } - - return 0; } /* ValidateBuffer @@ -293,34 +299,24 @@ KRColumnBlobValidate ( const KColumnBlob *bself ) * "cs_data" [ IN ] and "cs_data_size" [ IN ] - returned checksum data from ReadAll */ static -rc_t KColumnBlobValidateBufferCRC32 ( const void * buffer, size_t size, uint32_t cs ) +rc_t validateBufferCRC32 ( const void * buffer, size_t size, uint32_t cs ) { - uint32_t crc32 = CRC32 ( 0, buffer, size ); - - if ( cs != crc32 ) - return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); - - return 0; + uint32_t const crc32 = CRC32 ( 0, buffer, size ); + + return cs == crc32 ? 0 : RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); } static -rc_t KColumnBlobValidateBufferMD5 ( const void * buffer, size_t size, const uint8_t cs [ 16 ] ) +rc_t validateBufferMD5 ( const void * buffer, size_t size, const uint8_t cs [ 16 ] ) { - MD5State md5; uint8_t digest [ 16 ]; + MD5State md5; MD5StateInit ( & md5 ); - - /* calculate checksum */ MD5StateAppend ( & md5, buffer, size ); - - /* finish MD5 digest */ MD5StateFinish ( & md5, digest ); - if ( memcmp ( cs, digest, sizeof digest ) != 0 ) - return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); - - return 0; + return memcmp ( cs, digest, sizeof digest ) == 0 ? 0 : RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); } static @@ -340,19 +336,21 @@ KRColumnBlobValidateBuffer ( const KColumnBlob * bself, return RC ( rcDB, rcBlob, rcValidating, rcData, rcInsufficient ); if ( bsize > self -> loc . u . blob . size ) return RC ( rcDB, rcBlob, rcValidating, rcData, rcExcessive ); + if ( bsize == 0 ) + return 0; - if ( bsize != 0 ) switch ( self -> col -> checksum ) + switch ( self -> col -> checksum ) { - case kcsNone: - break; case kcsCRC32: - return KColumnBlobValidateBufferCRC32 ( buffer -> base, bsize, + return validateBufferCRC32 ( buffer -> base, bsize, self -> bswap ? bswap_32 ( cs_data -> crc32 ) : cs_data -> crc32 ); case kcsMD5: - return KColumnBlobValidateBufferMD5 ( buffer -> base, bsize, cs_data -> md5_digest ); + return validateBufferMD5 ( buffer -> base, bsize, cs_data -> md5_digest ); + case kcsNone: + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcChecksum, rcNotFound ); + default: + return RC ( rcDB, rcBlob, rcValidating, rcType, rcUnexpected ); } - - return 0; } diff --git a/libs/kdb/rdatabase.c b/libs/kdb/rdatabase.c index 681fb372d..6c2643818 100644 --- a/libs/kdb/rdatabase.c +++ b/libs/kdb/rdatabase.c @@ -81,7 +81,7 @@ static KDatabase_vt KRDatabase_vt = KRDatabaseVOpenTableRead, KRDatabaseOpenMetadataRead, KRDatabaseVOpenIndexRead, - KRDatabaseListDB, + KRDatabaseListDB, KRDatabaseListTbl, KRDatabaseListIdx, KRDatabaseGetPath diff --git a/libs/kdb/rtable.c b/libs/kdb/rtable.c index 765283d35..a871e2891 100644 --- a/libs/kdb/rtable.c +++ b/libs/kdb/rtable.c @@ -66,7 +66,7 @@ static rc_t CC KRTableListCol ( const KTable *self, KNamelist **names ); static rc_t CC KRTableListIdx ( const KTable *self, KNamelist **names ); static rc_t CC KRTableMetaCompare( const KTable *self, const KTable *other, const char * path, bool * equal ); -static KTableBase_vt KRTable_vt = +static KTable_vt KRTable_vt = { KRTableWhack, KTableBaseAddRef, diff --git a/libs/kdb/table-base.c b/libs/kdb/table-base.c index a7e65be87..af655768f 100644 --- a/libs/kdb/table-base.c +++ b/libs/kdb/table-base.c @@ -105,7 +105,7 @@ rc_t KTableSever ( const KTable *self ) if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ else \ - return RC ( rcVDB, rcTable, rcAccessing, rcSelf, rcNull ); + return RC ( rcDB, rcTable, rcAccessing, rcSelf, rcNull ); #define DISPATCH_BOOL(call) \ if ( self != NULL && self -> vt != NULL ) \ return self -> vt -> call; \ diff --git a/libs/kdb/table-base.h b/libs/kdb/table-base.h index 76b6a2742..78e665599 100644 --- a/libs/kdb/table-base.h +++ b/libs/kdb/table-base.h @@ -48,8 +48,8 @@ struct KNamelist; */ typedef struct KTable KTable; -typedef struct KTableBase_vt KTableBase_vt; -struct KTableBase_vt +typedef struct KTable_vt KTable_vt; +struct KTable_vt { /* Public API */ rc_t ( CC * whack ) ( KTable * self ); @@ -80,7 +80,7 @@ extern rc_t CC KTableBaseRelease ( const KTable *self ); struct KTable { - const KTableBase_vt * vt; + const KTable_vt * vt; KRefcount refcount; }; diff --git a/libs/kdb/wtable.c b/libs/kdb/wtable.c index c27eeb1ff..927128729 100644 --- a/libs/kdb/wtable.c +++ b/libs/kdb/wtable.c @@ -66,7 +66,7 @@ static rc_t CC KWTableListCol ( const KTable *self, KNamelist **names ); static rc_t CC KWTableListIdx ( const KTable *self, KNamelist **names ); static rc_t CC KWTableMetaCompare( const KTable *self, const KTable *other, const char * path, bool * equal ); -static KTableBase_vt KWTable_vt = +static KTable_vt KWTable_vt = { KWTableWhack, KTableBaseAddRef, @@ -699,7 +699,7 @@ LIB_EXPORT rc_t CC KTableOpenParentUpdate ( KTable *bself, KDatabase **db ) if ( self == NULL ) rc = RC ( rcDB, rcTable, rcAccessing, rcSelf, rcNull ); else if ( self -> db != NULL && self -> db -> read_only ) - rc = RC ( rcVDB, rcTable, rcAccessing, rcDatabase, rcReadonly ); + rc = RC ( rcDB, rcTable, rcAccessing, rcDatabase, rcReadonly ); else { rc = KDatabaseAddRef ( self -> db ); diff --git a/libs/kdbtext/CMakeLists.txt b/libs/kdbtext/CMakeLists.txt new file mode 100644 index 000000000..879bad635 --- /dev/null +++ b/libs/kdbtext/CMakeLists.txt @@ -0,0 +1,39 @@ +# =========================================================================== +# +# PUBLIC DOMAIN NOTICE +# National Center for Biotechnology Information +# +# This software/database is a "United States Government Work" under the +# terms of the United States Copyright Act. It was written as part of +# the author's official duties as a United States Government employee and +# thus cannot be copyrighted. This software/database is freely available +# to the public for use. The National Library of Medicine and the U.S. +# Government have not placed any restriction on its use or reproduction. +# +# Although all reasonable efforts have been taken to ensure the accuracy +# and reliability of the software and data, the NLM and the U.S. +# Government do not and cannot warrant the performance or results that +# may be obtained by using this software or data. The NLM and the U.S. +# Government disclaim all warranties, express or implied, including +# warranties of performance, merchantability or fitness for any particular +# purpose. +# +# Please cite the author in any work or product based on this material. +# +# =========================================================================== + +set( KDBTEXT_SRC + api-manager.cpp + manager.cpp + database.cpp + table.cpp + column.cpp + index.cpp + path.cpp + metadata.cpp + columnblob.cpp +) + +GenerateStaticLibs( kdbtext "${KDBTEXT_SRC}" ) + +add_compile_definitions (__mod__="libs/kdbtext") \ No newline at end of file diff --git a/libs/kdbtext/Makefile b/libs/kdbtext/Makefile new file mode 100644 index 000000000..d2e2e1cba --- /dev/null +++ b/libs/kdbtext/Makefile @@ -0,0 +1,28 @@ +# =========================================================================== +# +# PUBLIC DOMAIN NOTICE +# National Center for Biotechnology Information +# +# This software/database is a "United States Government Work" under the +# terms of the United States Copyright Act. It was written as part of +# the author's official duties as a United States Government employee and +# thus cannot be copyrighted. This software/database is freely available +# to the public for use. The National Library of Medicine and the U.S. +# Government have not placed any restriction on its use or reproduction. +# +# Although all reasonable efforts have been taken to ensure the accuracy +# and reliability of the software and data, the NLM and the U.S. +# Government do not and cannot warrant the performance or results that +# may be obtained by using this software or data. The NLM and the U.S. +# Government disclaim all warranties, express or implied, including +# warranties of performance, merchantability or fitness for any particular +# purpose. +# +# Please cite the author in any work or product based on this material. +# +# =========================================================================== + +TOP ?= $(abspath ../..) +MODULE = libs/kdbtext + +include $(TOP)/build/Makefile.env diff --git a/libs/kdbtext/api-manager.cpp b/libs/kdbtext/api-manager.cpp new file mode 100644 index 000000000..8ebddbf0f --- /dev/null +++ b/libs/kdbtext/api-manager.cpp @@ -0,0 +1,310 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "api-manager.hpp" + +#include "manager.hpp" +#include "database.hpp" +#include "table.hpp" +#include "path.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include + +using namespace std; + +const uint32_t LIBKDBTEXT_VERS = 0; + +static rc_t CC KTextManagerWhack ( KDBManager *self ); +static rc_t CC KTextManagerVersion ( const KDBManager *self, uint32_t *version ); +static bool CC KTextManagerVExists ( const KDBManager *self, uint32_t requested, const char *name, va_list args ); +static rc_t CC KTextManagerVWritable ( const KDBManager *self, const char * path, va_list args ); +static rc_t CC KTextManagerRunPeriodicTasks ( const KDBManager *self ); +static int CC KTextManagerPathTypeVP( const KDBManager * self, const VPath * path ); +static int CC KTextManagerVPathType ( const KDBManager * self, const char *path, va_list args ); +static int CC KTextManagerVPathTypeUnreliable ( const KDBManager * self, const char *path, va_list args ); +static rc_t CC KTextManagerVOpenDBRead ( const KDBManager *self, const KDatabase **db, const char *path, va_list args ); +static rc_t CC KTextManagerVOpenTableRead ( const KDBManager *self, const KTable **tbl, const char *path, va_list args ); +static rc_t CC KTextManagerOpenTableReadVPath(const KDBManager *self, const KTable **tbl, const struct VPath *path); +static rc_t CC KTextManagerVOpenColumnRead ( const KDBManager *self, const KColumn **col, const char *path, va_list args ); +static rc_t CC KTextManagerVPathOpenLocalDBRead ( struct KDBManager const * self, struct KDatabase const ** p_db, struct VPath const * vpath ); +static rc_t CC KTextManagerVPathOpenRemoteDBRead ( struct KDBManager const * self, struct KDatabase const ** p_db, struct VPath const * remote, struct VPath const * cache ); + +static KDBManager_vt KTextManager_vt = +{ + KTextManagerWhack, + KDBManagerBaseAddRef, + KDBManagerBaseRelease, + KTextManagerVersion, + KTextManagerVExists, + KTextManagerVWritable, + KTextManagerRunPeriodicTasks, + KTextManagerPathTypeVP, + KTextManagerVPathType, + KTextManagerVPathTypeUnreliable, + KTextManagerVOpenDBRead, + KTextManagerVOpenTableRead, + KTextManagerOpenTableReadVPath, + KTextManagerVOpenColumnRead, + KTextManagerVPathOpenLocalDBRead, + KTextManagerVPathOpenRemoteDBRead +}; + +using namespace KDBText; + +#define CAST() assert( bself -> dad . vt == & KTextManager_vt ); const Manager *self = static_cast(bself); + +static +rc_t CC +KTextManagerWhack ( KDBManager *self ) +{ + assert( self -> dad . vt == & KTextManager_vt ); + delete reinterpret_cast( self ); + return 0; +} + +static +rc_t CC +KTextManagerVersion ( const KDBManager *self, uint32_t * version ) +{ + if ( version == nullptr ) + { + return SILENT_RC ( rcDB, rcMgr, rcAccessing, rcParam, rcNull ); + } + + * version = LIBKDBTEXT_VERS; + return 0; +} + +static +rc_t +PrintToString( const char *fmt, va_list args, string & out ) +{ + KDataBuffer buf; + rc_t rc = KDataBufferMake ( & buf, 8, 0 ); + if ( rc != 0 ) + { + return false; + } + + rc = KDataBufferVPrintf ( & buf, fmt, args ); + if ( rc != 0 ) + { + return rc; + } + + out = string((const char *) (buf . base)); // will be 0-terminated + + rc = KDataBufferWhack ( & buf ); + if ( rc != 0 ) + { + return rc; + } + + return rc; +} + +static +bool CC +KTextManagerVExists ( const KDBManager *bself, uint32_t requested, const char *fmt, va_list args ) +{ + CAST(); + return self -> exists( requested, Path( fmt, args ) ); +} + +static +rc_t CC +KTextManagerVWritable ( const KDBManager *bself, const char * fmt, va_list args ) +{ + CAST(); + + string path; + PrintToString( fmt, args, path ); + + return self -> writable( Path(path) ); +} + +static +rc_t CC +KTextManagerRunPeriodicTasks ( const KDBManager *self ) +{ + return 0; +} + +static +int CC +KTextManagerPathTypeVP( const KDBManager * bself, const VPath * path ) +{ + CAST(); + + // parse and resolve the path + const String * p; + rc_t rc = VPathMakeString ( path, &p ); + if ( rc == 0 ) + { + int ret = self -> pathType( string ( p -> addr, p -> size ) ); + StringWhack ( p ); + return ret; + } + + return kptNotFound; +} + +static +int CC +KTextManagerVPathType ( const KDBManager * bself, const char *fmt, va_list args ) +{ + CAST(); + + string path; + PrintToString( fmt, args, path ); + + return self -> pathType( path ); +} + +static +int CC +KTextManagerVPathTypeUnreliable ( const KDBManager * self, const char *path, va_list args ) +{ + return KTextManagerVPathType( self, path, args ); +} + +static +rc_t CC +KTextManagerVOpenDBRead ( const KDBManager *bself, const KDatabase **p_db, const char *fmt, va_list args ) +{ + CAST(); + + string path; + PrintToString( fmt, args, path ); + const Database * db = nullptr; + rc_t rc = self -> openDatabase( Path( path ), db ); + if ( rc == 0 ) + { + *p_db = (const KDatabase *)db; + } + + return rc; +} + +static +rc_t CC +KTextManagerVOpenTableRead ( const KDBManager *bself, const KTable **p_tbl, const char *fmt, va_list args ) +{ + CAST(); + + string path; + PrintToString( fmt, args, path ); + const Table * tbl = nullptr; + rc_t rc = self -> openTable( Path( path ), tbl ); + if ( rc == 0 ) + { + *p_tbl = (const KTable *)tbl; + } + + return rc; +} + +static +rc_t CC +KTextManagerOpenTableReadVPath(const KDBManager *bself, const KTable **p_tbl, const struct VPath *p_path) +{ + CAST(); + + const Table * tbl = nullptr; + rc_t rc = self -> openTable( Path( p_path ), tbl ); + if ( rc == 0 ) + { + *p_tbl = (const KTable *)tbl; + } + + return rc; +} + +static +rc_t CC +KTextManagerVOpenColumnRead ( const KDBManager *bself, const KColumn **p_col, const char *fmt, va_list args ) +{ // not supported here + return SILENT_RC ( rcDB, rcMgr, rcAccessing, rcColumn, rcUnsupported ); +} + +static +rc_t CC +KTextManagerVPathOpenLocalDBRead ( const KDBManager * bself, struct KDatabase const ** p_db, struct VPath const * vpath ) +{ + CAST(); + + const Database * db = nullptr; + rc_t rc = self -> openDatabase( Path( vpath ), db ); + if ( rc == 0 ) + { + *p_db = (const KDatabase *)db; + } + + return rc; +} + +static +rc_t CC +KTextManagerVPathOpenRemoteDBRead ( struct KDBManager const * self, struct KDatabase const ** p_db, struct VPath const * remote, struct VPath const * cache ) +{ + return SILENT_RC (rcDB, rcMgr, rcOpening, rcType, rcInvalid); +} + +rc_t CC +KDBManagerMakeText ( const KDBManager ** p_mgr, const char * input, char * error, size_t error_size ) +{ + if ( p_mgr == nullptr ) + { + return SILENT_RC ( rcDB, rcMgr, rcCreating, rcSelf, rcNull ); + } + + Manager * mgr = new Manager( KTextManager_vt ); + + rc_t rc = mgr -> parse( input, error, error_size ); + if ( rc == 0 ) + { + *p_mgr = mgr; + } + else + { + delete mgr; + } + + return rc; +} + diff --git a/libs/kdbtext/api-manager.hpp b/libs/kdbtext/api-manager.hpp new file mode 100644 index 000000000..7c94f1c14 --- /dev/null +++ b/libs/kdbtext/api-manager.hpp @@ -0,0 +1,36 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +typedef struct KDBManager KDBManager; +#define KDBMGR_IMPL KDBManager +#include "../libs/kdb/manager-base.h" + +struct KDBManager +{ + KDBManagerBase dad; +}; diff --git a/libs/kdbtext/column.cpp b/libs/kdbtext/column.cpp new file mode 100644 index 000000000..669202266 --- /dev/null +++ b/libs/kdbtext/column.cpp @@ -0,0 +1,398 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "column.hpp" + +#include "manager.hpp" +#include "metadata.hpp" +#include "columnblob.hpp" + +#include + +#include + +using namespace KDBText; +using namespace std; + +/*-------------------------------------------------------------------------- + * KTextColumn (formerly KColumn) + * a read-only collection of blobs indexed by oid; file system-based + */ + +static rc_t CC KTextColumnWhack ( KColumn *self ); +static bool CC KTextColumnLocked ( const KColumn *self ); +static rc_t CC KTextColumnVersion ( const KColumn *self, uint32_t *version ); +static rc_t CC KTextColumnByteOrder ( const KColumn *self, bool *reversed ); +static rc_t CC KTextColumnIdRange ( const KColumn *self, int64_t *first, uint64_t *count ); +static rc_t CC KTextColumnFindFirstRowId ( const KColumn * self, int64_t * found, int64_t start ); +static rc_t CC KTextColumnOpenManagerRead ( const KColumn *self, const KDBManager **mgr ); +static rc_t CC KTextColumnOpenParentRead ( const KColumn *self, const KTable **tbl ); +static rc_t CC KTextColumnOpenMetadataRead ( const KColumn *self, const KMetadata **metap ); +static rc_t CC KTextColumnOpenBlobRead ( const KColumn *self, const KColumnBlob **blobp, int64_t id ); + +static KColumn_vt KTextColumn_vt = +{ + /* Public API */ + KTextColumnWhack, + KColumnBaseAddRef, + KColumnBaseRelease, + KTextColumnLocked, + KTextColumnVersion, + KTextColumnByteOrder, + KTextColumnIdRange, + KTextColumnFindFirstRowId, + KTextColumnOpenManagerRead, + KTextColumnOpenParentRead, + KTextColumnOpenMetadataRead, + KTextColumnOpenBlobRead +}; + +#define CAST() assert( bself->vt == &KTextColumn_vt ); Column * self = (Column *)bself + +Column::Column( const KJsonObject * p_json, const Manager * p_mgr, const Table * p_parent ) +: m_mgr( p_mgr ), m_parent( p_parent ), m_json ( p_json ) +{ + dad . vt = & KTextColumn_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Column", "ctor", "db" ); + Manager::addRef( m_mgr ); +} + +Column::~Column() +{ + Metadata::release( m_meta ); + Manager::release( m_mgr ); + + for( auto d : m_data ) + { + KDataBufferWhack( & d.second ); + } + KRefcountWhack ( & dad . refcount, "KDBText::Column" ); +} + +void +Column::addRef( const Column * col ) +{ + if ( col != nullptr ) + { + KColumnAddRef( (const KColumn*) col ); + } +} + +void +Column::release( const Column * col ) +{ + if ( col != nullptr ) + { + KColumnRelease( (const KColumn*) col ); + } +} + +rc_t +Column::inflate( char * p_error, size_t p_error_size ) +{ + rc_t rc = 0; + + const KJsonValue * name = KJsonObjectGetMember ( m_json, "name" ); + if ( name != nullptr ) + { + const char * nameStr = nullptr; + rc = KJsonGetString ( name, & nameStr ); + if ( rc == 0 ) + { + m_name = nameStr; + } + } + else + { + string_printf ( p_error, p_error_size, nullptr, "Column name is missing" ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + + const KJsonValue * type = KJsonObjectGetMember ( m_json, "type" ); + if ( type != nullptr ) + { + const char * typeStr = nullptr; + rc = KJsonGetString ( type, & typeStr ); + if ( rc == 0 ) + { + m_type = typeStr; + } + } + else + { + string_printf ( p_error, p_error_size, nullptr, "Column type is missing" ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + + // data + const KJsonValue * data = KJsonObjectGetMember ( m_json, "data" ); + if ( data != nullptr ) + { + const KJsonArray * dataarr = KJsonValueToArray ( data ); + if ( dataarr == nullptr ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.data is not an array", m_name.c_str() ); + return SILENT_RC( rcDB, rcColumn, rcCreating, rcParam, rcInvalid ); + } + + uint32_t len = KJsonArrayGetLength ( dataarr ); + for ( uint32_t i = 0; i < len; ++i ) + { + const KJsonValue * v = KJsonArrayGetElement ( dataarr, i ); + assert( v != nullptr ); + const KJsonObject * obj = KJsonValueToObject ( v ); + if( obj != nullptr ) + { + const KJsonValue * id = KJsonObjectGetMember ( obj, "row" ); + if ( id == nullptr ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.data[%i].row is missing", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcColumn, rcCreating, rcParam, rcInvalid ); + } + const KJsonValue * value = KJsonObjectGetMember ( obj, "value" ); + if ( value == nullptr ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.data[%i].value is missing", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcColumn, rcCreating, rcParam, rcInvalid ); + } + + int64_t rowId; + rc = KJsonGetNumber ( id, &rowId ); + if ( rc == 0 ) + { + if ( m_data.find( rowId ) != m_data . end() ) + { + string_printf ( p_error, p_error_size, nullptr, "Duplicate row id: %s", rowId ); + return SILENT_RC( rcDB, rcColumn, rcCreating, rcParam, rcInvalid ); + } + + const char * valueStr = nullptr; + rc = KJsonGetString ( value, & valueStr ); + if ( rc == 0 ) + { + KDataBuffer b; + KDataBufferMakeBytes( & b, strlen( valueStr ) + 1 ); + strcpy( (char*)b.base, valueStr ); + m_data [ rowId ] = b; + } + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.data[%i].row is not an integer", m_name.c_str(), i ); + return rc; + } + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.data[%i] is not an object", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcColumn, rcCreating, rcParam, rcInvalid ); + } + } + } + + // metadata + const KJsonValue * meta = KJsonObjectGetMember ( m_json, "metadata" ); + if ( meta != nullptr ) + { + const KJsonObject * obj = KJsonValueToObject ( meta ); + if( obj != nullptr ) + { + Metadata * m = new Metadata( obj ); + rc = m -> inflate( p_error, p_error_size ); + if ( rc != 0 ) + { + delete m; + return rc; + } + m_meta = m; + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.metadata is not an object", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + + return rc; +} + +pair< int64_t, uint64_t > +Column::idRange() const +{ // { first, last - first + 1 } + + if ( m_data.empty() ) + { + return make_pair( 0, 0 ); + } + return make_pair( m_data.begin()->first, m_data.rbegin()->first - m_data.begin()->first + 1 ); +} + +int64_t +Column::findFirst( int64_t row ) const +{ + auto it = m_data . lower_bound( row ); + if ( it == m_data.end() ) + { + return 0; + } + return it -> first; +} + +const ColumnBlob * +Column::openBlob( int64_t id ) const +{ + auto it = m_data.find( id ); + if ( it != m_data.end() ) + { + return new ColumnBlob( it -> second . base, it -> second . elem_count, this, id, 1 ); + } + return nullptr; +} + +// API + +static +rc_t CC +KTextColumnWhack ( KColumn *bself ) +{ + CAST(); + + delete reinterpret_cast( self ); + return 0; +} + +static +bool CC +KTextColumnLocked ( const KColumn *self ) +{ + return false; +} + +static +rc_t CC +KTextColumnVersion ( const KColumn *self, uint32_t *version ) +{ + *version = 0; + return 0; +} + +static +rc_t CC +KTextColumnByteOrder ( const KColumn *self, bool *reversed ) +{ + *reversed = false; + return 0; +} + +static +rc_t CC +KTextColumnIdRange ( const KColumn * bself, int64_t *first, uint64_t *count ) +{ + CAST(); + + auto p = self->idRange(); + + *first = p.first; + *count = p.second; + return 0; +} + +static +rc_t CC +KTextColumnFindFirstRowId ( const KColumn * bself, int64_t * found, int64_t start ) +{ + CAST(); + + *found = self -> findFirst( start ); + if ( *found == 0 ) + { + return SILENT_RC ( rcDB, rcColumn, rcSelecting, rcRow, rcNotFound ); + } + return 0; +} + +static +rc_t CC +KTextColumnOpenManagerRead ( const KColumn *bself, const KDBManager **mgr ) +{ + CAST(); + + const Manager * m = self -> getManager(); + if ( m != nullptr ) + { + Manager::addRef( m ); + } + + *mgr = (const KDBManager*)m; + + return 0; +} + +static +rc_t CC +KTextColumnOpenParentRead ( const KColumn *bself, const KTable **tbl ) +{ + CAST(); + + const Table * p = self -> getParent(); + if ( p != nullptr ) + { + Table::addRef( p ); + } + + *tbl = (const KTable*) p; + + return 0; +} + +static +rc_t CC +KTextColumnOpenMetadataRead ( const KColumn *bself, const KMetadata **metap ) +{ + CAST(); + const Metadata * m = self->openMetadata(); + if ( m != nullptr ) + { + Metadata::addRef( m ); + *metap = (const KMetadata*)m; + return 0; + } + return SILENT_RC( rcDB, rcColumn, rcOpening, rcMetadata, rcNotFound ); +} + +static +rc_t CC +KTextColumnOpenBlobRead ( const KColumn *bself, const KColumnBlob **blobp, int64_t id ) +{ + CAST(); + const ColumnBlob * b = self->openBlob( id ); + if ( b != nullptr ) + { + *blobp = (const KColumnBlob*)b; + return 0; + } + return SILENT_RC( rcDB, rcColumn, rcOpening, rcBlob, rcNotFound ); +} diff --git a/libs/kdbtext/column.hpp b/libs/kdbtext/column.hpp new file mode 100644 index 000000000..13b468c36 --- /dev/null +++ b/libs/kdbtext/column.hpp @@ -0,0 +1,86 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "../libs/kdb/column-base.h" + +#include +#include +#include + +#include +#include + +typedef struct KTextColumn KTextColumn; +struct KTextColumn +{ + KColumn dad; +}; + +namespace KDBText +{ + class Table; + class Manager; + class Metadata; + class ColumnBlob; + + class Column : public KTextColumn + { + public: + static void addRef( const Column* ); + static void release( const Column *); + + public: + Column( const KJsonObject * p_json, const Manager * mgr = nullptr, const Table * parent = nullptr ); + ~Column(); + + rc_t inflate( char * error, size_t error_size ); + + const Manager * getManager() const { return m_mgr; } + const Table * getParent() const { return m_parent; } + const std::string & getName() const { return m_name; } + + std::pair< int64_t, uint64_t > idRange() const; // { first, last - first + 1 }; { 0, 0 } if empty + + int64_t findFirst( int64_t row ) const; + + const Metadata * openMetadata() const { return m_meta; } + + const ColumnBlob * openBlob( int64_t id ) const; + + private: + const Manager * m_mgr = nullptr; + const Table * m_parent = nullptr; + + const KJsonObject * m_json = nullptr; + std::string m_name; + std::string m_type; + + std::map< uint64_t, KDataBuffer > m_data; + const Metadata * m_meta = nullptr; + }; +} diff --git a/libs/kdbtext/columnblob.cpp b/libs/kdbtext/columnblob.cpp new file mode 100644 index 000000000..dc61bcfd1 --- /dev/null +++ b/libs/kdbtext/columnblob.cpp @@ -0,0 +1,211 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "columnblob.hpp" + +#include +#include +#include + +using namespace KDBText; +using namespace std; + +/*-------------------------------------------------------------------------- + * KTextColumnBlob + * one or more rows of column data + */ + +static rc_t KTextColumnBlobWhack ( KColumnBlob *self ); +static rc_t CC KTextColumnBlobRead ( const KColumnBlob *self, size_t offset, void *buffer, size_t bsize, size_t *num_read, size_t *remaining ); +static rc_t CC KTextColumnBlobReadAll ( const KColumnBlob * self, KDataBuffer * buffer, KColumnBlobCSData * opt_cs_data, size_t cs_data_size ); +static rc_t CC KTextColumnBlobValidate ( const KColumnBlob *self ); +static rc_t CC KTextColumnBlobValidateBuffer ( const KColumnBlob * self, const KDataBuffer * buffer, const KColumnBlobCSData * cs_data, size_t cs_data_size ); +static rc_t CC KTextColumnBlobIdRange ( const KColumnBlob *self, int64_t *first, uint32_t *count ); + +static KColumnBlob_vt KTextColumnBlob_vt = +{ + /* Public API */ + KTextColumnBlobWhack, + KColumnBlobBaseAddRef, + KColumnBlobBaseRelease, + KTextColumnBlobRead, + KTextColumnBlobReadAll, + KTextColumnBlobValidate, + KTextColumnBlobValidateBuffer, + KTextColumnBlobIdRange +}; + +#define CAST() assert( bself->vt == &KTextColumnBlob_vt ); ColumnBlob * self = (ColumnBlob *)bself + +ColumnBlob::ColumnBlob( const void * data, size_t size, const Column * col, int64_t id, uint64_t count ) +: m_data ( data ), + m_size( size ), + m_parent( col ), + m_firstRow( id ), + m_count( count ) +{ + dad . vt = & KTextColumnBlob_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::ColumnBlob", "ctor", "db" ); +} + +ColumnBlob::~ColumnBlob() +{ + KRefcountWhack ( & dad . refcount, "KDBText::ColumnBlob" ); +} + +// API + +static +rc_t +KTextColumnBlobWhack ( KColumnBlob *bself ) +{ + CAST(); + + delete reinterpret_cast( self ); + return 0; +} + +static +rc_t CC +KTextColumnBlobRead ( const KColumnBlob *bself, size_t offset, void *buffer, size_t bsize, size_t *num_read, size_t *remaining ) +{ + CAST(); + + rc_t rc = 0; + if ( buffer == NULL ) + { + rc = SILENT_RC ( rcDB, rcBlob, rcReading, rcBuffer, rcNull ); + } + else if ( num_read == nullptr ) + { + rc = SILENT_RC ( rcDB, rcBlob, rcReading, rcParam, rcNull ); + } + else if ( offset > self -> getSize() ) + { + rc = SILENT_RC ( rcDB, rcBlob, rcReading, rcParam, rcExcessive ); + } + else + { + size_t toRead = self -> getSize() - offset; + if ( toRead > bsize ) + { + * remaining = toRead - bsize; + toRead -= *remaining; + } + else + { + * remaining = 0; + } + memmove( buffer, (const char*)(self -> getData()) + offset, toRead ); + *num_read = toRead; + } + + return rc; +} + +static +rc_t CC +KTextColumnBlobReadAll ( const KColumnBlob * bself, KDataBuffer * buffer, KColumnBlobCSData * opt_cs_data, size_t cs_data_size ) +{ + CAST(); + + if ( opt_cs_data != NULL ) + { + memset ( opt_cs_data, 0, cs_data_size ); // we do not populate checksum here + } + + rc_t rc = 0; + if ( buffer == NULL ) + { + rc = SILENT_RC ( rcDB, rcBlob, rcReading, rcParam, rcNull ); + } + else + { + rc = KDataBufferMakeBytes ( buffer, self->getSize() ); + if ( rc == 0 && self->getSize() > 0 ) + { + memmove( buffer -> base, self -> getData(), self->getSize() ); + } + } + + return rc; +} + +static +rc_t CC +KTextColumnBlobValidate ( const KColumnBlob *self ) +{ + return 0; +} + +static +rc_t CC +KTextColumnBlobValidateBuffer ( const KColumnBlob * bself, const KDataBuffer * buffer, const KColumnBlobCSData * cs_data, size_t cs_data_size ) +{ + CAST(); + + if ( buffer == NULL ) + { + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcParam, rcNull ); + } + if ( cs_data == NULL ) + { + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcParam, rcNull ); + } + + // check the buffer's size + size_t bsize = KDataBufferBytes ( buffer ); + if ( bsize < self -> getSize() ) + { + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcData, rcInsufficient ); + } + if ( bsize > self -> getSize() ) + { + return SILENT_RC ( rcDB, rcBlob, rcValidating, rcData, rcExcessive ); + } + + return 0; +} + +static +rc_t CC +KTextColumnBlobIdRange ( const KColumnBlob * bself, int64_t *first, uint32_t *count ) +{ + CAST(); + + if ( first == NULL ) + { + return SILENT_RC ( rcDB, rcBlob, rcAccessing, rcParam, rcNull ); + } + if ( count == NULL ) + { + return SILENT_RC ( rcDB, rcBlob, rcAccessing, rcParam, rcNull ); + } + + *first = self->getIdRange().first; + *count = self->getIdRange().second; + return 0; +} diff --git a/libs/kdbtext/columnblob.hpp b/libs/kdbtext/columnblob.hpp new file mode 100644 index 000000000..6118c4c6b --- /dev/null +++ b/libs/kdbtext/columnblob.hpp @@ -0,0 +1,65 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "../libs/kdb/columnblob-base.h" + +#include + +typedef struct KTextBlob KTextBlob; +struct KTextBlob +{ + KColumnBlob dad; +}; + +namespace KDBText +{ + class Column; + + class ColumnBlob : public KTextBlob + { + public: + static void addRef( const ColumnBlob* ); + static void release( const ColumnBlob *); + + public: + ColumnBlob( const void * data, size_t size, const Column * col, int64_t id, uint64_t count ); + ~ColumnBlob(); + + const void * getData() const { return m_data; } + const size_t getSize() const { return m_size; } + + std::pair< int64_t, uint32_t > getIdRange() const{ return std::make_pair( m_firstRow, m_count ); } + + private: + const void * m_data = nullptr; + size_t m_size = 0; + const Column * m_parent = nullptr; + int64_t m_firstRow = 0; + uint64_t m_count = 0; + }; +} diff --git a/libs/kdbtext/database.cpp b/libs/kdbtext/database.cpp new file mode 100644 index 000000000..809d5b600 --- /dev/null +++ b/libs/kdbtext/database.cpp @@ -0,0 +1,687 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "database.hpp" + +#include "table.hpp" +#include "manager.hpp" +#include "metadata.hpp" + +#include +#include +#include + +#include +#include + +#include + +using namespace KDBText; +using namespace std; + +static rc_t CC KTextDatabaseWhack ( KTextDatabase *self ); +static bool CC KTextDatabaseLocked ( const KTextDatabase *self ); +static bool CC KTextDatabaseVExists ( const KTextDatabase *self, uint32_t type, const char *name, va_list args ); +static bool CC KTextDatabaseIsAlias ( const KTextDatabase *self, uint32_t type, char *resolved, size_t rsize, const char *name ); +static rc_t CC KTextDatabaseVWritable ( const KTextDatabase *self, uint32_t type, const char *name, va_list args ); +static rc_t CC KTextDatabaseOpenManagerRead ( const KTextDatabase *self, const KDBManager **mgr ); +static rc_t CC KTextDatabaseOpenParentRead ( const KTextDatabase *self, const KDatabase **par ); +static rc_t CC KTextDatabaseOpenDirectoryRead ( const KTextDatabase *self, const KDirectory **dir ); +static rc_t CC KTextDatabaseVOpenDBRead ( const KTextDatabase *self, const KDatabase **dbp, const char *name, va_list args ); +static rc_t CC KTextDatabaseVOpenTableRead ( const KTextDatabase *self, const KTable **tblp, const char *name, va_list args ); +static rc_t CC KTextDatabaseOpenMetadataRead ( const KTextDatabase *self, const KMetadata **metap ); +static rc_t CC KTextDatabaseVOpenIndexRead ( const KTextDatabase *self, const KIndex **idxp, const char *name, va_list args ); +static rc_t CC KTextDatabaseListDB ( const KTextDatabase *self, KNamelist **names ); +static rc_t CC KTextDatabaseListTbl ( struct KTextDatabase const *self, KNamelist **names ); +static rc_t CC KTextDatabaseListIdx ( struct KTextDatabase const *self, KNamelist **names ); +static rc_t CC KTextDatabaseGetPath ( KTextDatabase const *self, const char **path ); + +static KDatabase_vt KTextDatabase_vt = +{ + KTextDatabaseWhack, + KDatabaseBaseAddRef, + KDatabaseBaseRelease, + KTextDatabaseLocked, + KTextDatabaseVExists, + KTextDatabaseIsAlias, + KTextDatabaseVWritable, + KTextDatabaseOpenManagerRead, + KTextDatabaseOpenParentRead, + KTextDatabaseOpenDirectoryRead, + KTextDatabaseVOpenDBRead, + KTextDatabaseVOpenTableRead, + KTextDatabaseOpenMetadataRead, + KTextDatabaseVOpenIndexRead, + KTextDatabaseListDB, + KTextDatabaseListTbl, + KTextDatabaseListIdx, + KTextDatabaseGetPath +}; + +static char error[1024]; + +Database::Database( const KJsonObject * p_json, const Manager * p_mgr, const Database * p_parent ) +: m_mgr( p_mgr ), m_parent( p_parent ), m_json ( p_json ) +{ + dad . vt = & KTextDatabase_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Database", "ctor", "db" ); + Manager::addRef( m_mgr ); +} + +Database::~Database() +{ + Metadata::release( m_meta ); + Manager::release( m_mgr ); + KRefcountWhack ( & dad . refcount, "KDBText::Database" ); +} + +void +Database::addRef( const Database * db ) +{ + if ( db != nullptr ) + { + KDatabaseAddRef( (const KDatabase*) db ); + } +} + +void +Database::release( const Database * db ) +{ + if ( db != nullptr ) + { + KDatabaseRelease( (const KDatabase*) db ); + } +} + +const Database * +Database::openDatabase( Path & p_path ) const +{ + if ( ! p_path.empty() && p_path.front() == m_name ) + { + p_path.pop(); + if ( p_path.empty() ) + { // return a new copy of this db + Database * ret = new Database( m_json, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + if ( p_path.front() == "db" ) + { + p_path.pop(); + if ( ! p_path.empty() ) + { + auto j = m_subdbs.find( p_path.front() ); + if ( j != m_subdbs.end() ) + { + Database * ret = new Database( j -> second, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + } + } + } + return nullptr; +} + +const Database * +Database::openSubDatabase( const std::string & name ) const +{ + auto j = m_subdbs.find( name ); + if ( j != m_subdbs.end() ) + { + Database * ret = new Database( j -> second, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + return nullptr; +} + +const Table * +Database::openTable( Path & p_path ) const +{ + if ( ! p_path.empty() && p_path.front() == m_name ) + { // remove a possible db name from the front + p_path.pop(); + } + if ( ! p_path.empty() ) + { + if ( p_path.front() == "tbl" ) + { + p_path.pop(); + if ( ! p_path.empty() ) + { + auto j = m_tables.find( p_path.front() ); + if ( j != m_tables.end() ) + { + Table * ret = new Table( j -> second, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + } + } + else if ( p_path.front() == "db" ) + { + p_path.pop(); + if ( ! p_path.empty() ) + { + auto j = m_subdbs.find( p_path.front() ); + if ( j != m_subdbs.end() ) + { + Database * db = new Database( j -> second, m_mgr, this ); + if ( db != nullptr ) + { + db -> inflate( error, sizeof error); + const Table * ret = db -> openTable( p_path ); + delete db; + return ret; + } + } + } + } + } + return nullptr; +} + +const Table * +Database::openTable( const string & name ) const +{ + auto j = m_tables.find( name ); + if ( j != m_tables.end() ) + { + Table * ret = new Table( j -> second, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + return nullptr; +} + +rc_t +Database::inflate( char * p_error, size_t error_size ) +{ + rc_t rc = 0; + + const KJsonValue * name = KJsonObjectGetMember ( m_json, "name" ); + if ( name != nullptr ) + { + const char * nameStr = nullptr; + rc = KJsonGetString ( name, & nameStr ); + if ( rc == 0 ) + { + m_name = nameStr; + } + } + else + { + string_printf ( p_error, error_size, nullptr, "Database name is missing" ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + + const KJsonValue * type = KJsonObjectGetMember ( m_json, "type" ); + if ( type != nullptr ) + { + const char * typeStr = nullptr; + rc = KJsonGetString ( type, & typeStr ); + if ( rc == 0 ) + { + if ( strcmp( "database", typeStr ) != 0 ) + { + string_printf ( p_error, error_size, nullptr, "%s.type is not 'database'('%s')", m_name.c_str(), typeStr ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + else + { + string_printf ( p_error, error_size, nullptr, "%s.type is invalid", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + else + { + string_printf ( p_error, error_size, nullptr, "%s.type is missing", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + + // nested DBs + const KJsonValue * dbs = KJsonObjectGetMember ( m_json, "databases" ); + if ( dbs != nullptr ) + { + const KJsonArray * dbarr = KJsonValueToArray ( dbs ); + if ( dbarr == nullptr ) + { + string_printf ( p_error, error_size, nullptr, "%s.databases is not an array", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + + uint32_t len = KJsonArrayGetLength ( dbarr ); + for ( uint32_t i = 0; i < len; ++i ) + { + const KJsonValue * v = KJsonArrayGetElement ( dbarr, i ); + assert( v != nullptr ); + const KJsonObject * obj = KJsonValueToObject ( v ); + if( obj != nullptr ) + { + Database subdb ( obj, m_mgr, this ); // temporary, for Json verification + rc = subdb . inflate ( p_error, error_size ); + if ( rc != 0 ) + { + return rc; + } + + if ( m_subdbs.find( subdb . getName() ) != m_subdbs . end() ) + { + string_printf ( p_error, error_size, nullptr, "Duplicate nested db: %s", subdb.getName().c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + m_subdbs [ subdb . getName() ] = obj; + } + else + { // not an object + string_printf ( p_error, error_size, nullptr, "%s.databases[%i] is not an object", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + } + + // tables + const KJsonValue * tables = KJsonObjectGetMember ( m_json, "tables" ); + if ( tables != nullptr ) + { + const KJsonArray * tblarr = KJsonValueToArray ( tables ); + if ( tblarr == nullptr ) + { + string_printf ( p_error, error_size, nullptr, "%s.tables is not an array", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + uint32_t len = KJsonArrayGetLength ( tblarr ); + for ( uint32_t i = 0; i < len; ++i ) + { + const KJsonValue * v = KJsonArrayGetElement ( tblarr, i ); + assert( v != nullptr ); + const KJsonObject * obj = KJsonValueToObject ( v ); + if( obj != nullptr ) + { + Table tbl ( obj ); + rc = tbl . inflate ( p_error, error_size ); + if ( rc != 0 ) + { + return rc; + } + + if ( m_tables.find( tbl . getName() ) != m_tables . end() ) + { + string_printf ( p_error, error_size, nullptr, "Duplicate table: %s", tbl.getName().c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + m_tables [ tbl . getName() ] = obj; + } + else + { // not an object + string_printf ( p_error, error_size, nullptr, "%s.tables[%i] is not an object", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + } + + const KJsonValue * meta = KJsonObjectGetMember ( m_json, "metadata" ); + if ( meta != nullptr ) + { + const KJsonObject * obj = KJsonValueToObject ( meta ); + if( obj != nullptr ) + { + Metadata * m = new Metadata( obj ); + rc = m -> inflate( p_error, error_size ); + if ( rc != 0 ) + { + delete m; + return rc; + } + m_meta = m; + } + else + { // not an object + string_printf ( p_error, error_size, nullptr, "%s.metadata is not an object", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + + return rc; +} + +int +Database::pathType( Path & path ) const +{ //TODO: use only Json? + int ret = kptNotFound; + if ( path.size() == 1 && m_name == path.front() ) + { + ret = kptDatabase; + } + else if ( ! path.empty() ) + { + if ( m_name == path.front() ) + { + path.pop(); + if ( path.size() >= 1 ) + { + if ( path.front() == "db" ) + { + path.pop(); + const Database * db = openSubDatabase( path.front() ); + if ( db != nullptr ) + { + ret = db->pathType( path ); + delete db; + } + } + else if ( path.front() == "tbl" ) + { + const Table * tbl = openTable( path ); + if ( tbl != nullptr ) + { + ret = tbl -> pathType( path ); + delete tbl; + } + } + else if ( path.front() == "md" ) + { + if ( m_meta != nullptr ) + { + return kptMetadata; + } + } + } + } + } + + return ret; +} + +bool +Database::exists( uint32_t requested, Path & path ) const +{ //TODO: use only Json? + if ( ! path.empty() && m_name == path.front() ) + { + path.pop(); + if ( path.empty() ) + { + return requested == kptDatabase; + } + + if ( string("db") == path.front() ) + { + path.pop(); + const Database * db = openSubDatabase( path.front() ); + if ( db != nullptr ) + { + bool ret = db->exists( requested, path ); + delete db; + return ret; + } + } + else if ( string("tbl") == path.front() ) + { + const Table * tbl = openTable( path ); + if ( tbl != nullptr ) + { + bool ret = tbl -> exists( requested, path ); + delete tbl; + return ret; + } + } + else if ( string("md") == path.front() ) + { + return m_meta != nullptr; + } + } + + return false; +} + +const Metadata * +Database::openMetadata() const +{ + return m_meta; +} + + +// API functions + +#define CAST() assert( bself -> dad . vt == & KTextDatabase_vt ); const Database *self = static_cast(bself); + +static +rc_t CC +KTextDatabaseWhack ( KTextDatabase *bself ) +{ + CAST(); + delete self; + return 0; +} + +static +bool CC +KTextDatabaseLocked ( const KTextDatabase *self ) +{ + return false; +} + +static +bool CC +KTextDatabaseVExists ( const KTextDatabase *bself, uint32_t type, const char *name, va_list args ) +{ + CAST(); + Path p( name, args ); + return self -> exists( type, p ); +} + +static +bool CC +KTextDatabaseIsAlias ( const KTextDatabase *self, uint32_t type, char *resolved, size_t rsize, const char *name ) +{ + return false; +} + +static +rc_t CC +KTextDatabaseVWritable ( const KTextDatabase *self, uint32_t type, const char *name, va_list args ) +{ + return false; +} + +static +rc_t CC +KTextDatabaseOpenManagerRead ( const KTextDatabase *bself, const KDBManager **mgr ) +{ + CAST(); + + const Manager * m = self -> getManager(); + if ( m != nullptr ) + { + Manager::addRef( m ); + } + + *mgr = (const KDBManager*)m; + + return 0; +} + +static +rc_t CC +KTextDatabaseOpenParentRead ( const KTextDatabase *bself, const KDatabase **par ) +{ + CAST(); + + const Database * p = self -> getParent(); + if ( p != nullptr ) + { + Database::addRef( p ); + } + + *par = (const KDatabase*) p; + + return 0; +} + +static +rc_t CC +KTextDatabaseOpenDirectoryRead ( const KTextDatabase *self, const KDirectory **dir ) +{ + return SILENT_RC ( rcDB, rcDatabase, rcAccessing, rcDirectory, rcUnsupported ); +} + +static +rc_t CC +KTextDatabaseVOpenDBRead ( const KTextDatabase *bself, const KDatabase **dbp, const char *name, va_list args ) +{ + CAST(); + + Path p( name, args ); + if ( p.size() > 0 ) + { + const Database * db = self -> openSubDatabase( p.front() ); + if ( p.size() > 1 ) + { + p.pop(); + const Database * topdb = db; + db = self -> openDatabase( p ); + Database::release( topdb ); + } + + if ( db != nullptr ) + { + * dbp = (const KDatabase *)db; + return 0; + } + } + + return SILENT_RC( rcDB, rcDatabase, rcOpening, rcParam, rcInvalid ); +} + +static +rc_t CC +KTextDatabaseVOpenTableRead ( const KTextDatabase *bself, const KTable **tblp, const char * fmt, va_list args ) +{ + CAST(); + + string name; + rc_t rc = Path::PrintToString( fmt, args, name ); + if ( rc == 0 ) + { + const Table * t = self -> openTable( name ); + if ( t != nullptr ) + { + *tblp = (const KTable *) t; + } + else + { + rc = SILENT_RC( rcDB, rcDatabase, rcOpening, rcTable, rcNotFound ); + } + } + + return rc; +} + +static +rc_t CC +KTextDatabaseOpenMetadataRead ( const KTextDatabase *bself, const KMetadata **metap ) +{ + CAST(); + const Metadata * m = self->openMetadata(); + if ( m != nullptr ) + { + Metadata::addRef( m ); + *metap = (const KMetadata*)m; + return 0; + } + return SILENT_RC( rcDB, rcMetadata, rcOpening, rcParam, rcInvalid );; +} + +static +rc_t CC +KTextDatabaseVOpenIndexRead ( const KTextDatabase *bself, const KIndex **idxp, const char *name, va_list args ) +{ // no database-level indices here + return SILENT_RC( rcDB, rcDatabase, rcAccessing, rcIndex, rcUnsupported ); +} + +static +rc_t CC +KTextDatabaseListDB ( const KTextDatabase *bself, KNamelist **names ) +{ + CAST(); + + VNamelist * ret; + const Database::Subobjects & dbs = self -> getDatabases(); + rc_t rc = VNamelistMake( & ret, dbs.size() ); + if (rc == 0 ) + { + for (auto & key_val : dbs ) + { + VNamelistAppend ( ret, key_val . first . c_str() ); + } + *names = (KNamelist*) ret; + } + + return rc; +} + +static +rc_t CC +KTextDatabaseListTbl ( struct KTextDatabase const *bself, KNamelist **names ) +{ + CAST(); + + VNamelist * ret; + const Database::Subobjects & dbs = self -> getTables(); + rc_t rc = VNamelistMake( & ret, dbs.size() ); + if (rc == 0 ) + { + for (auto & key_val : dbs ) + { + VNamelistAppend ( ret, key_val . first . c_str() ); + } + *names = (KNamelist*) ret; + } + + return rc; +} + +static +rc_t CC +KTextDatabaseListIdx ( struct KTextDatabase const *self, KNamelist **names ) +{ // an empty list + VNamelist * ret; + rc_t rc = VNamelistMake( & ret, 0 ); + if (rc == 0 ) + { + *names = (KNamelist*) ret; + } + return rc; +} + +static +rc_t CC +KTextDatabaseGetPath ( KTextDatabase const *self, const char **path ) +{ + return SILENT_RC( rcDB, rcDatabase, rcAccessing, rcPath, rcUnsupported ); +} diff --git a/libs/kdbtext/database.hpp b/libs/kdbtext/database.hpp new file mode 100644 index 000000000..339e1d467 --- /dev/null +++ b/libs/kdbtext/database.hpp @@ -0,0 +1,97 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +typedef struct KTextDatabase KTextDatabase; +#define KDATABASE_IMPL KTextDatabase +#include "../libs/kdb/database-base.h" + +struct KTextDatabase +{ + KDatabaseBase dad; +}; + +#include "path.hpp" + +#include +#include + +#include +#include +#include + +namespace KDBText +{ + class Manager; + class Table; + class Metadata; + + class Database : public KTextDatabase + { + public: + static void addRef( const Database* ); + static void release( const Database *); + + public: + Database( const KJsonObject * json, const Manager * mgr = nullptr, const Database * parent = nullptr ); + ~Database(); + + rc_t inflate( char * error, size_t error_size ); + + const Manager * getManager() const { return m_mgr; } + const Database * getParent() const { return m_parent; } + const std::string & getName() const { return m_name; } + + const Database * openDatabase( Path & path ) const; + const Database * openSubDatabase( const std::string & name ) const; + + const Table *openTable( Path & path ) const; + const Table *openTable( const std::string & name ) const; + + int pathType( Path & ) const; + bool exists( uint32_t requested, Path & p_path ) const; + + const Metadata * openMetadata() const; + + // verified Jsons: + typedef std::map< std::string, const KJsonObject * > Subobjects; + + const Subobjects& getDatabases() const { return m_subdbs; } + const Subobjects& getTables() const { return m_tables; } + + private: + const Manager * m_mgr = nullptr; + const Database * m_parent = nullptr; + + const KJsonObject * m_json = nullptr; + std::string m_name; + + Subobjects m_subdbs; + Subobjects m_tables; + const Metadata * m_meta = nullptr; + }; +} diff --git a/libs/kdbtext/index.cpp b/libs/kdbtext/index.cpp new file mode 100644 index 000000000..6ff212c74 --- /dev/null +++ b/libs/kdbtext/index.cpp @@ -0,0 +1,153 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "index.hpp" + +#include + +#include + +using namespace KDBText; + +static rc_t KTextIndexWhack ( KIndex *self ); +// static bool CC KTextIndexLocked ( const KIndex *self ); +// static bool CC KTextIndexVExists ( const KIndex *self, uint32_t type, const char *name, va_list args ); +// static bool CC KTextIndexIsAlias ( const KIndex *self, uint32_t type, char *resolved, size_t rsize, const char *name ); +// static rc_t CC KTextIndexVWritable ( const KIndex *self, uint32_t type, const char *name, va_list args ); +// static rc_t CC KTextIndexOpenManagerRead ( const KIndex *self, const KDBManager **mgr ); +// static rc_t CC KTextIndexOpenParentRead ( const KIndex *self, const KDatabase **db ); +// static bool CC KTextIndexHasRemoteData ( const KIndex *self ); +// static rc_t CC KTextIndexOpenDirectoryRead ( const KIndex *self, const KDirectory **dir ); +// static rc_t CC KTextIndexVOpenColumnRead ( const KIndex *self, const KColumn **colp, const char *name, va_list args ); +// static rc_t CC KTextIndexOpenMetadataRead ( const KIndex *self, const KMetadata **metap ); +// static rc_t CC KTextIndexVOpenIndexRead ( const KIndex *self, const KIndex **idxp, const char *name, va_list args ); +// static rc_t CC KTextIndexGetPath ( const KIndex *self, const char **path ); +// static rc_t CC KTextIndexGetName (KIndex const *self, char const **rslt); +// static rc_t CC KTextIndexListCol ( const KIndex *self, KNamelist **names ); +// static rc_t CC KTextIndexListIdx ( const KIndex *self, KNamelist **names ); +// static rc_t CC KTextIndexMetaCompare( const KIndex *self, const KIndex *other, const char * path, bool * equal ); + +static KIndex_vt KTextIndex_vt = +{ + KTextIndexWhack, + KIndexBaseAddRef, + KIndexBaseRelease, + // KTextIndexLocked, + // KTextIndexVExists, + // KTextIndexIsAlias, + // KTextIndexVWritable, + // KTextIndexOpenManagerRead, + // KTextIndexOpenParentRead, + // KTextIndexHasRemoteData, + // KTextIndexOpenDirectoryRead, + // KTextIndexVOpenColumnRead, + // KTextIndexOpenMetadataRead, + // KTextIndexVOpenIndexRead, + // KTextIndexGetPath, + // KTextIndexGetName, + // KTextIndexListCol, + // KTextIndexListIdx, + // KTextIndexMetaCompare +}; + +#define CAST() assert( bself->vt == &KTextIndex_vt ); Index * self = (Index *)bself + +Index::Index( const KJsonObject * p_json, const Table * p_parent ) +: m_json ( p_json ), m_parent( p_parent ) +{ + dad . vt = & KTextIndex_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Index", "ctor", "db" ); +} + +Index::~Index() +{ + KRefcountWhack ( & dad . refcount, "KDBText::Index" ); +} + +void +Index::addRef( const Index * idx ) +{ + if ( idx != nullptr ) + { + KIndexAddRef( (const KIndex*) idx ); + } +} + +void +Index::release( const Index * idx ) +{ + if ( idx != nullptr ) + { + KIndexRelease( (const KIndex*) idx ); + } +} + +rc_t +Index::inflate( char * error, size_t error_size ) +{ + rc_t rc = 0; + + const KJsonValue * name = KJsonObjectGetMember ( m_json, "name" ); + if ( name != nullptr ) + { + const char * nameStr = nullptr; + rc = KJsonGetString ( name, & nameStr ); + if ( rc == 0 ) + { + m_name = nameStr; + } + } + else + { + string_printf ( error, error_size, nullptr, "Index name is missing" ); + return SILENT_RC( rcDB, rcIndex, rcCreating, rcParam, rcInvalid ); + } + + // const KJsonValue * type = KJsonObjectGetMember ( m_json, "type" ); + // if ( type != nullptr ) + // { + // const char * typeStr = nullptr; + // rc = KJsonGetString ( type, & typeStr ); + // //TBD + // } + // else + // { + // string_printf ( error, error_size, nullptr, "%s.type is missing", m_name.c_str() ); + // return SILENT_RC( rcDB, rcIndex, rcCreating, rcParam, rcInvalid ); + // } + + return rc; +} + +static +rc_t CC +KTextIndexWhack ( KIndex *bself ) +{ + CAST(); + + delete reinterpret_cast( self ); + return 0; +} \ No newline at end of file diff --git a/libs/kdbtext/index.hpp b/libs/kdbtext/index.hpp new file mode 100644 index 000000000..a489a1815 --- /dev/null +++ b/libs/kdbtext/index.hpp @@ -0,0 +1,66 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "../libs/kdb/index-base.h" + +#include +#include + +#include + +typedef struct KTextIndex KTextIndex; +struct KTextIndex +{ + KIndex dad; +}; + +namespace KDBText +{ + class Table; + + class Index : public KTextIndex + { + public: + static void addRef( const Index* ); + static void release( const Index *); + + public: + Index( const KJsonObject * p_json, const Table * parent = nullptr ); + ~Index(); + + rc_t inflate( char * error, size_t error_size ); + + const std::string & getName() const { return m_name; } + + private: + const KJsonObject * m_json = nullptr; + std::string m_name; + + const Table * m_parent = nullptr; + }; +} diff --git a/libs/kdbtext/manager.cpp b/libs/kdbtext/manager.cpp new file mode 100644 index 000000000..6c293c53a --- /dev/null +++ b/libs/kdbtext/manager.cpp @@ -0,0 +1,279 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "manager.hpp" + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +using namespace std; +using namespace KDBText; + +Manager::Manager( const KDBManager_vt & vt) +: m_isDb( false ) +{ + dad . vt = & vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Manager", "ctor", "kmgr" ); +} + +Manager::~Manager() +{ + KJsonValueWhack( m_root ); + KRefcountWhack ( & dad . refcount, "KDBText::Manager" ); +} + +void +Manager::addRef( const Manager * mgr ) +{ + if ( mgr != nullptr ) + { + KDBManagerAddRef( (const KDBManager*) mgr ); + } +} + +void +Manager::release( const Manager * mgr ) +{ + if ( mgr != nullptr ) + { + KDBManagerRelease( (const KDBManager*) mgr ); + } +} + +rc_t +Manager::parse( const char * input, char * error, size_t error_size ) +{ + rc_t rc = KJsonValueMake ( & m_root, input, error, error_size ); + if ( rc != 0 ) + { + return rc; + } + + if ( KJsonGetValueType ( m_root ) != jsObject ) + { + string_printf ( error, error_size, nullptr, "Json root is invalid" ); + return SILENT_RC( rcDB, rcMgr, rcCreating, rcParam, rcInvalid ); + } + + const KJsonObject * rootObj = KJsonValueToObject ( m_root ); + assert( rootObj != nullptr ); + + const KJsonValue * type = KJsonObjectGetMember ( rootObj, "type" ); + if ( type != nullptr ) + { + const char * typeStr = nullptr; + rc = KJsonGetString ( type, & typeStr ); + if ( rc == 0 ) + { // determine the type of the root object (which is then discarded) + if ( strcmp( typeStr, "database") == 0 ) + { + rc = Database( rootObj, this ) . inflate( error, error_size ); + if ( rc == 0 ) + { + m_isDb = true; + } + } + else if ( strcmp( typeStr, "table") == 0 ) + { + rc = Table( rootObj ) . inflate( error, error_size ); + if ( rc == 0 ) + { + m_isDb = false; + } + } + } + } + + return rc; +} + +const Database * +Manager::getRootDatabase() const +{ + Database * db = new Database( KJsonValueToObject ( m_root ), this ); + char error[1024]; + if ( db -> inflate( error, sizeof error ) == 0 ) + { + return db; + } + delete db; + return nullptr; +} + +const Table * +Manager::getRootTable() const +{ + Table * tbl = new Table( KJsonValueToObject ( m_root ), this, nullptr ); + char error[1024]; + if ( tbl -> inflate( error, sizeof error ) == 0 ) + { + return tbl; + } + delete tbl; + return nullptr; +} + + +int +Manager::pathType( const Path & p_path ) const +{ + Path path( p_path ); + int ret = kptNotFound; + + //TODO: do it without creating a DB/Tbl, by walking the Json + + if ( m_isDb ) + { + const Database * db = getRootDatabase(); + if ( db != nullptr ) + { + ret = db -> pathType( path ); + } + delete db; + } + else + { + const Table * tbl = getRootTable(); + if ( tbl != nullptr ) + { + ret = tbl -> pathType( path ); + } + delete tbl; + } + return ret; +} + +bool +Manager::exists( uint32_t requested, const Path & p_path ) const +{ + Path path( p_path ); + bool ret = false; + + //TODO: do it without creating a DB/Tbl, by walking the Json + if ( m_isDb ) + { + const Database * db = getRootDatabase(); + if ( db != nullptr ) + { + ret = db -> exists( requested, path ); + } + delete db; + } + else + { + const Table * tbl = getRootTable(); + if ( tbl ) + { + ret = tbl -> exists( requested, path ); + } + delete tbl; + } + return ret; +} + +rc_t +Manager::writable( const Path & p_path ) const +{ + //if the object is valid return rcReadOnly, otherwise rcNotFound + return SILENT_RC ( rcDB, rcPath, rcAccessing, rcPath, pathType( p_path ) != kptNotFound ? rcReadonly : rcNotFound ); +} + +rc_t +Manager::openDatabase( const Path & p_path, const Database *& p_db ) const +{ + if ( ! p_path.empty() ) + { + if ( m_isDb ) + { + Path path ( p_path ); + const Database * root = getRootDatabase(); + if ( root != nullptr ) + { + const Database * db = root -> openDatabase( path ); // could be same as root + if ( db != root ) + { + KDatabaseRelease( (const KDatabase *)root ); + p_db = db; + return 0; + } + else + { + p_db = root; + return 0; + } + } + } + } + return SILENT_RC (rcDB, rcMgr, rcOpening, rcType, rcInvalid); +} + +rc_t +Manager::openTable( const Path & p_path, const Table *& p_tbl ) const +{ + if ( ! p_path.empty() ) + { + if ( m_isDb ) + { + Path path ( p_path ); + const Database * root = getRootDatabase(); + if ( root != nullptr ) + { + const Table * tbl = root -> openTable( path ); + KDatabaseRelease( (const KDatabase *)root ); + if ( tbl != nullptr ) + { + p_tbl = tbl; + return 0; + } + } + } + else + { + Path path ( p_path ); + const Table * tbl = getRootTable(); + if ( tbl != nullptr ) + { + if ( tbl -> getName() == p_path.front() ) + { + p_tbl = tbl; + return 0; + } + } + KTableRelease( (const KTable*)tbl ); + } + } + return SILENT_RC (rcDB, rcMgr, rcOpening, rcType, rcInvalid); +} diff --git a/libs/kdbtext/manager.hpp b/libs/kdbtext/manager.hpp new file mode 100644 index 000000000..ae6ebe8fc --- /dev/null +++ b/libs/kdbtext/manager.hpp @@ -0,0 +1,68 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "api-manager.hpp" + +#include + +#include "database.hpp" +#include "table.hpp" +#include "path.hpp" + +namespace KDBText +{ + class Manager : public KDBManager + { + public: + static void addRef( const Manager* ); + static void release( const Manager *); + + public: + Manager( const KDBManager_vt& ); + ~Manager(); + + rc_t parse( const char * input, char * error, size_t error_size ); + + const Database * getRootDatabase() const; + const Table * getRootTable() const; + + int pathType( const Path & path ) const; + + bool exists( uint32_t requested, const Path & ) const; + + rc_t writable( const Path & ) const; + + rc_t openDatabase( const Path &, const Database *& ) const; + rc_t openTable( const Path &, const Table *& ) const; + + private: + KJsonValue * m_root = nullptr; //TODO: convert to KJsonObject + bool m_isDb; + }; +} + diff --git a/libs/kdbtext/metadata.cpp b/libs/kdbtext/metadata.cpp new file mode 100644 index 000000000..5ec90e089 --- /dev/null +++ b/libs/kdbtext/metadata.cpp @@ -0,0 +1,153 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "metadata.hpp" + +#include +#include + +#include + +using namespace KDBText; + +static rc_t KTextMetadataWhack ( KMetadata *self ); +// static bool CC KTextMetadataLocked ( const KMetadata *self ); +// static bool CC KTextMetadataVExists ( const KMetadata *self, uint32_t type, const char *name, va_list args ); +// static bool CC KTextMetadataIsAlias ( const KMetadata *self, uint32_t type, char *resolved, size_t rsize, const char *name ); +// static rc_t CC KTextMetadataVWritable ( const KMetadata *self, uint32_t type, const char *name, va_list args ); +// static rc_t CC KTextMetadataOpenManagerRead ( const KMetadata *self, const KDBManager **mgr ); +// static rc_t CC KTextMetadataOpenParentRead ( const KMetadata *self, const KDatabase **db ); +// static bool CC KTextMetadataHasRemoteData ( const KMetadata *self ); +// static rc_t CC KTextMetadataOpenDirectoryRead ( const KMetadata *self, const KDirectory **dir ); +// static rc_t CC KTextMetadataVOpenColumnRead ( const KMetadata *self, const KColumn **colp, const char *name, va_list args ); +// static rc_t CC KTextMetadataOpenMetadataRead ( const KMetadata *self, const KMetadata **metap ); +// static rc_t CC KTextMetadataVOpenMetadataRead ( const KMetadata *self, const KMetadata **idxp, const char *name, va_list args ); +// static rc_t CC KTextMetadataGetPath ( const KMetadata *self, const char **path ); +// static rc_t CC KTextMetadataGetName (KMetadata const *self, char const **rslt); +// static rc_t CC KTextMetadataListCol ( const KMetadata *self, KNamelist **names ); +// static rc_t CC KTextMetadataListIdx ( const KMetadata *self, KNamelist **names ); +// static rc_t CC KTextMetadataMetaCompare( const KMetadata *self, const KMetadata *other, const char * path, bool * equal ); + +static KMetadata_vt KTextMetadata_vt = +{ + KTextMetadataWhack, + KMetadataBaseAddRef, + KMetadataBaseRelease, + // KTextMetadataLocked, + // KTextMetadataVExists, + // KTextMetadataIsAlias, + // KTextMetadataVWritable, + // KTextMetadataOpenManagerRead, + // KTextMetadataOpenParentRead, + // KTextMetadataHasRemoteData, + // KTextMetadataOpenDirectoryRead, + // KTextMetadataVOpenColumnRead, + // KTextMetadataOpenMetadataRead, + // KTextMetadataVOpenMetadataRead, + // KTextMetadataGetPath, + // KTextMetadataGetName, + // KTextMetadataListCol, + // KTextMetadataListIdx, + // KTextMetadataMetaCompare +}; + +#define CAST() assert( bself->vt == &KTextMetadata_vt ); Metadata * self = (Metadata *)bself + +Metadata::Metadata( const KJsonObject * p_json ) : m_json ( p_json ) +{ + dad . vt = & KTextMetadata_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Metadata", "ctor", "db" ); +} + +Metadata::~Metadata() +{ + KRefcountWhack ( & dad . refcount, "KDBText::Metadata" ); +} + +void +Metadata::addRef( const Metadata * meta ) +{ + if ( meta != nullptr ) + { + KMetadataAddRef( (const KMetadata*) meta ); + } +} + +void +Metadata::release( const Metadata * meta ) +{ + if ( meta != nullptr ) + { + KMetadataRelease( (const KMetadata*) meta ); + } +} + +rc_t +Metadata::inflate( char * error, size_t error_size ) +{ + rc_t rc = 0; + + const KJsonValue * name = KJsonObjectGetMember ( m_json, "name" ); + if ( name != nullptr ) + { + const char * nameStr = nullptr; + rc = KJsonGetString ( name, & nameStr ); + if ( rc == 0 ) + { + m_name = nameStr; + } + } + else + { + string_printf ( error, error_size, nullptr, "Metadata name is missing" ); + return SILENT_RC( rcDB, rcMetadata, rcCreating, rcParam, rcInvalid ); + } + + // const KJsonValue * type = KJsonObjectGetMember ( m_json, "type" ); + // if ( type != nullptr ) + // { + // const char * typeStr = nullptr; + // rc = KJsonGetString ( type, & typeStr ); + // //TBD + // } + // else + // { + // string_printf ( error, error_size, nullptr, "%s.type is missing", m_name.c_str() ); + // return SILENT_RC( rcDB, rcMetadata, rcCreating, rcParam, rcInvalid ); + // } + + return rc; +} + +static +rc_t CC +KTextMetadataWhack ( KMetadata *bself ) +{ + CAST(); + + delete reinterpret_cast( self ); + return 0; +} \ No newline at end of file diff --git a/libs/kdbtext/metadata.hpp b/libs/kdbtext/metadata.hpp new file mode 100644 index 000000000..2cdb0fe11 --- /dev/null +++ b/libs/kdbtext/metadata.hpp @@ -0,0 +1,62 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "../libs/kdb/meta-base.h" + +#include +#include + +#include + +typedef struct KTextMetadata KTextMetadata; +struct KTextMetadata +{ + KMetadata dad; +}; + +namespace KDBText +{ + class Metadata : public KTextMetadata + { + public: + static void addRef( const Metadata* ); + static void release( const Metadata *); + + public: + Metadata( const KJsonObject * p_json ); + ~Metadata(); + + rc_t inflate( char * error, size_t error_size ); + + const std::string & getName() const { return m_name; } + + private: + const KJsonObject * m_json = nullptr; + std::string m_name; + }; +} diff --git a/libs/kdbtext/path.cpp b/libs/kdbtext/path.cpp new file mode 100644 index 000000000..0b29d4364 --- /dev/null +++ b/libs/kdbtext/path.cpp @@ -0,0 +1,99 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "path.hpp" + +#include +#include +#include + +#include + +#include + +using namespace std; +using namespace KDBText; + +rc_t +Path::PrintToString( const char *fmt, va_list args, string & out ) +{ + KDataBuffer buf; + rc_t rc = KDataBufferMake ( & buf, 8, 0 ); + if ( rc != 0 ) + { + return false; + } + + rc = KDataBufferVPrintf ( & buf, fmt, args ); + if ( rc != 0 ) + { + return rc; + } + + out = string((const char *) (buf . base)); // will be 0-terminated + + rc = KDataBufferWhack ( & buf ); + if ( rc != 0 ) + { + return rc; + } + + return rc; +} + +void +Path::fromString( const string & p_source ) +{ + string word; + istringstream in( p_source ); + while( getline(in, word, '/') ) + { + push( word ); + } +} + +Path::Path( const std::string & p_source ) +{ + fromString( p_source ); +} + +Path::Path( const char *fmt, va_list args ) +{ + string p; + PrintToString( fmt, args, p ); + fromString( p ); +} + +Path::Path( const struct VPath * p_path ) +{ + const String * str; + rc_t rc = VPathMakeString ( p_path, &str ); + if ( rc == 0 ) + { + fromString( string( str -> addr, str -> size ) ); + StringWhack( str ); + } +} diff --git a/libs/kdbtext/path.hpp b/libs/kdbtext/path.hpp new file mode 100644 index 000000000..d494d4158 --- /dev/null +++ b/libs/kdbtext/path.hpp @@ -0,0 +1,52 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include +#include + +#include + +struct VPath; + +namespace KDBText +{ + // converts a string path with '/' as the separator into a vector of individual elements + class Path : public std::queue< std::string > + { + public: + static rc_t PrintToString( const char *fmt, va_list args, std::string & out ); + + public: + Path( const std::string & p_source ); + Path( const char *fmt, va_list args ); + Path( const struct VPath * path ); + + private: + void fromString( const std::string & p_source ); + }; +} diff --git a/libs/kdbtext/table.cpp b/libs/kdbtext/table.cpp new file mode 100644 index 000000000..bdd229192 --- /dev/null +++ b/libs/kdbtext/table.cpp @@ -0,0 +1,591 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#include "table.hpp" + +#include "metadata.hpp" +#include "manager.hpp" +#include "database.hpp" +#include "column.hpp" + +#include +#include + +#include +#include + +using namespace KDBText; +using namespace std; + +static rc_t KTextTableWhack ( KTable *self ); +static bool CC KTextTableLocked ( const KTable *self ); +static bool CC KTextTableVExists ( const KTable *self, uint32_t type, const char *name, va_list args ); +static bool CC KTextTableIsAlias ( const KTable *self, uint32_t type, char *resolved, size_t rsize, const char *name ); +static rc_t CC KTextTableVWritable ( const KTable *self, uint32_t type, const char *name, va_list args ); +static rc_t CC KTextTableOpenManagerRead ( const KTable *self, const KDBManager **mgr ); +static rc_t CC KTextTableOpenParentRead ( const KTable *self, const KDatabase **db ); +static bool CC KTextTableHasRemoteData ( const KTable *self ); +static rc_t CC KTextTableOpenDirectoryRead ( const KTable *self, const KDirectory **dir ); +static rc_t CC KTextTableVOpenColumnRead ( const KTable *self, const KColumn **colp, const char *name, va_list args ); +static rc_t CC KTextTableOpenMetadataRead ( const KTable *self, const KMetadata **metap ); +static rc_t CC KTextTableVOpenIndexRead ( const KTable *self, const KIndex **idxp, const char *name, va_list args ); +static rc_t CC KTextTableGetPath ( const KTable *self, const char **path ); +static rc_t CC KTextTableGetName (KTable const *self, char const **rslt); +static rc_t CC KTextTableListCol ( const KTable *self, KNamelist **names ); +static rc_t CC KTextTableListIdx ( const KTable *self, KNamelist **names ); +static rc_t CC KTextTableMetaCompare( const KTable *self, const KTable *other, const char * path, bool * equal ); + +static KTable_vt KTextTable_vt = +{ + KTextTableWhack, + KTableBaseAddRef, + KTableBaseRelease, + KTextTableLocked, + KTextTableVExists, + KTextTableIsAlias, + KTextTableVWritable, + KTextTableOpenManagerRead, + KTextTableOpenParentRead, + KTextTableHasRemoteData, + KTextTableOpenDirectoryRead, + KTextTableVOpenColumnRead, + KTextTableOpenMetadataRead, + KTextTableVOpenIndexRead, + KTextTableGetPath, + KTextTableGetName, + KTextTableListCol, + KTextTableListIdx, + KTextTableMetaCompare +}; + +#define CAST() assert( bself->vt == &KTextTable_vt ); Table * self = (Table *)bself + +static char error[1024]; + +Table::Table( const KJsonObject * p_json, const Manager * p_mgr, const Database * p_parent ) +: m_mgr( p_mgr ), m_parent( p_parent ), m_json ( p_json ) +{ + dad . vt = & KTextTable_vt; + KRefcountInit ( & dad . refcount, 1, "KDBText::Table", "ctor", "db" ); + Manager::addRef( m_mgr ); +} + +Table::~Table() +{ + Metadata::release( m_meta ); + Manager::release( m_mgr ); + KRefcountWhack ( & dad . refcount, "KDBText::Table" ); +} + +void +Table::addRef( const Table * tbl ) +{ + if ( tbl != nullptr ) + { + KTableAddRef( (const KTable*) tbl ); + } +} + +void +Table::release( const Table * tbl ) +{ + if ( tbl != nullptr ) + { + KTableRelease( (const KTable*) tbl ); + } +} + +const Column * +Table::openColumn( const string & name ) const +{ + auto j = m_columns.find( name ); + if ( j != m_columns.end() ) + { + Column * ret = new Column( j -> second, m_mgr, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + return nullptr; +} + +const Index * +Table::openIndex( const string & name ) const +{ + auto j = m_indexes.find( name ); + if ( j != m_indexes.end() ) + { + Index * ret = new Index( j -> second, this ); + ret -> inflate( error, sizeof error ); + return ret; + } + return nullptr; +} + +rc_t +Table::inflate( char * p_error, size_t p_error_size ) +{ + rc_t rc = 0; + + const KJsonValue * name = KJsonObjectGetMember ( m_json, "name" ); + if ( name != nullptr ) + { + const char * nameStr = nullptr; + rc = KJsonGetString ( name, & nameStr ); + if ( rc == 0 ) + { + m_name = nameStr; + } + } + else + { + string_printf ( p_error, p_error_size, nullptr, "Table name is missing" ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + + const KJsonValue * type = KJsonObjectGetMember ( m_json, "type" ); + if ( type != nullptr ) + { + const char * typeStr = nullptr; + rc = KJsonGetString ( type, & typeStr ); + if ( rc == 0 ) + { + if ( strcmp( "table", typeStr ) != 0 ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.type is not 'table'('%s')", m_name.c_str(), typeStr ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + } + else + { + string_printf ( p_error, p_error_size, nullptr, "%s.type is invalid", m_name.c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + } + else + { + string_printf ( p_error, p_error_size, nullptr, "%s.type is missing", m_name.c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + + // Columns + const KJsonValue * columns = KJsonObjectGetMember ( m_json, "columns" ); + if ( columns != nullptr ) + { + const KJsonArray * colarr = KJsonValueToArray ( columns ); + if ( colarr == nullptr ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.columns is not an array", m_name.c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + + uint32_t len = KJsonArrayGetLength ( colarr ); + for ( uint32_t i = 0; i < len; ++i ) + { + const KJsonValue * v = KJsonArrayGetElement ( colarr, i ); + assert( v != nullptr ); + const KJsonObject * obj = KJsonValueToObject ( v ); + if( obj != nullptr ) + { + Column col( obj ); // a temporary for Json verification + rc = col . inflate ( p_error, p_error_size ); + if ( rc != 0 ) + { + return rc; + } + + if ( m_columns.find( col . getName() ) != m_columns . end() ) + { + string_printf ( p_error, p_error_size, nullptr, "Duplicate column: %s", col . getName().c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + m_columns [ col . getName() ] = obj; + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.columns[%i] is not an object", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + } + } + + // Indexes + const KJsonValue * indexes = KJsonObjectGetMember ( m_json, "indexes" ); + if ( indexes != nullptr ) + { + const KJsonArray * idxarr = KJsonValueToArray ( indexes ); + if ( idxarr == nullptr ) + { + string_printf ( p_error, p_error_size, nullptr, "%s.indexes is not an array", m_name.c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + + uint32_t len = KJsonArrayGetLength ( idxarr ); + for ( uint32_t i = 0; i < len; ++i ) + { + const KJsonValue * v = KJsonArrayGetElement ( idxarr, i ); + assert( v != nullptr ); + const KJsonObject * obj = KJsonValueToObject ( v ); + if( obj != nullptr ) + { + Index idx( obj ); // a temporary for Json verification + rc = idx . inflate ( p_error, p_error_size ); + if ( rc != 0 ) + { + return rc; + } + + if ( m_indexes.find( idx . getName() ) != m_indexes . end() ) + { + string_printf ( p_error, p_error_size, nullptr, "Duplicate index: %s", idx.getName().c_str() ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + m_indexes [ idx . getName() ] = obj; + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.indexes[%i] is not an object", m_name.c_str(), i ); + return SILENT_RC( rcDB, rcTable, rcCreating, rcParam, rcInvalid ); + } + } + } + + // metadata + const KJsonValue * meta = KJsonObjectGetMember ( m_json, "metadata" ); + if ( meta != nullptr ) + { + const KJsonObject * obj = KJsonValueToObject ( meta ); + if( obj != nullptr ) + { + Metadata * m = new Metadata( obj ); + rc = m -> inflate( p_error, p_error_size ); + if ( rc != 0 ) + { + delete m; + return rc; + } + m_meta = m; + } + else + { // not an object + string_printf ( p_error, p_error_size, nullptr, "%s.metadata is not an object", m_name.c_str() ); + return SILENT_RC( rcDB, rcDatabase, rcCreating, rcParam, rcInvalid ); + } + } + + return rc; +} + +int +Table::pathType( Path & path ) const +{ + if ( ! path.empty() ) + { + if ( path.front() == m_name ) + { + path.pop(); + if ( path.empty() ) + { + return kptTable; + } + if ( path.front() == "col" ) + { + path.pop(); + if ( path.size() == 1 && hasColumn( path.front() ) ) + { + return kptColumn; + } + } + else if ( path.front() == "md" ) + { + if ( path.size() == 1 && m_meta != nullptr ) + { + return kptMetadata; + } + } + else if ( path.front() == "idx" ) + { + path.pop(); + if ( path.size() == 1 && hasIndex( path.front() ) ) + { + return kptIndex; + } + } + } + } + return kptNotFound; +} + +bool +Table::exists( uint32_t requested, Path & path ) const +{ + if ( ! path.empty() && m_name == path.front() ) + { + path.pop(); + if ( path.empty() ) + { + return requested == kptTable; + } + } + + if ( ! path.empty() ) + { + if ( path.front() == "idx" ) + { + path.pop(); + if ( path.size() == 1 ) + { + return hasIndex( path.front() ); + } + } + else if ( path.front() == "col" ) + { + path.pop(); + if ( path.size() == 1 ) + { + return hasColumn( path.front() ); + } + } + else if ( path.front() == "md" ) + { + return path.size() == 1 && m_meta != nullptr; + } + } + + return false; +} + +static +rc_t CC +KTextTableWhack ( KTable *bself ) +{ + CAST(); + + delete reinterpret_cast( self ); + return 0; +} + +static +bool CC +KTextTableLocked ( const KTable *self ) +{ + return false; +} + +static +bool CC +KTextTableVExists ( const KTable *bself, uint32_t type, const char *name, va_list args ) +{ + CAST(); + Path p ( name, args ); + return self -> exists( type, p ); +} + +static +bool CC +KTextTableIsAlias ( const KTable *self, uint32_t type, char *resolved, size_t rsize, const char *name ) +{ + return false; +} + +static +rc_t CC +KTextTableVWritable ( const KTable *self, uint32_t type, const char *name, va_list args ) +{ + return false; +} + +static +rc_t CC +KTextTableOpenManagerRead ( const KTable *bself, const KDBManager **mgr ) +{ + CAST(); + + const Manager * m = self -> getManager(); + if ( m != nullptr ) + { + Manager::addRef( m ); + } + + *mgr = (const KDBManager*)m; + + return 0; +} + +static +rc_t CC +KTextTableOpenParentRead ( const KTable *bself, const KDatabase **par ) +{ + CAST(); + + const Database * p = self -> getParent(); + if ( p != nullptr ) + { + Database::addRef( p ); + } + + *par = (const KDatabase*) p; + + return 0; +} + +static +bool CC +KTextTableHasRemoteData ( const KTable *self ) +{ + return false; +} + +static +rc_t CC +KTextTableOpenDirectoryRead ( const KTable *self, const KDirectory **dir ) +{ + return SILENT_RC ( rcDB, rcTable, rcAccessing, rcDirectory, rcUnsupported ); +} + +static +rc_t CC +KTextTableVOpenColumnRead ( const KTable * bself, const KColumn **colp, const char *fmt, va_list args ) +{ + CAST(); + + string name; + rc_t rc = Path::PrintToString( fmt, args, name ); + if ( rc == 0 ) + { + const Column * col = self -> openColumn( name ); + if ( col != nullptr ) + { + *colp = (const KColumn*)col; + } + else + { + rc = SILENT_RC( rcDB, rcTable, rcOpening, rcColumn, rcNotFound ); + } + } + return rc; +} + +static +rc_t CC +KTextTableOpenMetadataRead ( const KTable * bself, const KMetadata **metap ) +{ + CAST(); + const Metadata * m = self->openMetadata(); + if ( m != nullptr ) + { + Metadata::addRef( m ); + *metap = (const KMetadata*)m; + return 0; + } + return SILENT_RC( rcDB, rcTable, rcOpening, rcMetadata, rcInvalid );; +} + +static +rc_t CC +KTextTableVOpenIndexRead ( const KTable * bself, const KIndex **idxp, const char *fmt, va_list args ) +{ + CAST(); + + string name; + rc_t rc = Path::PrintToString( fmt, args, name ); + if ( rc == 0 ) + { + const Index * idx = self->openIndex( name ); + if ( idx != nullptr ) + { + *idxp = (const KIndex*)idx; + return 0; + } + } + return SILENT_RC( rcDB, rcTable, rcOpening, rcIndex, rcInvalid );; +} + +static +rc_t CC +KTextTableGetPath ( const KTable *self, const char **path ) +{ + return SILENT_RC ( rcDB, rcTable, rcAccessing, rcPath, rcUnsupported ); +} + +static +rc_t CC +KTextTableGetName (KTable const * bself, char const **rslt) +{ + CAST(); + + *rslt = self->getName().c_str(); + return 0; +} + +static +rc_t CC +KTextTableListCol ( const KTable * bself, KNamelist **names ) +{ + CAST(); + + VNamelist * ret; + const Table::Subobjects & cols = self -> getColumns(); + rc_t rc = VNamelistMake( & ret, cols.size() ); + if (rc == 0 ) + { + for (auto & key_val : cols ) + { + VNamelistAppend ( ret, key_val . first . c_str() ); + } + *names = (KNamelist*) ret; + } + + return rc; +} + +static +rc_t CC +KTextTableListIdx ( const KTable * bself, KNamelist **names ) +{ + CAST(); + + VNamelist * ret; + const Table::Subobjects & cols = self -> getIndexes(); + rc_t rc = VNamelistMake( & ret, cols.size() ); + if (rc == 0 ) + { + for (auto & key_val : cols ) + { + VNamelistAppend ( ret, key_val . first . c_str() ); + } + *names = (KNamelist*) ret; + } + + return rc; +} + +static +rc_t CC +KTextTableMetaCompare( const KTable *self, const KTable *other, const char * path, bool * equal ) +{ + return SILENT_RC ( rcDB, rcTable, rcComparing, rcMetadata, rcUnsupported ); +} + diff --git a/libs/kdbtext/table.hpp b/libs/kdbtext/table.hpp new file mode 100644 index 000000000..bff5c3fd7 --- /dev/null +++ b/libs/kdbtext/table.hpp @@ -0,0 +1,99 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +#pragma once + +#include "../libs/kdb/table-base.h" + +#include "path.hpp" +#include "index.hpp" + +#include +#include + +#include +#include + +typedef struct KTextTable KTextTable; +struct KTextTable +{ + KTable dad; +}; + +namespace KDBText +{ + class Metadata; + class Manager; + class Database; + class Column; + + class Table : public KTextTable + { + public: + static void addRef( const Table* ); + static void release( const Table *); + + public: + Table( const KJsonObject * p_json, const Manager * mgr = nullptr, const Database * parent = nullptr ); + ~Table(); + + rc_t inflate( char * error, size_t error_size ); + + const Manager * getManager() const { return m_mgr; } + const Database * getParent() const { return m_parent; } + const std::string & getName() const { return m_name; } + + bool hasColumn( const std::string& name ) const { return m_columns.find(name) != m_columns.end(); } + const Column * openColumn( const std::string& name ) const; + + bool hasIndex( const std::string& name ) const { return m_indexes.find(name) != m_indexes.end(); } + const Index * openIndex( const std::string& name ) const; + + int pathType( Path & ) const; + bool exists( uint32_t requested, Path & p_path ) const; + + const Metadata * openMetadata() const { return m_meta; } + + // verified Jsons: + typedef std::map< std::string, const KJsonObject * > Subobjects; + + const Subobjects& getColumns() const { return m_columns; } + const Subobjects& getIndexes() const { return m_indexes; } + + private: + const Manager * m_mgr = nullptr; + const Database * m_parent = nullptr; + + const KJsonObject * m_json = nullptr; + std::string m_name; + + // verified Jsons + Subobjects m_columns; + Subobjects m_indexes; + + const Metadata * m_meta = nullptr; + }; +} diff --git a/libs/kfs/win/sysdir.c b/libs/kfs/win/sysdir.c index b1e2813e1..f358b038c 100644 --- a/libs/kfs/win/sysdir.c +++ b/libs/kfs/win/sysdir.c @@ -563,10 +563,11 @@ uint32_t KSysDirFullFSPathType ( const wchar_t * path ) DWORD status = GetLastError (); switch( status ) { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: case ERROR_BAD_NETPATH: case ERROR_BAD_NET_NAME: + case ERROR_DIRECTORY: + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: /* try to follow the path, section by section if a section cannot be found try to resolve it as MS-shell-link ( .lnk file ) */ @@ -2426,6 +2427,9 @@ rc_t CC KSysDirOpenFileWrite ( KSysDir *self, if ( file_handle == INVALID_HANDLE_VALUE ) { + if(path[0] != '%' && path[1] != 's' && path[2] != '/' && path[3] != 'm' + && path[4] != 'd' && path[5] != '5' && path[6] != '\0') + /* don't print error when kdbmeta tries to open md5 file */ rc = print_error_for( GetLastError(), file_name, "CreateFileW", rcAccessing, klogErr ); } diff --git a/libs/klib/printf.c b/libs/klib/printf.c index e151d0e60..d6d7a29fa 100644 --- a/libs/klib/printf.c +++ b/libs/klib/printf.c @@ -2650,7 +2650,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, case sptNulTermString: /* special initialization to flag size/length unknown */ - StringInit ( & S, args [ arg_idx ] . s, (uint32_t)0, (uint32_t)-1 ); + StringInit ( & S, args [ arg_idx ] . s, 0, (uint32_t)-1 ); /* IF THE STRING IS INDEXED OR MAY NEED LEFT ALIGNMENT */ if ( f . u . f . start_idx != 0 || f . u . f . select_len != 0 || @@ -2904,7 +2904,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, #endif /* create text string */ assert(int_len >= 0); - StringInit ( & S, & text [ i ], (uint32_t)int_len, (uint32_t)int_len ); + StringInit ( & S, & text [ i ], int_len, (uint32_t)int_len ); /* zero-fill amount */ assert(f.u.f.precision == -1 || f . u . f . precision >= 0); @@ -3097,7 +3097,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, ); #endif assert(dst_len >= 0); - StringInit ( & S, text, (uint32_t)dst_len, (uint32_t)dst_len ); + StringInit ( & S, text, dst_len, (uint32_t)dst_len ); f.u.f.precision = dst_len; break; @@ -3223,7 +3223,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, } assert(dst_len >= 0); - StringInit ( & S, text, (uint32_t)dst_len, (uint32_t)dst_len ); + StringInit ( & S, text, dst_len, (uint32_t)dst_len ); break; case spfRC: @@ -3231,7 +3231,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, assert ( FITS_INTO_INT32 ( temp_size_t ) ); dst_len = (int32_t)temp_size_t; assert(dst_len >= 0); - StringInit ( & S, text, (uint32_t)dst_len, (uint32_t)dst_len ); + StringInit ( & S, text, dst_len, (uint32_t)dst_len ); break; case spfOSErr: @@ -3239,7 +3239,7 @@ rc_t structured_print_engine ( KBufferedWrtHandler *out, assert ( FITS_INTO_INT32 ( temp_size_t ) ); dst_len = (uint32_t)temp_size_t; assert(dst_len >= 0); - StringInit ( & S, text, (uint32_t)dst_len, (uint32_t)dst_len ); + StringInit ( & S, text, dst_len, (uint32_t)dst_len ); break; default: diff --git a/libs/klib/report-klib.c b/libs/klib/report-klib.c index 16deb52f6..e2629124d 100644 --- a/libs/klib/report-klib.c +++ b/libs/klib/report-klib.c @@ -517,7 +517,7 @@ static rc_t ReportRun(int indent, rc_t rc_in) { { name = val = "not found"; } report(indent + 1, "Home", 2, "name", 's', name, "value", 's', val); } -#endif // WINDOWS +#endif /* WINDOWS */ if ( self -> report_cwd != NULL ) rc = ( * self -> report_cwd ) ( & report_funcs, indent + 1 ); diff --git a/libs/klib/text.c b/libs/klib/text.c index 54bd4faa2..1a1eced8c 100644 --- a/libs/klib/text.c +++ b/libs/klib/text.c @@ -174,7 +174,7 @@ LIB_EXPORT String * CC StringTrim ( const String * str, String * trimmed ) assert ( FITS_INTO_INT32 ( end - i ) ); assert ( FITS_INTO_INT32 ( len - ( i + sz - end ) ) ); - StringInit ( trimmed, & addr [ i ], (uint32_t) (end - i), (uint32_t)(len - ( i + sz - end )) ); + StringInit ( trimmed, & addr [ i ], end - i, (uint32_t)(len - ( i + sz - end )) ); } } diff --git a/libs/klib/token.c b/libs/klib/token.c index 162a095c6..3fb990d45 100644 --- a/libs/klib/token.c +++ b/libs/klib/token.c @@ -266,7 +266,7 @@ LIB_EXPORT rc_t CC KTokenToVersion ( const KToken *self, uint32_t *vp ) case eMajMinRel: dot = string_rchr ( start, end - start, '.' ) + 1; assert ( dot > start && dot < end ); - StringInit ( & str, dot, (size_t)( end - dot ), (uint32_t)( end - dot ) ); + StringInit ( & str, dot, end - dot, (uint32_t)( end - dot ) ); rc = StringConvertDecimal ( & str, & i, 16 ); if ( rc != 0 ) break; @@ -281,7 +281,7 @@ LIB_EXPORT rc_t CC KTokenToVersion ( const KToken *self, uint32_t *vp ) return RC ( rcVDB, rcToken, rcConverting, rcType, rcIncorrect ); if ( ++ dot == end ) return RC ( rcVDB, rcToken, rcConverting, rcType, rcIncorrect ); - StringInit ( & str, dot, (size_t)( end - dot ), (uint32_t)( end - dot ) ); + StringInit ( & str, dot, end - dot, (uint32_t)( end - dot ) ); rc = StringConvertDecimal ( & str, & i, 8 ); if ( rc != 0 ) break; @@ -292,7 +292,7 @@ LIB_EXPORT rc_t CC KTokenToVersion ( const KToken *self, uint32_t *vp ) /* single-part versions */ case eOctal: case eDecimal: - StringInit ( & str, start, (size_t)( end - start ), (uint32_t)( end - start ) ); + StringInit ( & str, start, end - start, (uint32_t)( end - start ) ); rc = StringConvertDecimal ( & str, & i, 8 ); if ( rc != 0 ) break; diff --git a/libs/kns/http-client.c b/libs/kns/http-client.c index a3252bf57..857908af5 100644 --- a/libs/kns/http-client.c +++ b/libs/kns/http-client.c @@ -55,6 +55,8 @@ typedef struct KClientHttpStream KClientHttpStream; #include +#include "../klib/int_checks-priv.h" + #if _DEBUGGING && 0 #include #define TRACE( x, ... ) \ @@ -326,7 +328,8 @@ struct KEndPointArgsIterator * KNSManagerMakeKEndPointArgsIterator ( { struct KEndPointArgsIterator * i = calloc ( 1, sizeof * i ); - KEndPointArgsIteratorMake ( i, self, hostname, port, cnt ); + assert(FITS_INTO_INT16(port)); + KEndPointArgsIteratorMake ( i, self, hostname, (uint16_t)port, cnt ); return i; } @@ -478,7 +481,15 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP static bool INITED = false; static bool PRINT_STAT = false; if (!INITED) { +#ifdef WINDOWS + { + size_t buf_size; + errno_t err = getenv_s ( & buf_size, NULL, 0, "NCBI_VDB_STS_SILENT" ); + PRINT_STAT = ( err != 0 || buf_size == 0 ); + } +#else PRINT_STAT = getenv("NCBI_VDB_STS_SILENT") == NULL; +#endif INITED = true; } @@ -488,7 +499,8 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP mgr = self -> mgr; assert ( mgr ); - KEndPointArgsIteratorMake ( & it, mgr, aHostname, aPort, NULL ); + assert ( FITS_INTO_INT16 ( aPort ) ); + KEndPointArgsIteratorMake ( & it, mgr, aHostname, (uint16_t)aPort, NULL ); while ( KEndPointArgsIteratorNext ( & it, & hostname, & port, & proxy_default_port, & proxy_ep, NULL, NULL ) ) { @@ -518,8 +530,9 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP /* try to establish a connection */ /* "read_timeout", "write_timeout" - when negative, infinite timeout */ - rc = KNSManagerMakeTimedConnection(mgr, &sock, - self->read_timeout, self->write_timeout, NULL, &self->ep); + rc = KNSManagerMakeTimedConnectionExt(mgr, &sock, + self->conn_timeout, self->read_timeout, self->write_timeout, + NULL, &self->ep); /* if we connected to a proxy, try to follow-through to server */ if (proxy_ep && self->tls && rc == 0) @@ -702,7 +715,7 @@ rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, v if ( rc == 0 ) { - const char * s; + const char * s = NULL; const char * ua = NULL; rc = KNSManagerGetUserAgent(&ua); if (rc == 0) @@ -771,7 +784,7 @@ rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, v */ rc_t KNSManagerMakeClientHttpInt ( const KNSManager *self, KClientHttp **_http, const KDataBuffer *hostname_buffer, KStream *opt_conn, - ver_t vers, int32_t readMillis, int32_t writeMillis, + ver_t vers, int32_t connMillis, int32_t readMillis, int32_t writeMillis, const String *host, uint32_t port, bool reliable, bool tls ) { rc_t rc; @@ -787,6 +800,7 @@ rc_t KNSManagerMakeClientHttpInt ( const KNSManager *self, KClientHttp **_http, char save, *text; http -> mgr = self; + http -> conn_timeout = connMillis; http -> read_timeout = readMillis; http -> write_timeout = writeMillis; @@ -896,8 +910,9 @@ rc_t KNSManagerMakeTimedClientHttpInt ( const KNSManager *self, port = dflt_port; /* initialize http object - will create a new reference to hostname buffer */ - rc = KNSManagerMakeClientHttpInt ( self, _http, & hostname_buffer, - opt_conn, vers, readMillis, writeMillis, &_host, port, false, tls ); + rc = KNSManagerMakeClientHttpInt ( self, _http, + & hostname_buffer, opt_conn, vers, self->conn_timeout, + readMillis, writeMillis, &_host, port, false, tls ); /* release our reference to buffer */ KDataBufferWhack ( & hostname_buffer ); @@ -1506,7 +1521,7 @@ rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, u else { /* which version was returned? */ - * version = string_cmp ( "1.0", 3, buffer, sep - buffer, -1 ) == 0 ? 0x01000000 : 0x01010000; + * version = string_cmp ( "1.0", 3, buffer, sep - buffer, (uint32_t)-1 ) == 0 ? 0x01000000 : 0x01010000; /* move up to status code */ buffer = sep + 1; @@ -1915,7 +1930,7 @@ rc_t KClientHttpSendReceiveMsg ( KClientHttp *self, KClientHttpResult **rslt, const char *buffer, size_t len, const KDataBuffer *body, const char *url ) { rc_t rc = 0; - size_t sent; + size_t sent = 0; timeout_t tm; timeout_t * ptm = NULL; uint32_t status; @@ -2601,7 +2616,7 @@ LIB_EXPORT bool CC KClientHttpResultTestHeaderValue ( const KClientHttpResult *s sep = end; /* test for case-insensitive match of value */ - if ( strcase_cmp ( start, sep - start, value, val_size, -1 ) == 0 ) + if ( strcase_cmp ( start, sep - start, value, val_size, (uint32_t)-1 ) == 0 ) { /* found it - delete p if malloc'd */ if ( p != buffer && p != NULL ) diff --git a/libs/kns/http-file.c b/libs/kns/http-file.c index 76ded386f..f70e5135d 100644 --- a/libs/kns/http-file.c +++ b/libs/kns/http-file.c @@ -73,6 +73,8 @@ #include #endif +#include "../klib/int_checks-priv.h" + #if _DEBUGGING && 0 #include #define TRACE( x, ... ) \ @@ -793,12 +795,12 @@ rc_t CC KHttpFileTimedRead ( const KHttpFile *self, rc = KHttpFileTimedReadLocked ( self, pos, buffer, bsize, num_read, tm, & http_status ); if ( rc != 0 ) { - rc_t rc2=KClientHttpReopen ( self -> http ); + rc_t rc3=KClientHttpReopen ( self -> http ); DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ), ( "KHttpFileTimedRead: KHttpFileTimedReadLocked failed, reopening\n" ) ); - if ( rc2 == 0 ) + if ( rc3 == 0 ) { - rc2 = KHttpFileTimedReadLocked ( self, pos, buffer, bsize, num_read, tm, & http_status ); - if ( rc2 == 0 ) + rc3 = KHttpFileTimedReadLocked ( self, pos, buffer, bsize, num_read, tm, & http_status ); + if ( rc3 == 0 ) { DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ), ( "KHttpFileTimedRead: reopened successfully\n" ) ); rc= 0; @@ -1332,15 +1334,16 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, if ( rc == 0 ) { - KDataBuffer * buf = & f -> orig_url_buffer; + KDataBuffer * buf_f = & f -> orig_url_buffer; KClientHttp *http; - rc = KNSManagerMakeClientHttpInt ( self, & http, buf, conn, vers, + rc = KNSManagerMakeClientHttpInt ( self, & http, buf_f, conn, + vers, self -> conn_timeout, self -> http_read_timeout, self -> http_write_timeout, & f -> block . host, f -> block . port, reliable, f -> block . tls ); if ( rc == 0 ) { KClientHttpRequest *req; - rc = KClientHttpMakeRequestInt ( http, & req, & f -> block, buf ); + rc = KClientHttpMakeRequestInt ( http, & req, & f -> block, buf_f ); if ( rc == 0 ) { KClientHttpResult *rslt; @@ -1377,7 +1380,7 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, "Failed to KClientHttpRequestHEAD(" "'$(path)' ($(ip))) from '$(local)'", "path=%.*s,ip=%s,local=%s", - buf -> elem_count - 1, buf -> base, + buf_f -> elem_count - 1, buf_f -> base, ep . ip_address, local_ep . ip_address ) ); } } @@ -1452,15 +1455,15 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, KClientHttpGetLocalEndpoint ( http, & local_ep ); KClientHttpGetRemoteEndpoint ( http, & ep ); if ( KNSManagerLogNcbiVdbNetError ( self ) ) { - char * base = buf -> base; + char * base = buf_f -> base; bool print = true; - char * query = string_chr ( base, buf -> elem_count, '?' ); + char * query = string_chr ( base, buf_f -> elem_count, '?' ); String vdbcache; CONST_STRING ( & vdbcache, ".vdbcache" ); - if ( buf -> elem_count > vdbcache . size ) { + if ( buf_f -> elem_count > vdbcache . size ) { String ext; StringInit ( & ext, - base + buf -> elem_count - vdbcache . size - 1, + base + buf_f -> elem_count - vdbcache . size - 1, vdbcache . size, vdbcache . len ); if ( ext . addr [ ext . size ] == '\0' && StringEqual ( & vdbcache, & ext ) ) @@ -1468,9 +1471,10 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, print = false; } else if ( query != NULL ) { - size_t size = query - base; + assert ( FITS_INTO_SIZE_T ( query - base ) ); + size_t size2 = (size_t)(query - base); StringInit ( & ext, - base + size - vdbcache . size, + base + size2 - vdbcache . size, vdbcache . size, vdbcache . len ); if ( ext . addr [ ext . size ] == '?' && StringEqual ( & vdbcache, & ext ) ) @@ -1482,12 +1486,12 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, if ( ! reliable ) print = false; if ( print ) { - assert ( buf ); + assert ( buf_f ); PLOGERR ( klogErr, ( klogErr, rc, "Failed to KNSManagerVMakeHttpFileInt('$(path)' ($(ip)))" " from '$(local)'", "path=%.*s,ip=%s,local=%s", - ( int ) buf -> elem_count, buf -> base, + ( int ) buf_f -> elem_count, buf_f -> base, ep . ip_address, local_ep . ip_address ) ); } @@ -1495,7 +1499,7 @@ static rc_t KNSManagerVMakeHttpFileIntUnstableImpl( const KNSManager *self, else DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ), ( "Failed to KNSManagerVMakeHttpFileInt('%.*s' (%s))\n", - ( int ) buf -> elem_count, buf -> base, + ( int ) buf_f -> elem_count, buf_f -> base, ep . ip_address ) ); } } diff --git a/libs/kns/http-priv.h b/libs/kns/http-priv.h index 956ebd008..ba0048fab 100644 --- a/libs/kns/http-priv.h +++ b/libs/kns/http-priv.h @@ -175,6 +175,7 @@ struct KClientHttp KRefcount refcount; + int32_t conn_timeout; int32_t read_timeout; int32_t write_timeout; @@ -202,7 +203,7 @@ rc_t KClientHttpReopen ( struct KClientHttp * self ); rc_t KNSManagerMakeClientHttpInt ( struct KNSManager const *self, struct KClientHttp **_http, const KDataBuffer *hostname_buffer, struct KStream *opt_conn, - ver_t vers, int32_t readMillis, int32_t writeMillis, + ver_t vers, int32_t connMillis, int32_t readMillis, int32_t writeMillis, const String *host, uint32_t port, bool reliable, bool tls ); rc_t KClientHttpVAddHeader ( BSTree *hdrs, bool add, const char *_name, const char *_val, va_list args ); diff --git a/libs/kns/http-request.c b/libs/kns/http-request.c index a5e13dc49..33ba4e484 100644 --- a/libs/kns/http-request.c +++ b/libs/kns/http-request.c @@ -45,6 +45,7 @@ #include "http-priv.h" #include "mgr-priv.h" +#include "../klib/int_checks-priv.h" #if _DEBUGGING && 0 #include @@ -291,7 +292,9 @@ LIB_EXPORT rc_t CC KClientHttpMakeRequest ( const KClientHttp *self, */ static rc_t CC KNSManagerMakeClientRequestInt ( const KNSManager *self, - KClientHttpRequest **req, ver_t vers, KStream *conn, bool reliable, const char *url, va_list args ) + KClientHttpRequest **req, + ver_t vers, int32_t connMillis, int32_t readMillis, int32_t writeMillis, + KStream *conn, bool reliable, const char *url, va_list args ) { rc_t rc; @@ -326,8 +329,10 @@ rc_t CC KNSManagerMakeClientRequestInt ( const KNSManager *self, { KClientHttp * http; - rc = KNSManagerMakeClientHttpInt ( self, & http, & buf, conn, vers, - self -> http_read_timeout, self -> http_write_timeout, & block . host, block . port, reliable, block . tls ); + rc = KNSManagerMakeClientHttpInt ( self, & http, & buf, + conn, vers, connMillis, + readMillis, writeMillis, & block . host, + block . port, reliable, block . tls ); if ( rc == 0 ) { rc = KClientHttpMakeRequestInt ( http, req, & block, & buf ); @@ -347,8 +352,12 @@ LIB_EXPORT rc_t CC KNSManagerMakeClientRequest ( const KNSManager *self, { rc_t rc; va_list args; + if (self == NULL) + return RC(rcNS, rcNoTarg, rcValidating, rcSelf, rcNull); va_start ( args, url ); - rc = KNSManagerMakeClientRequestInt ( self, req, vers, conn, false, url, args ); + rc = KNSManagerMakeClientRequestInt ( self, req, vers, + self->conn_timeout, self->http_read_timeout, self->http_write_timeout, + conn, false, url, args ); va_end ( args ); return rc; } @@ -358,12 +367,30 @@ LIB_EXPORT rc_t CC KNSManagerMakeReliableClientRequest ( const KNSManager *self, { rc_t rc; va_list args; + if (self == NULL) + return RC(rcNS, rcNoTarg, rcValidating, rcSelf, rcNull); va_start ( args, url ); - rc = KNSManagerMakeClientRequestInt ( self, req, vers, conn, true, url, args ); + rc = KNSManagerMakeClientRequestInt ( self, req, vers, + self->conn_timeout, self->http_read_timeout, self->http_write_timeout, + conn, true, url, args ); va_end ( args ); return rc; } +LIB_EXPORT rc_t CC KNSManagerMakeTimedClientRequest(const KNSManager *self, + KClientHttpRequest **req, + ver_t vers, int32_t connMillis, int32_t readMillis, + int32_t writeMillis, KStream *conn, const char *url, ...) +{ + rc_t rc; + va_list args; + va_start(args, url); + rc = KNSManagerMakeClientRequestInt(self, req, + vers, connMillis, readMillis, writeMillis, conn, false, url, args); + va_end(args); + return rc; +} + /* AddRef * Release @@ -901,7 +928,8 @@ rc_t KClientHttpRequestUrlEncodeBase64(const String ** encoding) { size_t iFrom = 0, iTo = 0; const char *from = (*encoding)->addr; char *to = NULL; - uint32_t len = (*encoding)->size + n + n; + assert ( FITS_INTO_INT32 ((*encoding)->size + n + n ) ); + uint32_t len = (uint32_t)((*encoding)->size + n + n); String * encoded = (String *) calloc(1, sizeof * encoded + len + 1); if (encoded == NULL) @@ -1044,11 +1072,12 @@ static EUriForm EUriFormGuess ( const String * hostname, else { String amazonaws; CONST_STRING ( & amazonaws, "amazonaws.com" ); + assert ( FITS_INTO_INT32 ( amazonaws.size ) ); if ( hostname -> size > amazonaws . size && string_cmp ( amazonaws . addr, amazonaws . size, hostname -> addr + hostname -> size - amazonaws . size, amazonaws . size, - amazonaws . size ) == 0 ) + (uint32_t) amazonaws . size ) == 0 ) { return eUFOriginNoPort; } @@ -1237,6 +1266,22 @@ FormatForCloud( const KClientHttpRequest *cself, const char *method ) CloudMgrCurrentProvider ( cloudMgr, & cpId ); { +#ifdef WINDOWS + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, &buf_count, "NCBI_VDB_PROVIDER" ); + if ( ! err && e != NULL ) + { + if (e[0] != '\0') + { + CloudProviderId i = cloud_provider_none; + i = atoi(e); + if (i != cloud_provider_none) + cpId = i; + } + free ( e ); + } +#else const char * e = getenv("NCBI_VDB_PROVIDER"); if (e != NULL && e[0] != '\0') { CloudProviderId i = cloud_provider_none; @@ -1244,6 +1289,7 @@ FormatForCloud( const KClientHttpRequest *cself, const char *method ) if (i != cloud_provider_none) cpId = i; } +#endif } if ( cpId != cloud_provider_none && ( cself->ceRequired || cself->payRequired ) ) @@ -1689,7 +1735,16 @@ LIB_EXPORT rc_t CC KClientHttpRequestHEAD ( KClientHttpRequest *self, KClientHtt static int HEADLESS = -1; if ( HEADLESS < 0 ) { +#ifdef WINDOWS + const char * s = NULL; + char s_buf[16]; + size_t buf_count = 0; + errno_t err = getenv_s ( & buf_count, s_buf, sizeof(s_buf), "NCBI_VDB_GET_AS_HEAD" ); + if ( ! err && buf_count > 0 ) + s = s_buf; +#else char * s = getenv ( "NCBI_VDB_GET_AS_HEAD" ); +#endif if ( s == NULL ) HEADLESS = 0; else if ( s [ 0 ] != '\0' ) @@ -2015,12 +2070,12 @@ static bool GovSiteByHttp ( const char * path ) { return false; } else { - size_t size = 0; + size_t size2 = 0; String gov; CONST_STRING ( & gov, ".gov" ); - size = gov . size; + size2 = gov . size; if ( strcase_cmp - ( path + i - 5, size, gov . addr, size, gov.len ) == 0 ) + ( path + i - 5, size2, gov . addr, size2, gov.len ) == 0 ) { return true; } diff --git a/libs/kns/http-retrier.c b/libs/kns/http-retrier.c index b05bb8d0a..2161689d3 100644 --- a/libs/kns/http-retrier.c +++ b/libs/kns/http-retrier.c @@ -43,6 +43,7 @@ typedef struct KHttpFile KHttpFile; #include #include "mgr-priv.h" +#include "../klib/int_checks-priv.h" struct HttpRetrySchedule { @@ -116,7 +117,8 @@ rc_t HttpRetryCodesConfig( HttpRetrySchedule** self, uint16_t code, const String return RC ( rcNS, rcData, rcCreating, rcMemory, rcExhausted ); } (*self) -> code = code; - (*self) -> max_retries = retryCount; + assert ( FITS_INTO_INT8 ( retryCount ) ); + (*self) -> max_retries = (uint8_t)retryCount; (*self) -> open_ended = false; { /* populate sleep_before_retry and open_ended */ @@ -145,11 +147,13 @@ rc_t HttpRetryCodesConfig( HttpRetrySchedule** self, uint16_t code, const String { if ( ! in_number ) { /* first digit of a new number */ - cur_value = ch - '0'; + assert ( FITS_INTO_INT16 ( ch - '0' ) ); + cur_value = (uint16_t)(ch - '0'); } else { - cur_value = cur_value * 10 + ch - '0'; + assert ( FITS_INTO_INT16 ( cur_value * 10 + ch - '0' ) ); + cur_value = (uint16_t)(cur_value * 10 + ch - '0'); } in_number = true; } @@ -206,7 +210,7 @@ rc_t HttpRetrySpecsFromConfig ( HttpRetrySpecs* self, const KConfigNode* node ) bool has4xx = false; bool has5xx = false; KNamelist * names; - uint32_t nameCount; + uint32_t nameCount = 0; rc_t rc = KConfigNodeListChildren ( node, & names ); if ( rc == 0 ) { /* count 4xx and 5xx */ @@ -260,7 +264,8 @@ rc_t HttpRetrySpecsFromConfig ( HttpRetrySpecs* self, const KConfigNode* node ) rc_t rc2; uint32_t i; uint32_t cur = 0; - self -> count = total; + assert ( FITS_INTO_INT8 ( total ) ); + self -> count = (uint8_t)total; self -> codes = (HttpRetrySchedule**) calloc ( self -> count, sizeof * self -> codes ); for ( i = 0; i < nameCount; ++ i ) { @@ -461,7 +466,8 @@ bool KHttpRetrierWait ( KHttpRetrier * self, uint32_t status ) uint8_t max_retries; const uint16_t * sleep_before_retry; bool open_ended; - if ( ! HttpGetRetryCodes ( & self -> kns -> retry_specs, status, & max_retries, & sleep_before_retry, & open_ended ) ) + assert ( FITS_INTO_INT16 ( status ) ); + if ( ! HttpGetRetryCodes ( & self -> kns -> retry_specs, (uint16_t)status, & max_retries, & sleep_before_retry, & open_ended ) ) { return false; } diff --git a/libs/kns/manager.c b/libs/kns/manager.c index 80d881fc9..83634a2f8 100644 --- a/libs/kns/manager.c +++ b/libs/kns/manager.c @@ -55,6 +55,8 @@ #include +#include "../klib/int_checks-priv.h" + #if HAVE_GNU_LIBC_VERSION_H #include #endif @@ -334,27 +336,9 @@ LIB_EXPORT rc_t CC KNSManagerMakeConnection ( const KNSManager *self, return KNSManagerMakeRetryTimedConnection ( self, conn, ptm, self->conn_read_timeout, self->conn_write_timeout, from, to ); } -/* MakeTimedConnection - * create a connection-oriented stream - * - * "conn" [ OUT ] - a stream for communication with the server - * - * "retryTimeout" [ IN ] - if connection is refused, retry with 1ms intervals: - * when negative, retry infinitely, when 0, do not retry, positive gives maximum - * wait time in seconds - * - * "readMillis" [ IN ] and "writeMillis" - when negative, infinite timeout - * when 0, return immediately, positive gives maximum wait time in mS - * for reads and writes respectively. - * - * "from" [ IN ] - client endpoint - * - * "to" [ IN ] - server endpoint - * - * both endpoints have to be of type epIP; creates a TCP connection - */ -LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( - struct KNSManager const *self, struct KSocket **conn, int32_t readMillis, + +static rc_t KNSManagerMakeTimedConnectionInt ( struct KNSManager const *self, + struct KSocket **conn, int32_t connectMillis, int32_t readMillis, int32_t writeMillis, struct KEndPoint const *from, struct KEndPoint const *to ) { @@ -371,8 +355,8 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull ); } - if (self->conn_timeout >=0 ) { - TimeoutInit ( &tm, self->conn_timeout ); + if (connectMillis >=0 ) { + TimeoutInit ( &tm, connectMillis ); ptm = &tm; } @@ -380,6 +364,48 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( self, conn, ptm, readMillis, writeMillis, from, to ); } +/* MakeTimedConnection, MakeTimedConnectionExt + * create a connection-oriented stream + * + * "conn" [ OUT ] - a stream for communication with the server + * + * "retryTimeout" [ IN ] - if connection is refused, retry with 1ms intervals: + * when negative, retry infinitely, when 0, do not retry, positive gives maximum + * wait time in seconds + * + * "connectMillis", "readMillis" and "writeMillis" [ IN ] - when negative, + * infinite timeout; + * when 0, return immediately; positive gives maximum wait time in mS + * for connection, reads and writes respectively. + * + * "from" [ IN ] - client endpoint + * + * "to" [ IN ] - server endpoint + * + * both endpoints have to be of type epIP; creates a TCP connection + */ +/* Use connectMillis from KNSManager */ +LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection( + struct KNSManager const *self, struct KSocket **conn, int32_t readMillis, + int32_t writeMillis, struct KEndPoint const *from, + struct KEndPoint const *to) +{ + if (self == NULL) + return RC(rcNS, rcStream, rcConstructing, rcSelf, rcNull); + + return KNSManagerMakeTimedConnectionInt( + self, conn, self->conn_timeout, readMillis, writeMillis, from, to); +} +/* connectMillis is specified */ +LIB_EXPORT rc_t CC KNSManagerMakeTimedConnectionExt ( + struct KNSManager const *self, struct KSocket **conn, int32_t connectMillis, + int32_t readMillis, int32_t writeMillis, struct KEndPoint const *from, + struct KEndPoint const *to ) +{ + return KNSManagerMakeTimedConnectionInt( + self, conn, connectMillis, readMillis, writeMillis, from, to); + } + /* MakeRetryConnection * create a connection-oriented stream * @@ -528,9 +554,21 @@ bool OwnCertfromEnv(const char ** own_cert, const char ** pk_key) { char ** key = (char**)pk_key; assert(cert && key); +#ifdef WINDOWS + char* e = NULL; + { + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, "VCBI_VDB_OWN_CERT" ); + if ( err || e == NULL ) + return false; + } +#else char * e = getenv("VCBI_VDB_OWN_CERT"); if (e == NULL) return false; +#endif /* WINDOWS */ + + /* CAUTION: free( e ) must be called before exiting the function on WINDOWS */ KDirectory * dir = NULL; rc_t rc = KDirectoryNativeDir(&dir); @@ -546,7 +584,13 @@ bool OwnCertfromEnv(const char ** own_cert, const char ** pk_key) { if (rc == 0) *cert = calloc(1, size + 1); if (rc == 0 && *cert == NULL) + { +#ifdef WINDOWS + free ( e ); + e = NULL; +#endif return false; + } if (rc == 0) rc = KFileRead(file, 0, *cert, size + 1, &num_read); if (rc == 0) @@ -560,7 +604,13 @@ bool OwnCertfromEnv(const char ** own_cert, const char ** pk_key) { if (rc == 0) *key = calloc(1, size + 1); if (rc == 0 && *key == NULL) + { +#ifdef WINDOWS + free ( e ); + e = NULL; +#endif return false; + } if (rc == 0) rc = KFileRead(file, 0, *key, size + 1, &num_read); if (rc == 0) @@ -568,6 +618,9 @@ bool OwnCertfromEnv(const char ** own_cert, const char ** pk_key) { } KDirectoryRelease(dir); +#ifdef WINDOWS + free ( e ); +#endif return rc == 0; } @@ -734,7 +787,8 @@ static int32_t KNSManagerLoadConnTimeout ( KConfig *kfg ) if ( rc != 0 ) result = MAX_CONN_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static int32_t KNSManagerLoadConnReadTimeout ( KConfig *kfg ) { @@ -744,7 +798,8 @@ static int32_t KNSManagerLoadConnReadTimeout ( KConfig *kfg ) if ( rc != 0 ) result = MAX_CONN_READ_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static int32_t KNSManagerLoadConnWriteTimeout ( KConfig *kfg ) { @@ -755,7 +810,8 @@ static int32_t KNSManagerLoadConnWriteTimeout ( KConfig *kfg ) if ( rc != 0 ) result = MAX_CONN_WRITE_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static int32_t KNSManagerLoadHttpReadTimeout ( KConfig *kfg ) @@ -766,7 +822,8 @@ static int32_t KNSManagerLoadHttpReadTimeout ( KConfig *kfg ) if ( rc != 0 ) result = MAX_HTTP_READ_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static int32_t KNSManagerLoadHttpWriteTimeout ( KConfig *kfg ) { @@ -776,7 +833,8 @@ static int32_t KNSManagerLoadHttpWriteTimeout ( KConfig *kfg ) if ( rc != 0 ) result = MAX_HTTP_WRITE_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static @@ -786,13 +844,35 @@ int32_t KNSManagerLoadTotalWaitForReliableURLs(const KConfig *kfg) int64_t result = 0; +#ifdef WINDOWS + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, "NCBI_VDB_RELIABLE_WAIT" ); + if (!err && str != NULL) { + char *end = NULL; + result = strtou64(str, &end, 0); + char chend = end[0]; + free(str); + if (chend == 0) + { + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; + } + } + } +#else const char * str = getenv("NCBI_VDB_RELIABLE_WAIT"); if (str != NULL) { char *end = NULL; result = strtou64(str, &end, 0); if (end[0] == 0) - return result; + { + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; + } } +#endif /* WINDOWS */ rc = KConfigReadI64(kfg, "/http/reliable/wait", &result); if (rc != 0 @@ -801,7 +881,8 @@ int32_t KNSManagerLoadTotalWaitForReliableURLs(const KConfig *kfg) result = MAX_HTTP_TOTAL_READ_LIMIT; } - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static int32_t KNSManagerLoadTotalConnectWaitForReliableURLs( @@ -811,19 +892,42 @@ static int32_t KNSManagerLoadTotalConnectWaitForReliableURLs( int64_t result = 0; +#ifdef WINDOWS + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, "NCBI_VDB_RELIABLE_CONNECT_WAIT" ); + if (!err && str != NULL) { + char *end = NULL; + result = strtou64(str, &end, 0); + char chend = end[0]; + free(str); + if (chend == 0) + { + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; + } + } + } +#else const char * str = getenv("NCBI_VDB_RELIABLE_CONNECT_WAIT"); if (str != NULL) { char *end = NULL; result = strtou64(str, &end, 0); if (end[0] == 0) - return result; + { + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; + } } +#endif rc = KConfigReadI64(kfg, "/http/reliable/connect/wait", &result); if (rc != 0) result = MAX_HTTP_TOTAL_CONNECT_LIMIT; - return result; + assert ( FITS_INTO_INT32 ( result ) ); + return (int32_t)result; } static bool KNSManagerLoadRetryFirstRead(const KConfig *kfg) { @@ -831,6 +935,28 @@ static bool KNSManagerLoadRetryFirstRead(const KConfig *kfg) { bool result = 0; +#ifdef WINDOWS + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, "NCBI_VDB_RELIABLE_RETRY_FIRST_READ" ); + if (!err && str != NULL) + { + char ch = str[0]; + free(str); + if (ch != '\0') { + switch (ch) { + case 'f': + return false; + case 't': + return true; + default: + break; + } + } + } + } +#else const char * str = getenv("NCBI_VDB_RELIABLE_RETRY_FIRST_READ"); if (str != NULL && str[0] != '\0') { switch (str[0]) { @@ -842,6 +968,7 @@ static bool KNSManagerLoadRetryFirstRead(const KConfig *kfg) { break; } } +#endif rc = KConfigReadBool(kfg, "/http/reliable/retryFirstRead", &result); if (rc != 0) @@ -855,6 +982,28 @@ static bool KNSManagerLoadRetryFile(const KConfig *kfg) { bool result = 0; +#ifdef WINDOWS + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, "NCBI_VDB_RELIABLE_RETRY_FILE" ); + if (!err && str != NULL) + { + char ch = str[0]; + free(str); + if (ch != '\0') { + switch (ch) { + case 'f': + return false; + case 't': + return true; + default: + break; + } + } + } + } +#else const char * str = getenv("NCBI_VDB_RELIABLE_RETRY_FILE"); if (str != NULL && str[0] != '\0') { switch (str[0]) { @@ -866,6 +1015,7 @@ static bool KNSManagerLoadRetryFile(const KConfig *kfg) { break; } } +#endif rc = KConfigReadBool(kfg, "/http/reliable/retryFile", &result); if (rc != 0) @@ -882,11 +1032,39 @@ static uint8_t KNSManagerLoadMaxNumberOfRetriesOnFailureForReliableURLs if (rc != 0 || result < 0) result = 10; - return result; + assert ( FITS_INTO_INT8 ( result ) ); + return (uint8_t)result; } #if 1 static uint64_t KNSManagerLoadLogTlsErrors(KConfig* kfg) { +#ifdef WINDOWS + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, "NCBI_VDB_TLS_LOG_ERR" ); + uint64_t ret = 0; + if (!err && e != NULL) + { + if (e[0] == '\0') + ret = 0; + else { + if (e[0] == '0' || + e[0] == 'f') /* false */ + { + ret = 0; + } + else + { + int n = atoi(e); + assert( n >= 0 ); + ret = (uint64_t)n; + } + } + free(e); + return ret; + } + +#else const char * e = getenv("NCBI_VDB_TLS_LOG_ERR"); if (e != NULL) if (e[0] == '\0') @@ -900,6 +1078,7 @@ static uint64_t KNSManagerLoadLogTlsErrors(KConfig* kfg) { else return atoi(e); } +#endif else { uint64_t log = 0; rc_t rc = KConfigReadU64(kfg, "/tls/NCBI_VDB_TLS_LOG_ERR", &log); @@ -1111,6 +1290,25 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char **user_agent ) } char cloudtrunc[64]; + +#ifdef WINDOWS + { + char *cloudid = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & cloudid, & buf_count, ENV_MAGIC_CE_TOKEN ); + if ( !err && cloudid && strlen ( cloudid ) > 8 ) { + /* AWS access keys should always begin with AKIA, + * suffixes seems non-random */ + assert ( strlen ( cloudid ) - 4 < sizeof cloudtrunc ); + strncpy_s ( cloudtrunc, sizeof cloudtrunc, cloudid + 4, strlen ( cloudid ) - 4 ); + cloudtrunc[3] = '\0'; + } else { + strncpy_s ( cloudtrunc, sizeof cloudtrunc, "noc", 3 ); + } + if ( !err && cloudid ) + free ( cloudid ); + } +#else const char *cloudid = getenv ( ENV_MAGIC_CE_TOKEN ); if ( cloudid && strlen ( cloudid ) > 8 ) { /* AWS access keys should always begin with AKIA, @@ -1121,17 +1319,62 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char **user_agent ) } else { strcpy ( cloudtrunc, "noc" ); } +#endif +#ifdef WINDOWS + char sessid_buf[64]; + const char * sessid = NULL; + { + char *str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, ENV_VAR_SESSION_ID); + if ( err || str == NULL ) + { + sessid = "nos"; + } + else + { + size_t len = strlen(str); + assert ( len < sizeof sessid_buf ); + strncpy_s ( sessid_buf, sizeof sessid_buf, str, len ); + sessid = sessid_buf; + free ( str ); + } + } +#else const char *sessid = getenv ( ENV_VAR_SESSION_ID ); if ( sessid == NULL ) { sessid = "nos"; } +#endif const char *libc_version = ""; #if HAVE_GNU_GET_LIBC_VERSION_F libc_version = gnu_get_libc_version (); #endif +#ifdef WINDOWS + char opt_bitmap_buf[64]; + const char * opt_bitmap = NULL; + { + char *str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, ENV_MAGIC_OPT_BITMAP); + if ( err || str == NULL ) + { + opt_bitmap = "nob"; + } + else + { + size_t len = strlen(str); + assert ( len < sizeof opt_bitmap_buf ); + strncpy_s (opt_bitmap_buf, sizeof opt_bitmap_buf, str, len ); + opt_bitmap = opt_bitmap_buf; + free ( str ); + } + } +#else const char *opt_bitmap=getenv(ENV_MAGIC_OPT_BITMAP); // VDB_OPT_BITMAP if (!opt_bitmap) opt_bitmap="nob"; +#endif const char *guid = "nog"; @@ -1182,10 +1425,10 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char **user_agent ) if ( rc ) { return rc; } if ( kns_manager_lock ) { - rc_t rc = KLockAcquire ( kns_manager_lock ); - if ( rc ) { + rc_t rc2 = KLockAcquire ( kns_manager_lock ); + if ( rc2 ) { KDataBufferWhack ( &phid ); - return rc; + return rc2; } } @@ -1214,14 +1457,47 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char **user_agent ) } KDataBuffer platform; + const char* env_platform_name = NULL; + const char* env_platform_version = NULL; +#ifdef WINDOWS + char env_platform_name_buf[64], env_platform_version_buf[64]; + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, ENV_MAGIC_PLATFORM_NAME ); + if ( !err && str != NULL ) + { + assert ( strlen(str) < sizeof(env_platform_name_buf) ); + strncpy_s ( env_platform_name_buf, sizeof env_platform_name_buf, str, strlen(str) ); + env_platform_name = env_platform_name_buf; + free ( str ); + } + } + { + char * str = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & str, & buf_count, ENV_MAGIC_PLATFORM_VERSION ); + if ( !err && str != NULL ) + { + assert ( strlen(str) < sizeof(env_platform_version_buf) ); + strncpy_s ( env_platform_version_buf, sizeof env_platform_version_buf, str, strlen(str) ); + env_platform_version = env_platform_version_buf; + free ( str ); + } + } +#else + env_platform_name = getenv(ENV_MAGIC_PLATFORM_NAME); + env_platform_version = getenv(ENV_MAGIC_PLATFORM_VERSION); +#endif + KDataBufferMakeBytes(&platform, 0); - if (getenv(ENV_MAGIC_PLATFORM_NAME)) + if (env_platform_name) { - if (getenv(ENV_MAGIC_PLATFORM_VERSION)) + if (env_platform_version) { rc=KDataBufferPrintf(&platform," via %s %s", - getenv(ENV_MAGIC_PLATFORM_NAME), - getenv(ENV_MAGIC_PLATFORM_VERSION)); + env_platform_name, + env_platform_version); if (rc) { KDataBufferWhack ( &phid ); @@ -1231,7 +1507,7 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char **user_agent ) } } else{ - rc=KDataBufferPrintf(&platform," via %s", getenv(ENV_MAGIC_PLATFORM_NAME)); + rc=KDataBufferPrintf(&platform," via %s", env_platform_name); if (rc) { KDataBufferWhack ( &phid ); @@ -1301,6 +1577,24 @@ uint64_t KNSManagerLogNcbiVdbNetError ( const KNSManager *self ) return 0; if ( self->NCBI_VDB_NETnoLogError ) { return 0; } + +#ifdef WINDOWS + { + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, "NCBI_VDB_NET" ); + if ( !err && e != NULL ) + { + char ch = e[0]; + free ( e ); + if (ch == '\0' || ch == '0' || ch == 'f') /* false */ + { + return 0; + } + return 1; + } + } +#else const char *e = getenv ( "NCBI_VDB_NET" ); if ( e != NULL ) { if ( e[0] == '\0' || e[0] == '0' || e[0] == 'f' ) /* false */ @@ -1310,6 +1604,7 @@ uint64_t KNSManagerLogNcbiVdbNetError ( const KNSManager *self ) return 1; } +#endif if ( self->NCBI_VDB_NETkfgValueSet ) { return self->NCBI_VDB_NETkfgValue; } return self->logTlsErrors; diff --git a/libs/kns/proxy.c b/libs/kns/proxy.c index fb7d666ce..86ae6c563 100644 --- a/libs/kns/proxy.c +++ b/libs/kns/proxy.c @@ -45,6 +45,7 @@ #include #include /* time */ +#include "../klib/int_checks-priv.h" #define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false ) @@ -326,7 +327,10 @@ static rc_t KNSProxiesAddHttpProxyPath ( KNSProxies * self, HttpProxy * new_proxy = NULL; BSTItem * node = NULL; - HttpProxy add = { proxy_host, proxy_port, 0 }; + HttpProxy add; + add.proxy_host = proxy_host; + add.proxy_port = proxy_port; + add.next = 0; assert ( self ); @@ -396,7 +400,17 @@ static rc_t CC KNSProxiesAddHTTPProxyPath ( KNSProxies * self, static bool KNSProxiesHttpProxyInitFromEnvVar ( KNSProxies * self, const char * name ) { + bool ret = 0; +#ifdef WINDOWS + errno_t err = 0; + char* path = NULL; + { + size_t buf_count = 0; + err = _dupenv_s ( & path, & buf_count, name ); + } +#else const char * path = getenv ( name ); +#endif if ( path != NULL ) { assert ( self ); @@ -405,13 +419,21 @@ static bool KNSProxiesHttpProxyInitFromEnvVar ( KNSProxies * self, ( "Loading proxy env.var. %s='%s'\n", name, path ) ); if ( KNSProxiesAddHTTPProxyPath ( self, path ) != 0 ) - return false; - assert ( self -> http_proxy_enabled ); - - return true; + ret = false; + else + { + assert ( self -> http_proxy_enabled ); + ret = true; + } +#ifdef WINDOWS + assert ( ! err ); + free ( path ); +#endif } + else + ret = false; - return false; + return ret; } static bool KNSProxiesHttpProxyInitFromEnv ( KNSProxies * self ) { @@ -509,8 +531,10 @@ rc_t KNSProxiesVSetHTTPProxyPath ( KNSProxies * self, const char * port_spec = NULL; long port_num = 0; - int have = colon - p; - int remains = s - have; + int have = (int)(colon - p); + int remains = (int)(s - have); + assert ( FITS_INTO_INT ( colon - p ) ); + assert ( FITS_INTO_INT ( s - have ) ); if ( remains > 2 ) { assert ( colon [ 0 ] == ':' ); if ( colon [ 1 ] == '/' && colon [ 2 ] == '/' ) { @@ -629,8 +653,8 @@ KNSProxies * KNSManagerKNSProxiesMake ( struct KNSManager * mgr, { bool proxy_only = false; - rc_t rc = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only ); - if ( rc == 0 && proxy_only ) + rc_t rc2 = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only ); + if ( rc2 == 0 && proxy_only ) self-> http_proxy_only = true; } @@ -688,8 +712,9 @@ KNSProxies * KNSManagerKNSProxiesMake ( struct KNSManager * mgr, } self -> tmpS = 0; - n = self -> http_proxies_cnt; - srand ( time ( NULL ) ); + n = (int)(self -> http_proxies_cnt); + assert ( FITS_INTO_INT (self->http_proxies_cnt) ); + srand ( (unsigned int)time ( NULL ) ); while ( n > 0 ) { self -> rand = rand () % n; self -> tmpI = 0; diff --git a/libs/kns/stable-http-file.c b/libs/kns/stable-http-file.c index d511a7485..b3bd0429d 100644 --- a/libs/kns/stable-http-file.c +++ b/libs/kns/stable-http-file.c @@ -57,9 +57,22 @@ void RetrierReset(const KStableHttpFile * cself, const char * func) static int logLevel = -1; if (logLevel == -1) { logLevel = 0; +#ifdef WINDOWS + { + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, ENV_VAR_LOG_HTTP_RETRY ); + if ( !err && e != NULL ) + { + logLevel = atoi(e); + free ( e ); + } + } +#else const char * e = getenv(ENV_VAR_LOG_HTTP_RETRY); if (e != NULL) logLevel = atoi(e); +#endif } self->live = true; @@ -91,9 +104,22 @@ rc_t RetrierReopenRemote(KStableHttpFile * self, bool neverBefore) int logLevel = -1; if (logLevel == -1) { logLevel = 0; +#ifdef WINDOWS + { + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, ENV_VAR_LOG_HTTP_RETRY ); + if ( !err && e != NULL ) + { + logLevel = atoi(e); + free ( e ); + } + } +#else const char * e = getenv(ENV_VAR_LOG_HTTP_RETRY); if (e != NULL) logLevel = atoi(e); +#endif } KFileRelease(self->file); @@ -257,9 +283,22 @@ rc_t RetrierAgain(const KStableHttpFile * cself, static int logLevel = -1; if (logLevel == -1) { logLevel = 0; +#ifdef WINDOWS + { + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, ENV_VAR_LOG_HTTP_RETRY ); + if ( !err && e != NULL ) + { + logLevel = atoi(e); + free ( e ); + } + } +#else const char * e = getenv(ENV_VAR_LOG_HTTP_RETRY); if (e != NULL) logLevel = atoi(e); +#endif } if (logLevel > 1 || lvl > 0) { switch (self->_state) { @@ -609,6 +648,22 @@ rc_t KNSManagerVMakeHttpFileInt(const KNSManager *self, if (rc == 0) { static int sReliable = eUninitialized; if (sReliable == eUninitialized) { +#ifdef WINDOWS + { + char * e = NULL; + size_t buf_count = 0; + errno_t err = _dupenv_s ( & e, & buf_count, "NCBI_VDB_RELIABLE" ); + if (err || e == NULL) + sReliable = eNotSet; + else if (e != NULL && e[0] == '\0') + sReliable = eUnreliable; + else + sReliable = eReliable; + + if (!err && e != NULL) + free(e); + } +#else const char * e = getenv("NCBI_VDB_RELIABLE"); if (e == NULL) sReliable = eNotSet; @@ -616,6 +671,7 @@ rc_t KNSManagerVMakeHttpFileInt(const KNSManager *self, sReliable = eUnreliable; else sReliable = eReliable; +#endif } if (sReliable == eUnreliable) reliable = false; diff --git a/libs/kns/stream-from-buffer.c b/libs/kns/stream-from-buffer.c index 63582742d..ff761716b 100644 --- a/libs/kns/stream-from-buffer.c +++ b/libs/kns/stream-from-buffer.c @@ -37,6 +37,8 @@ typedef struct KBufferStream KBufferStream; #define rcStream rcFile #endif +#include "../klib/int_checks-priv.h" + struct KBufferStream { KStream dad; String buffer; @@ -80,9 +82,11 @@ rc_t CC KBufferStreamRead ( const KBufferStream * self, void * buffer, * num_read = string_copy ( buffer, bsize, src -> addr, src -> size ); + assert ( FITS_INTO_INT32 ( * num_read ) ); + src -> addr += * num_read; src -> size -= * num_read; - src -> len -= * num_read; + src -> len -= (uint32_t) (* num_read); return 0; } @@ -144,7 +148,8 @@ LIB_EXPORT rc_t CC KStreamMakeFromBuffer ( KStream ** self, const char * buffer, rc = KStreamInit ( & obj -> dad, ( const KStream_vt* ) & vtKBufferStream, "KBufferStream", "KBufferStream", true, false ); if ( rc == 0 ) { - StringInit ( & obj -> buffer, buffer, size, size ); + assert ( FITS_INTO_INT32 ( size ) ); + StringInit ( & obj -> buffer, buffer, size, (uint32_t)size ); * self = & obj -> dad; } else { diff --git a/libs/kns/stream.c b/libs/kns/stream.c index 20a5ce102..5cbfd7f8c 100644 --- a/libs/kns/stream.c +++ b/libs/kns/stream.c @@ -372,7 +372,7 @@ LIB_EXPORT rc_t CC KStreamTimedReadAll ( const KStream *self, LIB_EXPORT rc_t CC KStreamReadExactly ( const KStream *self, void *buffer, size_t bytes ) { - rc_t rc; + rc_t rc = 0; uint8_t *b; size_t total, count; @@ -418,7 +418,7 @@ LIB_EXPORT rc_t CC KStreamReadExactly ( const KStream *self, LIB_EXPORT rc_t CC KStreamTimedReadExactly ( const KStream *self, void *buffer, size_t bytes, struct timeout_t *tm ) { - rc_t rc; + rc_t rc = 0; uint8_t *b; size_t total, count; diff --git a/libs/kns/tls.c b/libs/kns/tls.c index 0fd744e38..624939ac1 100644 --- a/libs/kns/tls.c +++ b/libs/kns/tls.c @@ -20,7 +20,7 @@ * * Please cite the author in any work or product based on this material. * -* =========================================================================== +* =============================================================================$ * */ @@ -68,6 +68,8 @@ struct KTLSStream; #include #include +#include "../klib/int_checks-priv.h" + #if WINDOWS #define IGNORE_ALL_CERTS_ALLOWED 1 #endif @@ -607,12 +609,27 @@ static int set_threshold ( const KConfig * kfg ) { int64_t threshold = 0; const char * env = NULL; +#ifdef WINDOWS + char env_buf[64]; +#endif rc_t rc = KConfigReadI64 ( kfg, "/tls/NCBI_VDB_TLS", & threshold ); if ( rc == 0 ) set = true; +#ifdef WINDOWS + { + size_t buf_count = 0; + errno_t err = getenv_s ( & buf_count, env_buf, sizeof(env_buf), "NCBI_VDB_TLS" ); + if ( !err && buf_count > 0 ) + { + assert ( buf_count < sizeof(env_buf) ); + env = env_buf; + } + } +#else env = getenv ( "NCBI_VDB_TLS" ); +#endif if ( env != NULL ) { int NCBI_VDB_TLS = 0; @@ -803,10 +820,21 @@ rc_t CC KTLSStreamRead ( const KTLSStream * cself, KTLSStream * self = ( KTLSStream * ) cself; if (SILENCE_READING_MSG < 0) { +#ifdef WINDOWS + { + size_t buf_count = 0; + errno_t err = getenv_s ( & buf_count, NULL, 0, "NCBI_VDB_SILENCE_MBEDTLS_READ" ); + if ( !err && buf_count > 0 ) + SILENCE_READING_MSG = 1; + else + SILENCE_READING_MSG = 0; + } +#else if (getenv("NCBI_VDB_SILENCE_MBEDTLS_READ") != NULL) SILENCE_READING_MSG = 1; else SILENCE_READING_MSG = 0; +#endif } assert(self); @@ -832,7 +860,19 @@ rc_t CC KTLSStreamRead ( const KTLSStream * cself, ret = mbedtls_ssl_read( &self -> ssl, buffer, bsize ); if (!inited) { /* simulate mbedtls read timeout */ +#ifdef WINDOWS + const char * v = NULL; + char v_buf[64]; + size_t buf_count = 0; + errno_t err = getenv_s ( & buf_count, v_buf, sizeof(v_buf), "NCBI_VDB_ERR_MBEDTLS_READ" ); + if ( ! err && buf_count > 0 ) + { + assert ( buf_count < sizeof (v_buf) ); + v = v_buf; + } +#else const char * v = getenv("NCBI_VDB_ERR_MBEDTLS_READ"); +#endif if (v != NULL) { m = atoi(v); if (m < 0) @@ -1163,7 +1203,8 @@ rc_t KTLSGlobalsSetupOwnCert(KTLSGlobals * tlsg, else { size_t len = tlsg->clicert.subject.val.len; String subject; - StringInit(&subject, (char*)tlsg->clicert.subject.val.p, len, len); + assert ( FITS_INTO_INT32 ( len ) ); + StringInit(&subject, (char*)tlsg->clicert.subject.val.p, len, (uint32_t)len); STATUS(STAT_QA, "Setting '%S' client certificate", &subject); tlsg->clicert_was_set = true; } @@ -1264,7 +1305,19 @@ rc_t ktls_handshake ( KTLSStream *self ) ret = mbedtls_ssl_handshake( &self -> ssl ); if (!inited) { /* simulate mbedtls handshake timeout */ +#ifdef WINDOWS + const char * v = NULL; + char v_buf[64]; + size_t buf_count = 0; + errno_t err = getenv_s ( & buf_count, v_buf, sizeof(v_buf), "NCBI_VDB_ERR_MBEDTLS_HANDSHAKE" ); + if ( ! err && buf_count > 0 ) + { + assert ( buf_count < sizeof(v_buf) ); + v = v_buf; + } +#else const char * v = getenv("NCBI_VDB_ERR_MBEDTLS_HANDSHAKE"); +#endif if (v != NULL) { m = atoi(v); if (m < 0) @@ -1285,7 +1338,7 @@ rc_t ktls_handshake ( KTLSStream *self ) --e; } - while ( ret != 0 ) + for ( int i = 0; ret != 0; ) { if ( ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE ) @@ -1338,6 +1391,11 @@ rc_t ktls_handshake ( KTLSStream *self ) return rc; } ret = mbedtls_ssl_handshake( &self -> ssl ); + if (ret == MBEDTLS_ERR_SSL_WANT_WRITE && i++ > 99) { + // possible infinite loop on Mac and Bsd + //LOGERR(klogInt, 0, "ret== MBEDTLS_ERR_SSL_WANT_WRITE"); + return RC(rcKrypto, rcSocket, rcOpening, rcConnection, rcFailed); + } } return 0; @@ -1519,17 +1577,17 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACert ( const KTLSStream * self ) if ( flags != 0 ) { char buf [ 4096 ]; - rc_t rc = RC ( rcKrypto, rcToken, rcValidating, rcEncryption, rcFailed ); + rc_t rc2 = RC ( rcKrypto, rcToken, rcValidating, rcEncryption, rcFailed ); mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " ! ", flags ); - PLOGERR ( klogSys, ( klogSys, rc + PLOGERR ( klogSys, ( klogSys, rc2 , "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )" , "flags=0x%X,info=%s" , flags , buf ) ); - return rc; + return rc2; } } diff --git a/libs/kns/win/sysendpoint.c b/libs/kns/win/sysendpoint.c index f9291118b..fc05ad15c 100644 --- a/libs/kns/win/sysendpoint.c +++ b/libs/kns/win/sysendpoint.c @@ -24,6 +24,12 @@ * */ +#ifdef _WINSOCK_DEPRECATED_NO_WARNINGS +#define WINSOCK_DEP_NO_WARN_WAS_DEFINED 1 +#else +#define _WINSOCK_DEPRECATED_NO_WARNINGS +#endif + #include #include #include @@ -154,3 +160,9 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self, return rc; } + +#ifdef WINSOCK_DEP_NO_WARN_WAS_DEFINED +#undef WINSOCK_DEP_NO_WARN_WAS_DEFINED +#else +#undef _WINSOCK_DEPRECATED_NO_WARNINGS +#endif diff --git a/libs/kns/win/syssock.c b/libs/kns/win/syssock.c index 935a88805..098601e6f 100644 --- a/libs/kns/win/syssock.c +++ b/libs/kns/win/syssock.c @@ -24,6 +24,11 @@ * */ +#ifdef _WINSOCK_DEPRECATED_NO_WARNINGS +#define WINSOCK_DEP_NO_WARN_WAS_DEFINED 1 +#else +#define _WINSOCK_DEPRECATED_NO_WARNINGS +#endif /*-------------------------------------------------------------------------- * forwards @@ -237,7 +242,7 @@ static rc_t CC KIpv4SocketWhack ( KSocket * self ) data = &( self -> type_data.ipv4_data ); - if ( shutdown ( data -> fd, SHUT_WR ) == -1 ) + if ( shutdown ( data -> fd, SHUT_WR ) == SOCKET_ERROR ) rc = KSocketHandleShutdownCallWin (); else { @@ -256,7 +261,7 @@ static rc_t CC KIpv4SocketWhack ( KSocket * self ) } } - if ( shutdown ( data -> fd, SHUT_RD ) == -1 ) + if ( shutdown ( data -> fd, SHUT_RD ) == SOCKET_ERROR) rc = KSocketHandleShutdownCallWin (); else if ( closesocket ( data -> fd ) == SOCKET_ERROR ) rc = RC ( rcNS, rcSocket, rcClosing, rcError, rcUnknown ); /* maybe report */ @@ -264,7 +269,7 @@ static rc_t CC KIpv4SocketWhack ( KSocket * self ) #if 0 RC_CTX ( rcSocket, rcClosing ); - if ( shutdown ( data -> fd, SHUT_WR ) != -1 ) + if ( shutdown ( data -> fd, SHUT_WR ) != SOCKET_ERROR) { while ( 1 ) { @@ -274,7 +279,7 @@ static rc_t CC KIpv4SocketWhack ( KSocket * self ) break; } - if ( shutdown ( data -> fd, SHUT_RD ) != -1 ) + if ( shutdown ( data -> fd, SHUT_RD ) != SOCKET_ERROR) { if ( closesocket ( data -> fd ) == SOCKET_ERROR ) rc = RC ( rcNS, rcSocket, rcClosing, rcError, rcUnknown ); @@ -319,7 +324,7 @@ static rc_t CC KIpv4SocketTimedRead ( const KSocket * self, void * buffer, size_ selectRes = select( 0, &readFds, NULL, NULL, ( tm == NULL ) ? NULL : &ts ); /* check for error */ - if ( selectRes == -1 ) + if ( selectRes == SOCKET_ERROR ) rc = KSocketHandleSelectCallWin ( rcReading ); else if ( selectRes == 0 ) rc = RC ( rcNS, rcSocket, rcReading, rcTimeout, rcExhausted ); /* timeout */ @@ -435,7 +440,7 @@ static rc_t CC KIpv4SocketTimedWrite ( KSocket * self, const void * buffer, size selectRes = select( 0, NULL, &writeFds, NULL, ( tm == NULL ) ? NULL : &ts ); /* check for error */ - if ( selectRes == -1 ) + if ( selectRes == SOCKET_ERROR ) rc = KSocketHandleSelectCallWin ( rcWriting ); else if ( selectRes == 0 ) rc = RC ( rcNS, rcSocket, rcWriting, rcTimeout, rcExhausted ); /* timeout */ @@ -805,7 +810,7 @@ connect_wait(int sock, int32_t timeoutMs) FD_SET(sock, &writeFds); selectRes = select(0, NULL, &writeFds, NULL, &ts); - if (selectRes == -1) + if (selectRes == SOCKET_ERROR) rc = KSocketHandleSelectCallWin(rcCreating); else if (selectRes == 0) rc = RC(rcNS, rcSocket, rcCreating, rcTimeout, rcExhausted); /* timeout */ @@ -860,7 +865,6 @@ rc_t KSocketConnectIPv4 ( KSocket * self, const KEndPoint * from, const KEndPoin if (rc == 0) { - int res; SOCKET sock = self->type_data.ipv4_data.fd; ss.sin_port = htons(to->u.ipv4.port); ss.sin_addr.s_addr = htonl(to->u.ipv4.addr); @@ -914,7 +918,7 @@ static rc_t KNSManagerMakeIPv4Listener ( const KNSManager *self, KSocket **out, { KSocketIPv4 * data = &( listener -> type_data . ipv4_data ); data -> fd = socket ( AF_INET, SOCK_STREAM, 0 ); - if ( data -> fd < 0 ) + if ( data -> fd == INVALID_SOCKET ) rc = KSocketHandleSocketCallWin (); else { @@ -939,7 +943,7 @@ static rc_t KNSManagerMakeIPv4Listener ( const KNSManager *self, KSocket **out, } closesocket ( data -> fd ); - data -> fd = -1; + data -> fd = INVALID_SOCKET; } } @@ -974,14 +978,14 @@ static rc_t KListenerIPv4Accept ( KSocket * self, struct KSocket ** out ) len = sizeof new_data -> remote_addr; new_data -> fd = accept ( self_data -> fd, ( struct sockaddr * ) & new_data -> remote_addr, & len ); - if ( new_data -> fd < 0 ) + if ( new_data -> fd == INVALID_SOCKET ) rc = KSocketHandleAcceptCallWin (); else if ( len <= sizeof new_data -> remote_addr ) new_data -> remote_addr_valid = true; else { closesocket ( new_data -> fd ); - new_data -> fd = -1; + new_data -> fd = INVALID_SOCKET; rc = RC ( rcNS, rcConnection, rcWaiting, rcBuffer, rcInsufficient ); } @@ -1073,7 +1077,7 @@ static rc_t KNSManagerMakeIPv6Listener ( const KNSManager *self, KSocket **out, KSocketIPv6 * data = &( listener -> type_data.ipv6_data ); data -> fd = socket ( AF_INET6, SOCK_STREAM, 0 ); - if ( data -> fd < 0 ) + if ( data -> fd == INVALID_SOCKET ) rc = KSocketHandleSocketCallWin (); else { @@ -1100,7 +1104,7 @@ static rc_t KNSManagerMakeIPv6Listener ( const KNSManager *self, KSocket **out, } closesocket ( data -> fd ); - data -> fd = -1; + data -> fd = INVALID_SOCKET; } } } @@ -1134,14 +1138,14 @@ static rc_t KListenerIPv6Accept ( KSocket * self, struct KSocket ** out ) len = sizeof new_data -> remote_addr; new_data -> fd = accept ( self_data -> fd, ( struct sockaddr * ) & new_data -> remote_addr, & len ); - if ( new_data -> fd < 0 ) + if ( new_data -> fd == INVALID_SOCKET ) rc = KSocketHandleAcceptCallWin (); else if ( len <= sizeof new_data -> remote_addr ) new_data -> remote_addr_valid = true; else { closesocket ( new_data -> fd ); - new_data -> fd = -1; + new_data -> fd = INVALID_SOCKET; rc = RC ( rcNS, rcConnection, rcWaiting, rcBuffer, rcInsufficient ); } @@ -1465,7 +1469,6 @@ static rc_t CC KIPCSocketTimedRead ( const KSocket * self, void * buffer, size_t case WAIT_OBJECT_0 : { - DWORD count; DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_SOCKET ), ( "%p: successful\n", self ) ); /* wait to complete if necessary */ if ( GetOverlappedResult( data->pipe, &overlap, &count, TRUE ) ) @@ -1985,7 +1988,7 @@ KNS_EXTERN rc_t CC KNSManagerMakeRetryTimedConnection ( struct KNSManager const rc = RC ( rcNS, rcStream, rcConstructing, rcMemory, rcExhausted ); else { - const KStream_vt * vt; + const KStream_vt * vt = NULL; conn -> read_timeout = readMillis; conn -> write_timeout = writeMillis; @@ -2294,7 +2297,7 @@ LIB_EXPORT rc_t CC KListenerRelease( const KListener *self ) LIB_EXPORT rc_t CC KListenerAccept ( KListener * self, struct KSocket ** out ) { - rc_t rc; + rc_t rc = 0; if ( out == NULL ) rc = RC ( rcNS, rcSocket, rcConstructing, rcParam, rcNull ); @@ -2448,3 +2451,8 @@ static rc_t HandleErrnoEx ( const char *func_name, unsigned int lineno, rc_t rc_ return rc; } +#ifdef WINSOCK_DEP_NO_WARN_WAS_DEFINED +#undef WINSOCK_DEP_NO_WARN_WAS_DEFINED +#else +#undef _WINSOCK_DEPRECATED_NO_WARNINGS +#endif diff --git a/libs/kns/win/sysstream.c b/libs/kns/win/sysstream.c index 55e7e87f3..0708c0308 100644 --- a/libs/kns/win/sysstream.c +++ b/libs/kns/win/sysstream.c @@ -72,7 +72,7 @@ rc_t CC KStdIOStreamRead ( const KStdIOStream *self, to_read = ( DWORD ) bsize; if ( ( size_t ) to_read < bsize ) - to_read = -1; + to_read = ( DWORD ) -1; while ( 1 ) { @@ -116,7 +116,7 @@ rc_t CC KStdIOStreamWrite ( KStdIOStream *self, to_write = ( DWORD ) size; if ( ( size_t ) to_write < size ) - to_write = -1; + to_write = ( DWORD ) -1; * num_writ = 0; diff --git a/libs/ncbi-vdb/libncbi-vdb.vers b/libs/ncbi-vdb/libncbi-vdb.vers index a909317fe..fd2a01863 100644 --- a/libs/ncbi-vdb/libncbi-vdb.vers +++ b/libs/ncbi-vdb/libncbi-vdb.vers @@ -1 +1 @@ -3.0.10 +3.1.0 diff --git a/libs/vdb/libvdb.vers.h b/libs/vdb/libvdb.vers.h index 93083fdd8..5eb0968a2 100644 --- a/libs/vdb/libvdb.vers.h +++ b/libs/vdb/libvdb.vers.h @@ -24,4 +24,4 @@ * */ -#define LIBVDB_VERS 0x0300000A +#define LIBVDB_VERS 0x03010000 diff --git a/libs/vfs/services-priv.h b/libs/vfs/services-priv.h index 8c245c986..84e1e83f4 100644 --- a/libs/vfs/services-priv.h +++ b/libs/vfs/services-priv.h @@ -135,9 +135,6 @@ const KSrvResponse * KSrvRunIteratorGetResponse( 2: don't resolve */ rc_t KServiceResolveName ( struct KService * service, int resolve ); -/* Set quality type in service request */ -rc_t KServiceSetQuality(KService * self, const char * quality); - bool KServiceSkipLocal(const KService * self); bool KServiceSkipRemote(const KService * self); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 43ae54659..bf52f1806 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -60,6 +60,7 @@ add_subdirectory(cloud) add_subdirectory(krypto) add_subdirectory(cipher) add_subdirectory(kdb) +add_subdirectory(kdbtext) add_subdirectory(kproc) add_subdirectory(vdb) add_subdirectory(search) diff --git a/test/kdb/test-rcolumn.cpp b/test/kdb/test-rcolumn.cpp index 1f83637b4..01b0b77ed 100644 --- a/test/kdb/test-rcolumn.cpp +++ b/test/kdb/test-rcolumn.cpp @@ -130,7 +130,7 @@ FIXTURE_TEST_CASE(KRColumn_OpenParentRead, KColumn_Fixture) { Setup( GetName() ); const KTable * tbl = nullptr; - rc_t rc = SILENT_RC ( rcVDB, rcTable, rcAccessing, rcSelf, rcNull ); + rc_t rc = SILENT_RC ( rcDB, rcTable, rcAccessing, rcSelf, rcNull ); // no parent set REQUIRE_EQ( rc, KColumnOpenParentRead( & m_col -> dad, & tbl ) ); REQUIRE_NULL( tbl ); } diff --git a/test/kdb/test-wcolumn.cpp b/test/kdb/test-wcolumn.cpp index 201f08da0..5300fdc03 100644 --- a/test/kdb/test-wcolumn.cpp +++ b/test/kdb/test-wcolumn.cpp @@ -134,7 +134,7 @@ FIXTURE_TEST_CASE(KWColumn_OpenParentRead, KColumn_Fixture) { Setup( GetName() ); const KTable * tbl = nullptr; - rc_t rc = SILENT_RC ( rcVDB, rcTable, rcAccessing, rcSelf, rcNull ); + rc_t rc = SILENT_RC ( rcDB, rcTable, rcAccessing, rcSelf, rcNull ); // no parent set REQUIRE_EQ( rc, KColumnOpenParentRead( & m_col -> dad, & tbl ) ); REQUIRE_NULL( tbl ); } @@ -143,7 +143,7 @@ FIXTURE_TEST_CASE(KWColumn_OpenMetadataRead, KColumn_Fixture) { Setup( GetName() ); const KMetadata * meta = nullptr; - rc_t rc = SILENT_RC ( rcDB,rcMgr,rcOpening,rcMetadata,rcNotFound ); + rc_t rc = SILENT_RC ( rcDB, rcMgr, rcOpening, rcMetadata, rcNotFound ); REQUIRE_EQ( rc, KColumnOpenMetadataRead( & m_col -> dad, & meta ) ); } @@ -151,7 +151,7 @@ FIXTURE_TEST_CASE(KWColumn_OpenBlobRead, KColumn_Fixture) { Setup( GetName() ); const KColumnBlob * blob = nullptr; - rc_t rc = SILENT_RC ( rcDB,rcColumn,rcSelecting,rcBlob,rcNotFound ); + rc_t rc = SILENT_RC ( rcDB, rcColumn, rcSelecting,rcBlob,rcNotFound ); REQUIRE_EQ( rc, KColumnOpenBlobRead( & m_col -> dad, & blob, 1 ) ); } diff --git a/test/kdbtext/CMakeLists.txt b/test/kdbtext/CMakeLists.txt new file mode 100644 index 000000000..1c9513268 --- /dev/null +++ b/test/kdbtext/CMakeLists.txt @@ -0,0 +1,43 @@ +# =========================================================================== +# +# PUBLIC DOMAIN NOTICE +# National Center for Biotechnology Information +# +# This software/database is a "United States Government Work" under the +# terms of the United States Copyright Act. It was written as part of +# the author's official duties as a United States Government employee and +# thus cannot be copyrighted. This software/database is freely available +# to the public for use. The National Library of Medicine and the U.S. +# Government have not placed any restriction on its use or reproduction. +# +# Although all reasonable efforts have been taken to ensure the accuracy +# and reliability of the software and data, the NLM and the U.S. +# Government do not and cannot warrant the performance or results that +# may be obtained by using this software or data. The NLM and the U.S. +# Government disclaim all warranties, express or implied, including +# warranties of performance, merchantability or fitness for any particular +# purpose. +# +# Please cite the author in any work or product based on this material. +# +# =========================================================================== + +# if( WIN32 ) +# set( ADDITIONAL_LIBS Crypt32 ) +# set( TMPDIR ./data) +# else() +# set( ADDITIONAL_LIBS "" ) +# set( TMPDIR /tmp ) +# endif() + +add_compile_definitions( __mod__="test/kdbtext" ) + +set( LIBS ${COMMON_LIBS_READ};kdbtext ) +AddExecutableTest( Test_KDBText_Path "test-path" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Metadata "test-metadata" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Index "test-index" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Column "test-column" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Table "test-table" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Database "test-database" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Manager "test-manager" "${LIBS}" ) +AddExecutableTest( Test_KDBText_Blob "test-blob" "${LIBS}" ) diff --git a/test/kdbtext/Makefile b/test/kdbtext/Makefile new file mode 100644 index 000000000..d9b71f448 --- /dev/null +++ b/test/kdbtext/Makefile @@ -0,0 +1,30 @@ +# =========================================================================== +# +# PUBLIC DOMAIN NOTICE +# National Center for Biotechnology Information +# +# This software/database is a "United States Government Work" under the +# terms of the United States Copyright Act. It was written as part of +# the author's official duties as a United States Government employee and +# thus cannot be copyrighted. This software/database is freely available +# to the public for use. The National Library of Medicine and the U.S. +# Government have not placed any restriction on its use or reproduction. +# +# Although all reasonable efforts have been taken to ensure the accuracy +# and reliability of the software and data, the NLM and the U.S. +# Government do not and cannot warrant the performance or results that +# may be obtained by using this software or data. The NLM and the U.S. +# Government disclaim all warranties, express or implied, including +# warranties of performance, merchantability or fitness for any particular +# purpose. +# +# Please cite the author in any work or product based on this material. +# +# =========================================================================== + +default: runtests + +TOP ?= $(abspath ../..) +MODULE = test/kdbtext + +include $(TOP)/build/Makefile.env diff --git a/test/kdbtext/test-blob.cpp b/test/kdbtext/test-blob.cpp new file mode 100644 index 000000000..a685d563d --- /dev/null +++ b/test/kdbtext/test-blob.cpp @@ -0,0 +1,305 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBColumn for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/columnblob.hpp" + +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KTextColumnBlobTestSuite); + +const string Data = "0123456789"; + +class KTextColumnBlob_Fixture +{ +public: + KTextColumnBlob_Fixture() + { + } + ~KTextColumnBlob_Fixture() + { + delete m_blob; + } + + void Setup( const string & data = Data ) + { + m_blob = new ColumnBlob( data.c_str(), data.size(), nullptr, 1, 2 ); + } + + ColumnBlob * m_blob = nullptr; +}; + +FIXTURE_TEST_CASE(KTextColumnBlob_Make_Empty, KTextColumnBlob_Fixture) +{ + Setup(""); + REQUIRE_EQ( string(), string( (const char*) (m_blob -> getData()), m_blob -> getSize() ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_Make_Data, KTextColumnBlob_Fixture) +{ + Setup( Data ); + REQUIRE_EQ( Data, string( (const char*) (m_blob -> getData()), m_blob -> getSize() ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_GetIdRange, KTextColumnBlob_Fixture) +{ + Setup( Data ); + REQUIRE_EQ( Data, string( (const char*) (m_blob -> getData()), m_blob -> getSize() ) ); + REQUIRE_EQ( (int64_t)1, m_blob -> getIdRange().first ); + REQUIRE_EQ( (uint32_t)2, m_blob -> getIdRange().second ); +} + +class KTextColumnBlob_ApiFixture +{ +public: + KTextColumnBlob_ApiFixture() + { + KDataBufferMakeBytes( & m_buffer, 0 ); + } + ~KTextColumnBlob_ApiFixture() + { + KDataBufferWhack( & m_buffer ); + KColumnBlobRelease( m_blob ); + } + void Setup( const string & data = "abcdef" ) + { + m_blob = (const KColumnBlob*)new ColumnBlob( data.c_str(), data.size(), nullptr, 1, 2 ); + } + + const KColumnBlob * m_blob = nullptr; + + KDataBuffer m_buffer; +}; + +FIXTURE_TEST_CASE(KTextColumnBlob_AddRelease, KTextColumnBlob_ApiFixture) +{ + Setup(); + + REQUIRE_RC( KColumnBlobAddRef( m_blob ) ); + REQUIRE_RC( KColumnBlobRelease( m_blob ) ); + // use valgrind to find any leaks +} + +FIXTURE_TEST_CASE(KTextColumnBlob_BufferNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + size_t num_read = 0; + size_t remaining = 0; + REQUIRE_RC_FAIL( KColumnBlobRead ( m_blob, 0, nullptr, 0, &num_read, &remaining ) ); +} +FIXTURE_TEST_CASE(KTextColumnBlob_NumReadNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + char data[1024] = {0}; + size_t remaining = 0; + REQUIRE_RC_FAIL( KColumnBlobRead ( m_blob, 0, data, sizeof data, nullptr, &remaining ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_Read_From_0, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + char data[1024] = {0}; + size_t num_read = 0; + size_t remaining = 0; + REQUIRE_RC( KColumnBlobRead ( m_blob, 0, data, sizeof data, &num_read, &remaining ) ); + REQUIRE_EQ( Data, string(data, num_read ) ); + REQUIRE_EQ( (size_t)0, remaining ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_Read_From_Offset, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + char data[1024] = {0}; + size_t num_read = 0; + size_t remaining = 0; + const size_t Offset = 3; + REQUIRE_RC( KColumnBlobRead ( m_blob, Offset, data, sizeof data, &num_read, &remaining ) ); + REQUIRE_EQ( Data.substr( Offset ), string(data, num_read ) ); + REQUIRE_EQ( (size_t)0, remaining ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_Read_Offset_BeyondData, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + char data[1024] = {0}; + size_t num_read = 0; + REQUIRE_RC_FAIL( KColumnBlobRead ( m_blob, Data.size() + 1, data, sizeof data, &num_read, nullptr ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_Read_BufferTooShort, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + char data[4] = {0}; + size_t num_read = 0; + size_t remaining = 0; + REQUIRE_RC( KColumnBlobRead ( m_blob, 0, data, sizeof data, &num_read, &remaining ) ); + REQUIRE_EQ( sizeof data, num_read ); + REQUIRE_EQ( Data.size() - num_read, remaining ); + REQUIRE_EQ( Data.substr( 0, num_read ), string(data, num_read ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_ReadAll_BufferNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + REQUIRE_RC_FAIL( KColumnBlobReadAll ( m_blob, nullptr, nullptr, 0 ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_ReadAll_NoCS, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + REQUIRE_RC( KColumnBlobReadAll ( m_blob, & m_buffer, nullptr, 0 ) ); + REQUIRE_EQ( Data, string( (const char*)m_buffer.base, m_buffer.elem_count ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_ReadAll_WithCS, KTextColumnBlob_ApiFixture) +{ // checksum data is set to all 0s + Setup( Data ); + KColumnBlobCSData cs0; + memset ( & cs0, 0, sizeof( KColumnBlobCSData ) ); + KColumnBlobCSData cs; + memset ( & cs, 1, sizeof( KColumnBlobCSData ) ); + + REQUIRE_RC( KColumnBlobReadAll ( m_blob, & m_buffer, & cs, sizeof( KColumnBlobCSData ) ) ); + REQUIRE_EQ( Data, string( (const char*)m_buffer.base, m_buffer.elem_count ) ); + REQUIRE_EQ( 0, memcmp( & cs0, & cs, sizeof( KColumnBlobCSData ) ) ); +} + +// always succeeds +FIXTURE_TEST_CASE(KTextColumnBlob_Validate, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + REQUIRE_RC( KColumnBlobValidate ( m_blob ) ); +} + +// only checks size +FIXTURE_TEST_CASE(KTextColumnBlob_ValidateBuffer_BufferNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + + KColumnBlobCSData cs; + REQUIRE_RC_FAIL( KColumnBlobValidateBuffer ( m_blob, nullptr, &cs, 0 ) ); +} +FIXTURE_TEST_CASE(KTextColumnBlob_ValidateBuffer_CsNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + REQUIRE_RC_FAIL( KColumnBlobValidateBuffer ( m_blob, &m_buffer, nullptr, 0 ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_ValidateBuffer_SizeUnder, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + KDataBufferMakeBytes( & m_buffer, Data.size() - 1 ) ; + KColumnBlobCSData cs; + + REQUIRE_RC_FAIL( KColumnBlobValidateBuffer ( m_blob, &m_buffer, &cs, 0 ) ); +} +FIXTURE_TEST_CASE(KTextColumnBlob_ValidateBuffer_SizeOver, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + KDataBufferMakeBytes( & m_buffer, Data.size() + 1 ) ; + KColumnBlobCSData cs; + + REQUIRE_RC_FAIL( KColumnBlobValidateBuffer ( m_blob, &m_buffer, &cs, 0 ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_ValidateBuffer, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + KDataBufferMakeBytes( & m_buffer, Data.size() ) ; + KColumnBlobCSData cs; + + REQUIRE_RC( KColumnBlobValidateBuffer ( m_blob, &m_buffer, &cs, 0 ) ); +} + +FIXTURE_TEST_CASE(KTextColumnBlob_IdRange_FirstNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + uint32_t count; + REQUIRE_RC_FAIL( KColumnBlobIdRange ( m_blob, nullptr, &count ) ); +} +FIXTURE_TEST_CASE(KTextColumnBlob_IdRange_CountNull, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + int64_t first; + REQUIRE_RC_FAIL( KColumnBlobIdRange ( m_blob, &first, nullptr ) ); +} +FIXTURE_TEST_CASE(KTextColumnBlob_IdRange, KTextColumnBlob_ApiFixture) +{ + Setup( Data ); + int64_t first = 0; + uint32_t count = 0; + REQUIRE_RC( KColumnBlobIdRange ( m_blob, &first, &count ) ); + REQUIRE_EQ( (int64_t)1, first ); + REQUIRE_EQ( (uint32_t)2, count ); +} + +//KTextColumnBlobIdRange ( const KColumnBlob *self, int64_t *first, uint32_t *count ); + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Column"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KTextColumnBlobTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-column.cpp b/test/kdbtext/test-column.cpp new file mode 100644 index 000000000..cf34e1801 --- /dev/null +++ b/test/kdbtext/test-column.cpp @@ -0,0 +1,314 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBColumn for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/column.hpp" + +#include +#include +#include +#include + +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KTextColumnTestSuite); + +const char * TestColumn = R"({"name":"col", + "type":"ascii", + "data": + [ + {"row":1,"value":"AGCT"}, + {"row":2,"value":"AGCT"} + ]})"; + +class KTextColumn_Fixture +{ +public: + KTextColumn_Fixture() + { + } + ~KTextColumn_Fixture() + { + delete m_col; + KJsonValueWhack( m_json ); + } + + void Setup( const char * input ) + { + THROW_ON_RC( KJsonValueMake ( & m_json, input, m_error, sizeof m_error ) ); + THROW_ON_FALSE( jsObject == KJsonGetValueType ( m_json ) ); + + const KJsonObject * json = KJsonValueToObject ( m_json ); + THROW_ON_FALSE( json != nullptr ); + + m_col = new Column( json, nullptr ); + } + + KJsonValue * m_json = nullptr; + Column * m_col = nullptr; + char m_error[1024] = {0}; +}; + +FIXTURE_TEST_CASE(KTextColumn_Make_Empty, KTextColumn_Fixture) +{ + Setup(R"({})"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextColumn_Make_TypeMissing, KTextColumn_Fixture) +{ + Setup(R"({"name":"col"})"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextColumn_Make, KTextColumn_Fixture) +{ + Setup(TestColumn); + REQUIRE_RC( m_col -> inflate( m_error, sizeof m_error ) ); + REQUIRE_EQ( string("col"), m_col->getName() ); +} + +FIXTURE_TEST_CASE(KTextColumn_Make_DataNotArray, KTextColumn_Fixture) +{ + Setup(R"({"name":"col","data":{}})"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextColumn_Make_CellNotObject, KTextColumn_Fixture) +{ + Setup(R"({"name":"col","data":[1]})"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextColumn_CellMake_RowMissing, KTextColumn_Fixture) +{ + Setup(R"({"name":"col","data":[{}]})"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextColumn_CellMake_RowBad, KTextColumn_Fixture) +{ + Setup(R"({"name":"col","data": [ {"row":"a","value":"q"} ] })"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextColumn_CellMake_ValueMissing, KTextColumn_Fixture) +{ + Setup(R"({"name":"col","data": [ {"row":"1"} ] })"); + REQUIRE_RC_FAIL( m_col -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextColumn_IdRange, KTextColumn_Fixture) +{ + Setup(TestColumn); + REQUIRE_RC( m_col -> inflate( m_error, sizeof m_error ) ); + auto r = m_col->idRange(); + REQUIRE_EQ( (int64_t)1, r . first ); + REQUIRE_EQ( (uint64_t)2, r . second ); +} + +// API + +const char * FullTable = R"({"type": "table", "name": "testtbl", + "columns":[ + { + "name":"col", + "type":"ascii", + "data": + [ + {"row":1,"value":"AGCT"}, + {"row":2,"value":"AGCT"}, + {"row":11,"value":"AGCT"}, + {"row":22,"value":"AGCT"} + ], + "metadata":{"name":"", "value":"blah"} + } + ] +})"; + +class KTextColumn_ApiFixture +{ +public: + KTextColumn_ApiFixture() + { + } + ~KTextColumn_ApiFixture() + { + KColumnRelease( m_col ); + KTableRelease( m_tbl ); + KDBManagerRelease( m_mgr ); + } + void Setup( const char * input, const char * col ) + { + try + { + THROW_ON_RC( KDBManagerMakeText ( & m_mgr, input, m_error, sizeof m_error ) ); + THROW_ON_RC( KDBManagerOpenTableRead( m_mgr, & m_tbl, "%s", "testtbl" ) ); + THROW_ON_RC( KTableOpenColumnRead( m_tbl, & m_col, "%s", col ) ); + } + catch(const std::exception& e) + { + std::cerr << e.what() << " with '" << m_error << "'" << endl; + throw; + } + + } + + const KDBManager * m_mgr = nullptr; + const KTable * m_tbl = nullptr; + const KColumn * m_col = nullptr; + char m_error[1024] = {0}; +}; + +FIXTURE_TEST_CASE(KColumn_AddRelease, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + + REQUIRE_RC( KColumnAddRef( m_col ) ); + REQUIRE_RC( KColumnRelease( m_col ) ); + // use valgrind to find any leaks +} + +FIXTURE_TEST_CASE(KColumn_Locked, KTextColumn_ApiFixture) +{ // always false for this library + Setup(FullTable, "col"); + REQUIRE( ! KColumnLocked( m_col ) ); +} +FIXTURE_TEST_CASE(KColumn_Version, KTextColumn_ApiFixture) +{ // always 0 for this library + Setup(FullTable, "col"); + uint32_t version = 99; + REQUIRE_RC( KColumnVersion( m_col, &version ) ); + REQUIRE_EQ( (uint32_t)0, version ); +} +FIXTURE_TEST_CASE(KColumn_ByteOrder, KTextColumn_ApiFixture) +{ // always false for this library + Setup(FullTable, "col"); + bool reversed = true; + REQUIRE_RC( KColumnByteOrder( m_col, &reversed ) ); + REQUIRE( ! reversed ); +} + +FIXTURE_TEST_CASE(KColumn_IdRange, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + int64_t first = 0; + uint64_t count = 0; + REQUIRE_RC( KColumnIdRange ( m_col, & first, & count ) ); + REQUIRE_EQ( (int64_t)1, first ); + REQUIRE_EQ( (uint64_t)22, count ); +} + +FIXTURE_TEST_CASE(KColumn_FindFirstRowId, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + int64_t found = 0; + REQUIRE_RC( KColumnFindFirstRowId ( m_col, & found, 4 ) ); + REQUIRE_EQ( (int64_t)11, found ); +} + +FIXTURE_TEST_CASE(KColumn_OpenManagerRead, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + const KDBManager * mgr = nullptr; + REQUIRE_RC( KColumnOpenManagerRead ( m_col, & mgr ) ); + REQUIRE_EQ( m_mgr, mgr ); + REQUIRE_RC( KDBManagerRelease( mgr ) ); +} + +FIXTURE_TEST_CASE(KColumn_OpenParentRead, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + + const KTable * par = nullptr; + REQUIRE_RC( KColumnOpenParentRead( m_col, & par ) ); + REQUIRE_NOT_NULL( par ); + REQUIRE_EQ( m_tbl, par ); + KTableRelease( par ); +} + +FIXTURE_TEST_CASE(KColumn_OpenMetadataRead, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + const KMetadata * m = nullptr; + REQUIRE_RC( KColumnOpenMetadataRead ( m_col, &m ) ); + REQUIRE_NOT_NULL( m ); + REQUIRE_RC( KMetadataRelease( m ) ); +} + +FIXTURE_TEST_CASE(KColumn_OpenBlobRead, KTextColumn_ApiFixture) +{ + Setup(FullTable, "col"); + const KColumnBlob * blob = nullptr; + REQUIRE_RC( KColumnOpenBlobRead ( m_col, & blob, 11 ) ); + REQUIRE_NOT_NULL( blob ); + REQUIRE_RC( KColumnBlobRelease( blob ) ); +} + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Column"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KTextColumnTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-database.cpp b/test/kdbtext/test-database.cpp new file mode 100644 index 000000000..e31dc6354 --- /dev/null +++ b/test/kdbtext/test-database.cpp @@ -0,0 +1,597 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBDatabase for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/database.hpp" +#include "../../libs/kdbtext/table.hpp" +#include "../../libs/kdbtext/metadata.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KTextDatabaseTestSuite); + +const char * NestedDb = R"({ + "type": "database", + "name": "testdb", + "databases":[ + {"type": "database", "name":"subdb1"}, + {"type": "database", "name":"subdb2","tables":[ + {"type": "table", "name": "tbl2-1"}, + {"type": "table", "name": "tbl2-2"} + ] + } + ], + "tables":[ + {"type": "table", "name": "tbl0-1"}, + {"type": "table", "name": "tbl0-2"} + ] +})"; + +const char * DbWithMeta = R"({ + "type": "database", + "name": "testdb", + "metadata": { "name":"root", "value":"blah" } +})"; + +class KTextDatabase_Fixture +{ +public: + KTextDatabase_Fixture() + { + } + ~KTextDatabase_Fixture() + { + delete m_db; + KJsonValueWhack( m_json ); + } + + void Setup( const char * input ) + { + THROW_ON_RC( KJsonValueMake ( & m_json, input, m_error, sizeof m_error ) ); + THROW_ON_FALSE( jsObject == KJsonGetValueType ( m_json ) ); + + const KJsonObject * json = KJsonValueToObject ( m_json ); + THROW_ON_FALSE( json != nullptr ); + + m_db = new Database( json ); + } + void SetupAndInflate( const char * input ) + { + Setup( input ); + THROW_ON_RC( m_db -> inflate( m_error, sizeof m_error ) ); + } + + KJsonValue * m_json = nullptr; + Database * m_db = nullptr; + char m_error[1024] = {0}; +}; + +FIXTURE_TEST_CASE(KTextDatabase_Make_Empty, KTextDatabase_Fixture) +{ + Setup(R"({})"); + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_NoName, KTextDatabase_Fixture) +{ + Setup(R"({"type": "database"})"); + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_NoType, KTextDatabase_Fixture) +{ + Setup(R"({"name": "testdb"})"); + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_InvalidType, KTextDatabase_Fixture) +{ + Setup(R"({"type": [], "name": "testdb"})"); + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_WrongType, KTextDatabase_Fixture) +{ + Setup(R"({"type": "table", "name": "testdb"})"); + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextDatabase_Make_Flat, KTextDatabase_Fixture) +{ + SetupAndInflate(R"({"type": "database", "name": "testdb"})"); + REQUIRE_EQ( string("testdb"), m_db -> getName() ); +} + +FIXTURE_TEST_CASE(KTextDatabase_Make_Nested_NotArray, KTextDatabase_Fixture) +{ + Setup(R"({"type": "database", "name": "testdb", + "databases":{"type": "database", "name":"subdb1"} + })"); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Nested_ElementNull, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb","databases":[ null ]})" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Nested_ElementBad, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", + "databases":[ {"type": "NOTAdatabase", "name":"subdb1"} ] + })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Nested_Duplicate, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", + "databases":[ + {"type": "database", "name":"subdb1"} , + {"type": "database", "name":"subdb1"} + ] + })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextDatabase_Make_Tables_NotArray, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", "tables":{"type":"table","name":"tbl"} })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Tables_ElementNull, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", "tables":[null] })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Tables_ElementBad, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", "tables":[{"type":"NOTAtable","name":"tbl"}] })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextDatabase_Make_Tables_ElementDuplicate, KTextDatabase_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb", + "tables":[ + {"type":"table","name":"tbl"}, + {"type":"table","name":"tbl"} + ] + })" ); + + REQUIRE_RC_FAIL( m_db -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextDatabase_Make_Nested, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + + { Path p( "notthere" ); REQUIRE_NULL( m_db -> openDatabase( p ) ); } + { + Path p( "testdb/db/subdb1" ); + const Database * d = m_db -> openDatabase( p ); + REQUIRE_NOT_NULL( d ); + delete d; + } + { + Path p( "testdb/db/subdb2" ); + const Database * d = m_db -> openDatabase( p ); + REQUIRE_NOT_NULL( d ); + delete d; + } + + { Path p( "notthere"); REQUIRE_NULL( m_db -> openTable( p ) ); } + { + Path p( "testdb/tbl/tbl0-1"); + const Table * t = m_db -> openTable( p ); + REQUIRE_NOT_NULL( t ); + delete t; + } + { + Path p( "testdb/tbl/tbl0-2"); + const Table * t = m_db -> openTable( p ); + REQUIRE_NOT_NULL( t ); + delete t; + } +} +FIXTURE_TEST_CASE(KTextDatabase_openDatabase, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb" ); + const Database * db = m_db -> openDatabase( p ); + REQUIRE_NOT_NULL( db ); + REQUIRE_EQ( string("testdb"), db->getName() ); + Database::release( db ); +} +FIXTURE_TEST_CASE(KTextDatabase_openDatabase_Nested, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2" ); + const Database * db = m_db -> openDatabase( p ); + REQUIRE_NOT_NULL( db ); + REQUIRE_EQ( string("subdb2"), db->getName() ); + Database::release( db ); +} +FIXTURE_TEST_CASE(KTextDatabase_openSubDatabase, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + const Database * db = m_db -> openSubDatabase( "subdb2" ); + REQUIRE_NOT_NULL( db ); + REQUIRE_EQ( string("subdb2"), db->getName() ); + Database::release( db ); +} + +FIXTURE_TEST_CASE(KTextDatabase_openTable, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/tbl/tbl0-1" ); + const Table * tbl = m_db -> openTable( p ); + REQUIRE_NOT_NULL( tbl ); + REQUIRE_EQ( string("tbl0-1"), tbl->getName() ); + delete tbl; +} +FIXTURE_TEST_CASE(KTextDatabase_openTable_Nested, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2/tbl/tbl2-2" ); + const Table * tbl = m_db -> openTable( p ); + REQUIRE_NOT_NULL( tbl ); + REQUIRE_EQ( string("tbl2-2"), tbl->getName() ); + delete tbl; +} + +FIXTURE_TEST_CASE(KTextDatabase_pathType_empty, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "" ); + REQUIRE_EQ( (int)kptNotFound, m_db -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_pathType_miss, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "qq" ); + REQUIRE_EQ( (int)kptNotFound, m_db -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_pathType_self, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb" ); + REQUIRE_EQ( (int)kptDatabase, m_db -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_pathType_nestedDb, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2" ); + REQUIRE_EQ( (int)kptDatabase, m_db -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_pathType_nestedTable, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2/tbl/tbl2-2" ); + REQUIRE_EQ( (int)kptTable, m_db -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_pathType_metadata, KTextDatabase_Fixture) +{ + SetupAndInflate( DbWithMeta ); + Path p( "testdb/md" ); + REQUIRE_EQ( (int)kptMetadata, m_db -> pathType( p ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_exists_empty, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "" ); + REQUIRE( ! m_db -> exists( kptDatabase, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_Database_Not, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "notadb" ); + REQUIRE( ! m_db -> exists( kptDatabase, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_Database_WrongType, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb" ); + REQUIRE( ! m_db -> exists( kptTable, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_Database_Root, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb" ); + REQUIRE( m_db -> exists( kptDatabase, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_Database_Nested, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2" ); + REQUIRE( m_db -> exists( kptDatabase, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_Table, KTextDatabase_Fixture) +{ + SetupAndInflate( NestedDb ); + Path p( "testdb/db/subdb2/tbl/tbl2-1" ); + REQUIRE( m_db -> exists( kptTable, p ) ); +} +FIXTURE_TEST_CASE(KTextDatabase_exists_metadata, KTextDatabase_Fixture) +{ + SetupAndInflate( DbWithMeta ); + Path p( "testdb/md" ); + REQUIRE( m_db -> exists( kptMetadata, p ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_openMetadata, KTextDatabase_Fixture) +{ + SetupAndInflate( DbWithMeta ); + const Metadata * m = m_db -> openMetadata(); + REQUIRE_NOT_NULL( m ); + REQUIRE_EQ( string("root"), m -> getName() ); +} + +// API + +class KTextDatabase_ApiFixture +{ +public: + KTextDatabase_ApiFixture() + { + } + ~KTextDatabase_ApiFixture() + { + KDatabaseRelease( m_db ); + } + void Setup( const char * input ) + { + try + { + const KDBManager * mgr = nullptr; + THROW_ON_RC( KDBManagerMakeText ( & mgr, input, m_error, sizeof m_error ) ); + THROW_ON_RC( KDBManagerOpenDBRead( mgr, & m_db, "%s", "testdb" ) ); + KDBManagerRelease( mgr ); + } + catch(const std::exception& e) + { + std::cerr << e.what() << " with '" << m_error << "'" << endl; + } + + } + + const KDatabase * m_db = nullptr; + char m_error[1024]; +}; + +FIXTURE_TEST_CASE(KTextDatabase_AddRelease, KTextDatabase_ApiFixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + + REQUIRE_NOT_NULL( m_db ) ; + REQUIRE_RC( KDatabaseAddRef( m_db ) ); + REQUIRE_RC( KDatabaseRelease( m_db ) ); + // use valgrind to find any leaks +} + +FIXTURE_TEST_CASE(KTextDatabase_Locked, KTextDatabase_ApiFixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( ! KDatabaseLocked( m_db ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_Exists, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + REQUIRE( KDatabaseExists( m_db, kptTable, "%s", "testdb/db/subdb2/tbl/tbl2-2" ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_Alias, KTextDatabase_ApiFixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( ! KDatabaseIsAlias( m_db, kptDatabase, nullptr, 0, "testdb" ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_Writable, KTextDatabase_ApiFixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( ! KDatabaseWritable( m_db, kptDatabase, 0, "testdb" ) ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenManagerRead, KTextDatabase_ApiFixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + const KDBManager * mgr; + REQUIRE_RC( KDatabaseOpenManagerRead( m_db, & mgr ) ); + REQUIRE_NOT_NULL( mgr ); + KDBManagerRelease( mgr ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenDbRead, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const KDatabase * subdb = nullptr; + REQUIRE_RC( KDatabaseOpenDBRead( m_db, & subdb, "subdb1" ) ); + REQUIRE_NOT_NULL( subdb ); + KDatabaseRelease( subdb ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenParentRead, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const KDatabase * subdb = nullptr; + REQUIRE_RC( KDatabaseOpenDBRead( m_db, & subdb, "subdb1" ) ); + REQUIRE_NOT_NULL( subdb ); + + const KDatabase * parent = nullptr; + REQUIRE_RC( KDatabaseOpenParentRead( subdb, & parent ) ); + REQUIRE_EQ( m_db, parent ); + KDatabaseRelease( parent ); + KDatabaseRelease( subdb ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenDirectoryRead, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const KDirectory * dir; + rc_t rc = KDatabaseOpenDirectoryRead( m_db, & dir ); + REQUIRE_EQ( SILENT_RC( rcDB, rcDatabase, rcAccessing, rcDirectory, rcUnsupported ), rc ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenTableRead, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const KTable * tbl = nullptr; + REQUIRE_RC( KDatabaseOpenTableRead ( m_db, &tbl, "%s", "tbl0-2" ) ); + REQUIRE_NOT_NULL( tbl ); + KTableRelease( tbl ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenMetadataRead, KTextDatabase_ApiFixture) +{ + Setup( DbWithMeta ); + const KMetadata * m = nullptr; + REQUIRE_RC( KDatabaseOpenMetadataRead ( m_db, &m ) ); + REQUIRE_NOT_NULL( m ); + KMetadataRelease( m ); +} + +FIXTURE_TEST_CASE(KTextDatabase_OpenIndexRead, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const KIndex * idx; + rc_t rc = KDatabaseOpenIndexRead( m_db, & idx, "idx" ); + REQUIRE_EQ( SILENT_RC( rcDB, rcDatabase, rcAccessing, rcIndex, rcUnsupported ), rc ); +} + +FIXTURE_TEST_CASE(KTextDatabase_ListDB, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + KNamelist * names; + REQUIRE_RC( KDatabaseListDB( m_db, & names ) ); + REQUIRE_NOT_NULL( names ); + uint32_t count = 0; + REQUIRE_RC( KNamelistCount ( names, & count ) ); + REQUIRE_EQ( (uint32_t)2, count ); + REQUIRE( KNamelistContains( names, "subdb1" ) ); + REQUIRE( KNamelistContains( names, "subdb2" ) ); + KNamelistRelease( names ); +} + +FIXTURE_TEST_CASE(KTextDatabase_ListTbl, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + KNamelist * names; + REQUIRE_RC( KDatabaseListTbl( m_db, & names ) ); + REQUIRE_NOT_NULL( names ); + uint32_t count = 0; + REQUIRE_RC( KNamelistCount ( names, & count ) ); + REQUIRE_EQ( (uint32_t)2, count ); + REQUIRE( KNamelistContains( names, "tbl0-1" ) ); + REQUIRE( KNamelistContains( names, "tbl0-2" ) ); + KNamelistRelease( names ); +} + +FIXTURE_TEST_CASE(KTextDatabase_ListIdx, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + KNamelist * names; + + REQUIRE_RC( KDatabaseListIdx( m_db, & names ) ); + REQUIRE_NOT_NULL( names ); + uint32_t count = 1; + REQUIRE_RC( KNamelistCount ( names, & count ) ); + REQUIRE_EQ( (uint32_t)0, count ); + + KNamelistRelease( names ); +} + +FIXTURE_TEST_CASE(KTextDatabase_GetPath, KTextDatabase_ApiFixture) +{ + Setup( NestedDb ); + const char * path; + rc_t rc = KDatabaseGetPath( m_db, & path ); + REQUIRE_EQ( SILENT_RC( rcDB, rcDatabase, rcAccessing, rcPath, rcUnsupported ), rc ); +} + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Database"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KTextDatabaseTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-index.cpp b/test/kdbtext/test-index.cpp new file mode 100644 index 000000000..9b5165136 --- /dev/null +++ b/test/kdbtext/test-index.cpp @@ -0,0 +1,117 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBIndex for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/index.hpp" + +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KDBTextIndexTestSuite); + +class KDBTextIndex_Fixture +{ +public: + KDBTextIndex_Fixture() + { + } + ~KDBTextIndex_Fixture() + { + delete m_idx; + KJsonValueWhack( m_json ); + } + + void Setup( const char * input ) + { + THROW_ON_RC( KJsonValueMake ( & m_json, input, m_error, sizeof m_error ) ); + THROW_ON_FALSE( jsObject == KJsonGetValueType ( m_json ) ); + + const KJsonObject * json = KJsonValueToObject ( m_json ); + THROW_ON_FALSE( json != nullptr ); + + m_idx = new Index( json ); + } + + KJsonValue * m_json = nullptr; + Index * m_idx = nullptr; + char m_error[1024] = {0}; +}; + +FIXTURE_TEST_CASE(KDBTextIndex_Make_Empty, KDBTextIndex_Fixture) +{ + Setup(R"({})"); + REQUIRE_RC_FAIL( m_idx -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KDBTextIndex_Make, KDBTextIndex_Fixture) +{ + Setup(R"({"name":"idx"})"); + REQUIRE_RC( m_idx -> inflate( m_error, sizeof m_error ) ); + REQUIRE_EQ( string("idx"), m_idx->getName() ); +} + +//TODO: the rest + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Index"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KDBTextIndexTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-manager.cpp b/test/kdbtext/test-manager.cpp new file mode 100644 index 000000000..bed5ae61a --- /dev/null +++ b/test/kdbtext/test-manager.cpp @@ -0,0 +1,479 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBManager for reading textual data +*/ + +#include + +#include +#include +#include +#include //KDBManagerVPathTypeUnreliable + +#include + +#include +#include + +using namespace std; + +TEST_SUITE(KDBTextManagerTestSuite); + +class KDBTextManager_Fixture +{ +public: + KDBTextManager_Fixture() + { + } + ~KDBTextManager_Fixture() + { + KColumnRelease( m_col ); + + VPathRelease( m_path ); + KDBManagerRelease( m_mgr ); + } + void Setup( const char * input = "{}" ) + { + THROW_ON_RC( KDBManagerMakeText ( & m_mgr, input, m_error, sizeof m_error ) ); + } + void MakeVPath( const char * path ) + { + VFSManager * vfs; + THROW_ON_RC( VFSManagerMake ( & vfs ) ); + THROW_ON_RC( VFSManagerMakePath ( vfs, & m_path, "%s", path ) ); + THROW_ON_RC( VFSManagerRelease( vfs ) ); + } + + const KDBManager * m_mgr = nullptr; + VPath * m_path = nullptr; + const KColumn * m_col = nullptr; + char m_error[1024]; +}; + +FIXTURE_TEST_CASE(KDBTextManager_Make_Null, KDBTextManager_Fixture) +{ + REQUIRE_RC_FAIL( KDBManagerMakeText ( nullptr, "{}", m_error, sizeof m_error ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_Make_BadJson, KDBTextManager_Fixture) +{ + REQUIRE_RC_FAIL( KDBManagerMakeText ( & m_mgr, "notavalidjson", m_error, sizeof m_error ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_Make_WrongJsonRoot, KDBTextManager_Fixture) +{ + REQUIRE_RC_FAIL( KDBManagerMakeText ( & m_mgr, "[]", m_error, sizeof m_error ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_AddRelease, KDBTextManager_Fixture) +{ + Setup(); + + REQUIRE_NOT_NULL( m_mgr ) ; + REQUIRE_RC( KDBManagerAddRef( m_mgr ) ); + REQUIRE_RC( KDBManagerRelease( m_mgr ) ); + // use valgrind to find any leaks +} + +FIXTURE_TEST_CASE(KDBTextManager_Version_Null, KDBTextManager_Fixture) +{ + Setup(); + + REQUIRE_RC_FAIL( KDBManagerVersion( m_mgr, nullptr ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_Version, KDBTextManager_Fixture) +{ + Setup(); + + uint32_t version = 0; + REQUIRE_RC( KDBManagerVersion( m_mgr, & version ) ); + REQUIRE_EQ( (uint32_t)0, version ); +} + +FIXTURE_TEST_CASE(KDBTextManager_Db_Exists_Not, KDBTextManager_Fixture) +{ + Setup(); + REQUIRE( ! KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb" ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_Db_Exists, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb" ) ); +} + +const char * NestedDatabases = R"({ + "type": "database", + "name": "testdb", + "databases": [ + {"type": "database","name":"subdb1"}, + {"type": "database","name":"subdb2","tables":[ + {"type": "table", "name": "tbl1"}, + {"type": "table", "name": "tbl2"} + ]} + ] +})"; + +FIXTURE_TEST_CASE(KDBTextManager_SubDb_Exists_Not, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( ! KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb/db/subdb1" ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_SubDb_Exists, KDBTextManager_Fixture) +{ + Setup( NestedDatabases ); + REQUIRE( KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb/db/subdb1" ) ); + REQUIRE( KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb/db/subdb2" ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_Table_Exists_Not, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE( ! KDBManagerExists( m_mgr, kptDatabase, "%s", "testdb/tbl/tbl1" ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_Table_Exists, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database","name": "testdb", + "tables":[ + {"type": "table", "name": "tbl1"}, + {"type": "table", "name": "tbl2"} + ] + })" ); + REQUIRE( KDBManagerExists( m_mgr, kptTable, "%s", "testdb/tbl/tbl1" ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_Index_Exists, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database","name": "testdb", + "tables":[ + {"type": "table", "name": "tbl1", + "indexes":[ + {"name":"qwer","text":[]} + ]} + ] + })" ); + REQUIRE( KDBManagerExists( m_mgr, kptIndex, "%s", "testdb/tbl/tbl1/idx/qwer" ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_Writable_NotFound, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + rc_t rc = KDBManagerWritable( m_mgr, "%s", "proddb" ); + REQUIRE_EQ( SILENT_RC( rcDB, rcPath, rcAccessing, rcPath, rcNotFound ), rc ); +} +FIXTURE_TEST_CASE(KDBTextManager_Writable_Found, KDBTextManager_Fixture) +{ // for now, any existing object will be reported as readonly + Setup( R"({"type": "database", "name": "testdb"})" ); + rc_t rc = KDBManagerWritable( m_mgr, "%s", "testdb" ); + REQUIRE_EQ( SILENT_RC( rcDB, rcPath, rcAccessing, rcPath, rcReadonly ), rc ); +} + +FIXTURE_TEST_CASE(KDBTextManager_RunPeriodicTasks, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE_RC( KDBManagerRunPeriodicTasks( m_mgr ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_NotFound, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + MakeVPath( "proddb"); + REQUIRE_EQ( (int)kptNotFound, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Db, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + MakeVPath( "testdb"); + REQUIRE_EQ( (int)kptDatabase, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Db_Nested, KDBTextManager_Fixture) +{ + Setup( NestedDatabases ); + MakeVPath( "testdb/db/subdb2"); + REQUIRE_EQ( (int)kptDatabase, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Table_Root, KDBTextManager_Fixture) +{ + Setup( R"({"type": "table", "name": "tbl"})" ); + MakeVPath( "tbl"); + REQUIRE_EQ( (int)kptTable, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Table_Nested, KDBTextManager_Fixture) +{ + Setup( NestedDatabases ); + MakeVPath( "testdb/db/subdb2/tbl/tbl2"); + REQUIRE_EQ( (int)kptTable, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Column_NotFound, KDBTextManager_Fixture) +{ + Setup( R"({ "type": "table", "name": "tbl1", "columns":[{"name":"col1","type":"ascii"}] })" ); + MakeVPath( "tbl1/col/notcol"); + REQUIRE_EQ( (int)kptNotFound, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Column_RootTable, KDBTextManager_Fixture) +{ + Setup( R"({ "type": "table", "name": "tbl1", "columns":[{"name":"col1","type":"ascii"}] })" ); + MakeVPath( "tbl1/col/col1"); + REQUIRE_EQ( (int)kptColumn, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_PathTypeVP_Column_Db, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database","name": "testdb", + "tables":[ + { "type": "table", "name": "tbl1", + "columns":[{"name":"col1","type":"ascii"}] + } + ] + })" ); + MakeVPath( "testdb/tbl/tbl1/col/col1"); + REQUIRE_EQ( (int)kptColumn, KDBManagerPathTypeVP( m_mgr, m_path ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_PathType, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + REQUIRE_EQ( (int)kptDatabase, KDBManagerPathType( m_mgr, "%s", "testdb" ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_VPathTypeUnreliable, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + auto fn = [] ( const KDBManager * self, const char *path, ... ) -> int + { + va_list args; + va_start ( args, path ); + int res = KDBManagerVPathTypeUnreliable ( self, path, args ); + va_end (args); + return res; + }; + REQUIRE_EQ( (int)kptDatabase, fn( m_mgr, "%s", "testdb" ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_OpenDBRead, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + const KDatabase * db = nullptr; + REQUIRE_RC( KDBManagerOpenDBRead( m_mgr, & db, "%s", "testdb" ) ); + REQUIRE_NOT_NULL( db ); + REQUIRE_RC( KDatabaseRelease( db ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_OpenDBRead_Nested, KDBTextManager_Fixture) +{ + Setup( NestedDatabases ); + const KDatabase * db = nullptr; + REQUIRE_RC( KDBManagerOpenDBRead( m_mgr, & db, "%s", "testdb/db/subdb2" ) ); + REQUIRE_NOT_NULL( db ); + REQUIRE_RC( KDatabaseRelease( db ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_OpenTableRead, KDBTextManager_Fixture) +{ // root table + Setup( R"({"type": "table", "name": "tbl"})" ); + const KTable * tbl = nullptr; + REQUIRE_RC( KDBManagerOpenTableRead( m_mgr, & tbl, "%s", "tbl" ) ); + REQUIRE_NOT_NULL( tbl ); + REQUIRE_RC( KTableRelease( tbl ) ); +} +FIXTURE_TEST_CASE(KDBTextManager_OpenTableRead_NonRoot, KDBTextManager_Fixture) +{ + Setup( NestedDatabases ); + const KTable * tbl = nullptr; + REQUIRE_RC( KDBManagerOpenTableRead( m_mgr, & tbl, "%s", "testdb/db/subdb2/tbl/tbl2" ) ); + REQUIRE_NOT_NULL( tbl ); + REQUIRE_RC( KTableRelease( tbl ) ); +} + +FIXTURE_TEST_CASE(KDBTextManager_OpenTableReadVPath, KDBTextManager_Fixture) +{ + Setup( R"({"type": "table", "name": "tbl"})" ); + MakeVPath( "tbl"); + const KTable * tbl = nullptr; + REQUIRE_RC( KDBManagerOpenTableReadVPath( m_mgr, & tbl, m_path ) ); + REQUIRE_NOT_NULL( tbl ); + REQUIRE_RC( KTableRelease( tbl ) ); +} + +FIXTURE_TEST_CASE(KDBManager_VPathOpenLocalDBRead, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + MakeVPath( "testdb"); + const KDatabase * db = nullptr; + REQUIRE_RC( KDBManagerVPathOpenLocalDBRead( m_mgr, & db, m_path ) ); + REQUIRE_NOT_NULL( db ); + REQUIRE_RC( KDatabaseRelease( db ) ); +} + +FIXTURE_TEST_CASE(KDBManager_VPathOpenRemoteDBRead, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + MakeVPath( "testdb"); + const KDatabase * db = nullptr; + rc_t rc = KDBManagerVPathOpenRemoteDBRead( m_mgr, & db, m_path, nullptr ); + REQUIRE_EQ( SILENT_RC( rcDB, rcMgr, rcOpening, rcType, rcInvalid ), rc ); +} + +FIXTURE_TEST_CASE(KDBManager_OpenColumnRead, KDBTextManager_Fixture) +{ + Setup( R"({"type": "database", "name": "testdb"})" ); + const KColumn *col = nullptr; + rc_t rc = KDBManagerOpenColumnRead( m_mgr, & col, "%s", "testdb/col/col1" ); + REQUIRE_EQ( SILENT_RC( rcDB, rcMgr, rcAccessing, rcColumn, rcUnsupported ), rc ); +} + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Manager"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KDBTextManagerTestSuite(argc, argv); + return rc; +} + +} + +#if 0 + struct Database { + let name: String + let metadata: Metadata + + let tables: [Table] + let databases: [Database] + } + struct Node { + struct Attribute { + let name: String + let value: [UInt8] + } + let name: String + let value: [UInt8] + let attributes: [Attribute] + let children: [Node] + } + let root: Node + } + struct Table { + let name: String + let metadata: Metadata + + let columns: [Column] + let indices: [Index] + } + struct Column { + let name: String + let metadata: Metadata + let data: [(row: Int64, value: [UInt8])] + } + struct Index { + struct Value { + let startId: Int64 + let count: UInt64 + } + enum IndexType { + case text([String: Value]) + case rowId([UInt64: Value]) + } + let name: String + let text: IndexType + } + +{ + "type": "database", + "name": "testdb", + "metadata": + { + "name":"", + "value":"blah", + "attributes":{"attr0":"value", "attr1":"attr1value"}, + "children":{ + "name":"schema", + "value":"version 1;....", + "attributes":{}, + "children":{} + } + } + "tables":[ + { + "name":"SEQUENCE", + "metadata":null, + "columns":[ + { + "name":"READ", + "type":"ascii", + "metadata":null, + "data": + [ + {"row":1,"value":"AGCT"}, + {"row":2,"value":"AGCT"} + ] + } + ] + "indexes":[ + { + "name":"qwer", + "text":[ + {"key":"CG", "startId":1, "count":10}, + {"key":"AT", "startId":11, "count":2}, + ], + }, + { + "name":"rewq", + "projection":[ + {"value":"CG", "startId":1, "count":10}, + {"value":"AT", "startId":11, "count":2}, + ], + } + ] + } + ] +} + +metadata testdb.m +{ + name : +} +#endif \ No newline at end of file diff --git a/test/kdbtext/test-metadata.cpp b/test/kdbtext/test-metadata.cpp new file mode 100644 index 000000000..2bbab97d4 --- /dev/null +++ b/test/kdbtext/test-metadata.cpp @@ -0,0 +1,117 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBMetadata for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/metadata.hpp" + +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KDBTextMetadataTestSuite); + +class KDBTextMetadata_Fixture +{ +public: + KDBTextMetadata_Fixture() + { + } + ~KDBTextMetadata_Fixture() + { + delete m_meta; + KJsonValueWhack( m_json ); + } + + void Setup( const char * input ) + { + THROW_ON_RC( KJsonValueMake ( & m_json, input, m_error, sizeof m_error ) ); + THROW_ON_FALSE( jsObject == KJsonGetValueType ( m_json ) ); + + const KJsonObject * json = KJsonValueToObject ( m_json ); + THROW_ON_FALSE( json != nullptr ); + + m_meta = new Metadata( json ); + } + + KJsonValue * m_json = nullptr; + Metadata * m_meta = nullptr; + char m_error[1024] = {0}; +}; + +FIXTURE_TEST_CASE(KDBTextMetadata_Make_Empty, KDBTextMetadata_Fixture) +{ + Setup(R"({})"); + REQUIRE_RC_FAIL( m_meta -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KDBTextMetadata_Make, KDBTextMetadata_Fixture) +{ + Setup(R"({"name":"md"})"); + REQUIRE_RC( m_meta -> inflate( m_error, sizeof m_error ) ); + REQUIRE_EQ( string("md"), m_meta->getName() ); +} + +//TODO: the rest + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Metadata"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KDBTextMetadataTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-path.cpp b/test/kdbtext/test-path.cpp new file mode 100644 index 000000000..a2346c387 --- /dev/null +++ b/test/kdbtext/test-path.cpp @@ -0,0 +1,131 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for a file-system-type path represented as queue of strings +*/ + +#include + +#include "../../libs/kdbtext/path.hpp" + +#include +#include + +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KDBTextPathTestSuite); + +TEST_CASE(KDBTextPath_fromEmptyString) +{ + Path path( string("") ); + REQUIRE( path.empty() ); +} +TEST_CASE(KDBTextPath_fromString) +{ + Path path( string("a/bb/cccc") ); + REQUIRE( ! path.empty() ); + REQUIRE_EQ( string("a"), path.front() ); + path.pop(); + REQUIRE_EQ( string("bb"), path.front() ); + path.pop(); + REQUIRE_EQ( string("cccc"), path.front() ); + path.pop(); + REQUIRE( path.empty() ); +} +TEST_CASE(KDBTextPath_fromFmtArgs) +{ + auto fn = [] ( const char *fmt, ... ) -> Path + { + va_list args; + va_start ( args, fmt ); + return Path( fmt, args ); + }; + Path path = fn( "%s", "a/bb/cccc" ); + REQUIRE( ! path.empty() ); + REQUIRE_EQ( string("a"), path.front() ); + path.pop(); + REQUIRE_EQ( string("bb"), path.front() ); + path.pop(); + REQUIRE_EQ( string("cccc"), path.front() ); + path.pop(); + REQUIRE( path.empty() ); +} + +TEST_CASE(KDBTextPath_fromVPath) +{ + VPath * vpath; + VFSManager * vfs; + REQUIRE_RC( VFSManagerMake ( & vfs ) ); + REQUIRE_RC( VFSManagerMakePath ( vfs, & vpath, "%s", "a/bb/cccc" ) ); + REQUIRE_RC( VFSManagerRelease( vfs ) ); + + Path path ( vpath ); + REQUIRE_RC( VPathRelease( vpath ) ); + + REQUIRE( ! path.empty() ); + REQUIRE_EQ( string("a"), path.front() ); + path.pop(); + REQUIRE_EQ( string("bb"), path.front() ); + path.pop(); + REQUIRE_EQ( string("cccc"), path.front() ); + path.pop(); + REQUIRE( path.empty() ); +} +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Path"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KDBTextPathTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kdbtext/test-table.cpp b/test/kdbtext/test-table.cpp new file mode 100644 index 000000000..59dd09707 --- /dev/null +++ b/test/kdbtext/test-table.cpp @@ -0,0 +1,477 @@ +/*=========================================================================== +* +* PUBLIC DOMAIN NOTICE +* National Center for Biotechnology Information +* +* This software/database is a "United States Government Work" under the +* terms of the United States Copyright Act. It was written as part of +* the author's official duties as a United States Government employee and +* thus cannot be copyrighted. This software/database is freely available +* to the public for use. The National Library of Medicine and the U.S. +* Government have not placed any restriction on its use or reproduction. +* +* Although all reasonable efforts have been taken to ensure the accuracy +* and reliability of the software and data, the NLM and the U.S. +* Government do not and cannot warrant the performance or results that +* may be obtained by using this software or data. The NLM and the U.S. +* Government disclaim all warranties, express or implied, including +* warranties of performance, merchantability or fitness for any particular +* purpose. +* +* Please cite the author in any work or product based on this material. +* +* =========================================================================== +* +*/ + +/** +* Unit tests for KDBTable for reading textual data +*/ + +#include + +#include "../../libs/kdbtext/table.hpp" +#include "../../libs/kdbtext/column.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace std; +using namespace KDBText; + +TEST_SUITE(KTextTableTestSuite); + +class KTextTable_Fixture +{ +public: + KTextTable_Fixture() + { + } + ~KTextTable_Fixture() + { + delete m_tbl; + KJsonValueWhack( m_json ); + } + + void Setup( const char * input ) + { + THROW_ON_RC( KJsonValueMake ( & m_json, input, m_error, sizeof m_error ) ); + THROW_ON_FALSE( jsObject == KJsonGetValueType ( m_json ) ); + + const KJsonObject * json = KJsonValueToObject ( m_json ); + THROW_ON_FALSE( json != nullptr ); + + m_tbl = new Table( json ); + } + void SetupAndInflate( const char * input ) + { + Setup( input ); + THROW_ON_RC( m_tbl -> inflate( m_error, sizeof m_error ) ); + } + + KJsonValue * m_json = nullptr; + Table * m_tbl = nullptr; + char m_error[1024] = {0}; +}; + +const char * FullTable = R"({"type": "table", "name": "testtbl", + "columns":[ {"name":"col1","type":"ascii"},{"name":"col2","type":"ascii"} ], + "indexes":[ {"name":"i1","text":[]}, {"name":"i2","text":[]} ], + "metadata":{"name":"", "value":"blah"} +})"; + +FIXTURE_TEST_CASE(KTextTable_Make_Empty, KTextTable_Fixture) +{ + Setup(R"({})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_NoName, KTextTable_Fixture) +{ + Setup(R"({"type": "table"})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_NoType, KTextTable_Fixture) +{ + Setup(R"({"name": "testtbl"})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_InvalidType, KTextTable_Fixture) +{ + Setup(R"({"type": [], "name": "testtbl"})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_WrongType, KTextTable_Fixture) +{ + Setup(R"({"type": "database", "name": "testtbl"})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextTable_Make_Flat, KTextTable_Fixture) +{ + SetupAndInflate(R"({"type": "table", "name": "testtbl"})"); + REQUIRE_EQ( string("testtbl"), m_tbl -> getName() ); +} + +FIXTURE_TEST_CASE(KTextTable_Make_ColumnsNoArray, KTextTable_Fixture) +{ + Setup(R"({"type": "table", "name": "testtbl","columns":{"name":"col1","type":"ascii"}})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_ColumnDuplicate, KTextTable_Fixture) +{ + Setup(R"({"type": "table", "name": "testtbl","columns":[{"name":"col1","type":"ascii"},{"name":"col1","type":"ascii"}]})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} +FIXTURE_TEST_CASE(KTextTable_Make_ColumnNotObject, KTextTable_Fixture) +{ + Setup(R"({"type": "table", "name": "testtbl","columns":["1"]})"); + REQUIRE_RC_FAIL( m_tbl -> inflate( m_error, sizeof m_error ) ); + //cout << m_error << endl; +} + +FIXTURE_TEST_CASE(KTextTable_Make_WithColumns, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + + REQUIRE( ! m_tbl -> hasColumn( "nocol" ) ); + REQUIRE( m_tbl -> hasColumn( "col1" ) ); + REQUIRE( m_tbl -> hasColumn( "col2" ) ); +} + +FIXTURE_TEST_CASE(KTextTable_Make_WithIndex, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + + REQUIRE( ! m_tbl -> hasIndex( "noidx" ) ); + REQUIRE( m_tbl -> hasIndex( "i1" ) ); + REQUIRE( m_tbl -> hasIndex( "i2" ) ); +} + +FIXTURE_TEST_CASE(KTextTable_Make_WithMetadata, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + REQUIRE_NOT_NULL( m_tbl -> openMetadata() ); +} + +FIXTURE_TEST_CASE(KTextTable_exists_empty, KTextTable_Fixture) +{ + SetupAndInflate(R"({"type": "table", "name": "testtbl"})"); + Path p( "" ); + REQUIRE( ! m_tbl -> exists( kptTable, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists_WrongType, KTextTable_Fixture) +{ + SetupAndInflate(R"({"type": "table", "name": "testtbl"})"); + Path p( "testtbl" ); + REQUIRE( ! m_tbl -> exists( kptDatabase, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl" ); + REQUIRE( m_tbl -> exists( kptTable, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists_Index, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/idx/i1" ); + REQUIRE( m_tbl -> exists( kptIndex, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists_Column, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/col/col2" ); + REQUIRE( m_tbl -> exists( kptColumn, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists_Metadata, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/md" ); + REQUIRE( m_tbl -> exists( kptMetadata, p ) ); +} +FIXTURE_TEST_CASE(KTextTable_exists_Metadata_Not, KTextTable_Fixture) +{ + SetupAndInflate(R"({"type": "table", "name": "testtbl"})"); + Path p( "testtbl/md" ); + REQUIRE( ! m_tbl -> exists( kptMetadata, p ) ); +} + +//pathType +FIXTURE_TEST_CASE(KTextTable_pathType_Empty, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "" ); + REQUIRE_EQ( (int)kptNotFound, m_tbl -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextTable_pathType_Self, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl" ); + REQUIRE_EQ( (int)kptTable, m_tbl -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextTable_pathType_Column, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/col/col2" ); + REQUIRE_EQ( (int)kptColumn, m_tbl -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextTable_pathType_Index, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/idx/i1" ); + REQUIRE_EQ( (int)kptIndex, m_tbl -> pathType( p ) ); +} +FIXTURE_TEST_CASE(KTextTable_pathType_Metadata, KTextTable_Fixture) +{ + SetupAndInflate(FullTable); + Path p( "testtbl/md" ); + REQUIRE_EQ( (int)kptMetadata, m_tbl -> pathType( p ) ); +} + +// API + +class KTextTable_ApiFixture +{ +public: + KTextTable_ApiFixture() + { + } + ~KTextTable_ApiFixture() + { + KTableRelease( m_tbl ); + } + void Setup( const char * input ) + { + try + { + const KDBManager * mgr = nullptr; + THROW_ON_RC( KDBManagerMakeText ( & mgr, input, m_error, sizeof m_error ) ); + THROW_ON_RC( KDBManagerOpenTableRead( mgr, & m_tbl, "%s", "testtbl" ) ); + KDBManagerRelease( mgr ); + } + catch(const std::exception& e) + { + std::cerr << e.what() << " with '" << m_error << "'" << endl; + } + + } + + const KTable * m_tbl = nullptr; + char m_error[1024]; +}; + +FIXTURE_TEST_CASE(KTextTable_AddRelease, KTextTable_ApiFixture) +{ + Setup(FullTable); + + REQUIRE_NOT_NULL( m_tbl ) ; + REQUIRE_RC( KTableAddRef( m_tbl ) ); + REQUIRE_RC( KTableRelease( m_tbl ) ); + // use valgrind to find any leaks +} + +FIXTURE_TEST_CASE(KTextTable_Locked, KTextTable_ApiFixture) +{ + Setup(FullTable); + REQUIRE( ! KTableLocked( m_tbl ) ); +} + +FIXTURE_TEST_CASE(KTextTable_Exists, KTextTable_ApiFixture) +{ + Setup(FullTable); + REQUIRE( KTableExists( m_tbl, kptColumn, "col/%s", "col2" ) ); +} + +FIXTURE_TEST_CASE(KTextTable_IsAlias, KTextTable_ApiFixture) +{ + Setup(FullTable); + REQUIRE( ! KTableIsAlias( m_tbl, kptColumn, nullptr, 0, "col2" ) ); +} + +FIXTURE_TEST_CASE(KTextTable_IsWritable, KTextTable_ApiFixture) +{ + Setup(FullTable); + REQUIRE( ! KTableWritable( m_tbl, kptColumn, "%s", "col2" ) ); +} + +FIXTURE_TEST_CASE(KTextTable_OpenManagerRead, KTextTable_ApiFixture) +{ + Setup(FullTable); + const KDBManager * mgr = nullptr; + REQUIRE_RC( KTableOpenManagerRead( m_tbl, & mgr ) ); + REQUIRE_NOT_NULL( mgr ); + REQUIRE_RC( KDBManagerRelease( mgr ) ); +} + +FIXTURE_TEST_CASE(KTextTable_OpenParentRead_Null, KTextTable_ApiFixture) +{ + Setup(FullTable); + const KDatabase * par = nullptr; + REQUIRE_RC( KTableOpenParentRead( m_tbl, & par ) ); + REQUIRE_NULL( par ); +} +FIXTURE_TEST_CASE(KTextTable_OpenParentRead, KTextTable_ApiFixture) +{ + const char * Db = R"({ + "type": "database", + "name": "testdb", + "tables":[ {"type": "table", "name": "tbl0-1"} ] + })"; + const KDatabase * db; + const KDBManager * mgr = nullptr; + THROW_ON_RC( KDBManagerMakeText ( & mgr, Db, m_error, sizeof m_error ) ); + THROW_ON_RC( KDBManagerOpenDBRead( mgr, & db, "%s", "testdb" ) ); + KDBManagerRelease( mgr ); + + REQUIRE_RC( KDatabaseOpenTableRead ( db, & m_tbl, "%s", "tbl0-1" ) ); + REQUIRE_NOT_NULL( m_tbl ); + + const KDatabase * par = nullptr; + REQUIRE_RC( KTableOpenParentRead( m_tbl, & par ) ); + REQUIRE_NOT_NULL( par ); + REQUIRE_EQ( db, par ); + KDatabaseRelease( par ); + + KDatabaseRelease( db ); +} + +FIXTURE_TEST_CASE(KTextTable_HasRemoteData, KTextTable_ApiFixture) +{ + Setup(FullTable); + REQUIRE( ! KTableHasRemoteData( m_tbl ) ); +} + +FIXTURE_TEST_CASE(KTextTable_OpenDirectoryRead, KTextTable_ApiFixture) +{ + Setup(FullTable); + const KDirectory *dir = nullptr; + rc_t rc = KTableOpenDirectoryRead( m_tbl, & dir ); + REQUIRE_EQ( SILENT_RC ( rcDB, rcTable, rcAccessing, rcDirectory, rcUnsupported ), rc); +} + +FIXTURE_TEST_CASE(KTextTable_OpenColumnRead, KTextTable_ApiFixture) +{ + Setup(FullTable); + const KColumn *col = nullptr; + REQUIRE_RC( KTableOpenColumnRead( m_tbl, & col, "%s", "col2" ) ); + REQUIRE_NOT_NULL( col ); + REQUIRE_RC( KColumnRelease( col ) ); +} + +FIXTURE_TEST_CASE(KTextTable_OpenMetadataRead, KTextTable_ApiFixture) +{ + Setup( FullTable ); + const KMetadata * m = nullptr; + REQUIRE_RC( KTableOpenMetadataRead ( m_tbl, &m ) ); + REQUIRE_NOT_NULL( m ); + KMetadataRelease( m ); +} + +FIXTURE_TEST_CASE(KTextTable_OpenIndexRead, KTextTable_ApiFixture) +{ + Setup( FullTable ); + const KIndex * idx = nullptr; + REQUIRE_RC( KTableOpenIndexRead ( m_tbl, &idx, "%s", "i2" ) ); + REQUIRE_NOT_NULL( idx ); + KIndexRelease( idx ); +} + +FIXTURE_TEST_CASE(KTextTable_GetPath, KTextTable_ApiFixture) +{ + Setup(FullTable); + const char * p = nullptr; + rc_t rc = KTableGetPath( m_tbl, & p ); + REQUIRE_EQ( SILENT_RC ( rcDB, rcTable, rcAccessing, rcPath, rcUnsupported ), rc); +} + +FIXTURE_TEST_CASE(KTextTable_GetName, KTextTable_ApiFixture) +{ + Setup(FullTable); + const char * n = nullptr; + REQUIRE_RC( KTableGetName( m_tbl, & n ) ); + REQUIRE_EQ( string("testtbl"), string(n) ); +} + +FIXTURE_TEST_CASE(KTextTable_ListCol, KTextTable_ApiFixture) +{ + Setup( FullTable ); + KNamelist * names; + REQUIRE_RC( KTableListCol( m_tbl, & names ) ); + REQUIRE_NOT_NULL( names ); + uint32_t count = 0; + REQUIRE_RC( KNamelistCount ( names, & count ) ); + REQUIRE_EQ( (uint32_t)2, count ); + REQUIRE( KNamelistContains( names, "col1" ) ); + REQUIRE( KNamelistContains( names, "col2" ) ); + KNamelistRelease( names ); +} + +FIXTURE_TEST_CASE(KTextTable_ListIdx, KTextTable_ApiFixture) +{ + Setup( FullTable ); + KNamelist * names; + REQUIRE_RC( KTableListIdx( m_tbl, & names ) ); + REQUIRE_NOT_NULL( names ); + uint32_t count = 0; + REQUIRE_RC( KNamelistCount ( names, & count ) ); + REQUIRE_EQ( (uint32_t)2, count ); + REQUIRE( KNamelistContains( names, "i1" ) ); + REQUIRE( KNamelistContains( names, "i2" ) ); + KNamelistRelease( names ); +} + +FIXTURE_TEST_CASE(KTextTable_MetaCompare, KTextTable_ApiFixture) +{ + Setup( FullTable ); + bool equal = false; + rc_t rc = KTableMetaCompare( m_tbl, m_tbl, "", &equal ); + REQUIRE_EQ( SILENT_RC ( rcDB, rcTable, rcComparing, rcMetadata, rcUnsupported ), rc); +} + +//////////////////////////////////////////// Main +extern "C" +{ + +#include +#include + +ver_t CC KAppVersion ( void ) +{ + return 0x1000000; +} +rc_t CC UsageSummary (const char * progname) +{ + return 0; +} + +rc_t CC Usage ( const Args * args ) +{ + return 0; +} + +const char UsageDefaultName[] = "Test_KDBText_Table"; + +rc_t CC KMain ( int argc, char *argv [] ) +{ + KConfigDisableUserSettings(); + rc_t rc=KTextTableTestSuite(argc, argv); + return rc; +} + +} diff --git a/test/kns/test-aws-proxy.cpp b/test/kns/test-aws-proxy.cpp index 20cea15f7..b71562dcf 100644 --- a/test/kns/test-aws-proxy.cpp +++ b/test/kns/test-aws-proxy.cpp @@ -31,8 +31,7 @@ #include // TEST_SUITE #include /* VPathRelease */ #include /* KServiceRelease */ - -#include "../../libs/vfs/services-priv.h" /* KServiceSetQuality */ +#include /* KServiceSetQuality */ #include /* PATH_MAX */ #ifndef PATH_MAX