3.1.0 Release: Merge branch '3.1.0rc'

CHANGES.md

@@ -1,6 +1,15 @@
 # NCBI External Developer Release:
 
 
+## NCBI VDB 3.1.0
+**March 5, 2024**
+
+  **blast**: fixed VdbBlast2naReaderData() for ReferenceSet fills entire output array  
+  **cloud, kns, ngs-tools, sra-tools**: don't change global network timeouts when checking cloud location  
+  **kdb, kdbmeta, sra-tools**: removed confusing warning in Windows debug build   
+  **kns, ngs-tools, sra-tools**: fixed hanging on Mac and BSD when data access is unreliable  
+
+
 ## NCBI VDB 3.0.10
 **December 19, 2023**
 

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # NCBI-VDB and library version
-VERSION = 3.0.10
+VERSION = 3.1.0

interfaces/kdb/consistency-check.h

@@ -89,7 +89,7 @@ struct CCReportInfoBlock
         {
             uint64_t start;
             uint64_t count;
-
+            bool missingChecksum;
         } blob;
 
         struct ccrb_index_s
@@ -99,7 +99,6 @@ struct CCReportInfoBlock
             uint64_t num_keys;
             uint64_t num_rows;
             uint64_t num_holes;
-
         } index;
 
         struct ccrb_visit_s {

interfaces/kdb/manager.h

@@ -51,7 +51,7 @@ struct KDirectory;
  */
 enum
 {
-    /* must be handled carefully and can not go back to KDirectory 
+    /* must be handled carefully and can not go back to KDirectory
      * as this value has a different meaning in KDirectory */
     kptAny = 0,
     kptDatabase = kptLastDefined,
@@ -83,15 +83,16 @@ KDB_EXTERN rc_t CC KDBManagerRelease ( const KDBManager *self );
 
 /* MakeRead
  * MakeUpdate
+ * MakeText (reads data from a Json string)
+ *
  *  create library handle for specific use
- *  NB - only one of the functions will be implemented
  *
  *  "wd" [ IN, NULL OKAY ] - optional working directory for
  *  accessing the file system. mgr will attach its own reference.
  */
 KDB_EXTERN rc_t CC KDBManagerMakeRead ( const KDBManager **mgr, struct KDirectory const *wd );
 KDB_EXTERN rc_t CC KDBManagerMakeUpdate ( KDBManager **mgr, struct KDirectory *wd );
-
+KDB_EXTERN rc_t CC KDBManagerMakeText ( const KDBManager **mgr, const char * input, char * error, size_t error_size );
 
 /* Version
  *  returns the library version

interfaces/klib/vdb_release_version.h

@@ -26,6 +26,6 @@
 * ==============================================================================
 */
 
-#define VDB_RELEASE_VERSION 0x0300000A
+#define VDB_RELEASE_VERSION 0x03010000
 
 #endif /* _h_klib_vdb_release_version_ */

interfaces/kns/http.h

@@ -209,17 +209,29 @@ typedef struct KClientHttpRequest KClientHttpRequest, KHttpRequest;
  *
  *  "vers" [ IN ] - http version
  *
+ *  "connectMillis", "readMillis, writeMillis" [ IN ] - connect/read/write
+ *  timeouts to supply to sockets - when negative, infinite timeout;
+ *  when 0, return immediately; positive gives maximum wait time in sec/mS
+ *  for reads and writes respectively.
+ *
  *  "conn" [ IN, NULL OKAY ] - previously opened stream for communications.
  *
  *  "url" [ IN ] - full resource identifier. if "conn" is NULL,
  *   the url is parsed for remote endpoint and is opened by mgr.
  */
+/* Use timeouts from KNSManager */
 KNS_EXTERN rc_t CC KClientHttpMakeRequest ( const KClientHttp *self,
     KClientHttpRequest **req, const char *url, ... );
 
+/* Use timeouts from KNSManager */
 KNS_EXTERN rc_t CC KNSManagerMakeClientRequest ( struct KNSManager const *self,
     KClientHttpRequest **req, ver_t version, struct KStream *conn, const char *url, ... );
 
+/* Timeouts are specified */
+KNS_EXTERN rc_t CC KNSManagerMakeTimedClientRequest (
+    struct KNSManager const *self, KClientHttpRequest **req,
+    ver_t version, int32_t connMillis, int32_t readMillis,
+    int32_t writeMillis, struct KStream *conn, const char *url, ... );
 
 /* AddRef
  * Release

interfaces/kns/socket.h

@@ -68,8 +68,9 @@ typedef struct KSocket KSocket;
  *  "retryTimeout" [ IN ] - the connect request should be repeated upon failure
  *   until this timeout expires.
  *
- *  "readMillis" [ IN ] and "writeMillis" - when negative, infinite timeout
- *   when 0, return immediately, positive gives maximum wait time in mS
+ *  "connectMillis", "readMillis" and "writeMillis" [ IN ] - when negative,
+ *   infinite timeout;
+ *   when 0, return immediately; positive gives maximum wait time in mS
  *   for reads and writes respectively.
  *
  *  "from" [ IN ] - client endpoint
@@ -78,12 +79,17 @@ typedef struct KSocket KSocket;
  *
  *  both endpoints have to be of type epIP; creates a TCP connection
  */
+ /* Use connectMillis from KNSManager */
 KNS_EXTERN rc_t CC KNSManagerMakeConnection ( struct KNSManager const * self,
     struct KSocket ** conn, struct KEndPoint const * from, struct KEndPoint const * to );
-
 KNS_EXTERN rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * self,
     struct KSocket ** conn, int32_t readMillis, int32_t writeMillis,
     struct KEndPoint const * from, struct KEndPoint const * to );
+/* connectMillis is specified */
+KNS_EXTERN rc_t CC KNSManagerMakeTimedConnectionExt (
+    struct KNSManager const * self, struct KSocket ** conn,
+    int32_t connectMillis, int32_t readMillis, int32_t writeMillis,
+    struct KEndPoint const * from, struct KEndPoint const * to );
 
 KNS_EXTERN rc_t CC KNSManagerMakeRetryConnection ( struct KNSManager const * self,
     struct KSocket ** conn, struct timeout_t * retryTimeout,

interfaces/vfs/services-priv.h

@@ -66,7 +66,9 @@ rc_t KServiceTestNamesQueryExt ( KService * self, VRemoteProtocols protocols,
     const char * dir, const char * file, const char * expected );
 
 
+/* Get/Set quality type in service request */
 rc_t KServiceGetQuality(const KService * self, const char ** quality);
+rc_t KServiceSetQuality(KService * self, const char * quality);
 
 
 rc_t KService1Search ( const struct KNSManager * mgr, const char * cgi,

libs/CMakeLists.txt

@@ -33,6 +33,7 @@ add_subdirectory(cloud)
 add_subdirectory(ext)
 add_subdirectory(kapp)
 add_subdirectory(kdb)
+add_subdirectory(kdbtext)
 add_subdirectory(kfc)
 add_subdirectory(kfg)
 add_subdirectory(kfs)

libs/blast/blast-mgr.c

@@ -50,7 +50,7 @@
 #include <stdio.h> /* fprintf */
 #include <string.h> /* memset */
 
-#define TOOLKIT "sratoolkit3_0_10"
+#define TOOLKIT "sratoolkit3_1_0"
 
 /******************************************************************************/
 

libs/blast/reference.c

@@ -928,7 +928,7 @@ static uint32_t _ReferencesData2na(References *self,
                     self->eos = true;
                     *status = eVdbBlastNoErr; /* end of set */
                     S
-                    return 0;
+                    return num_read;
                 }
                 rfd1 = rfd;
                 rfd = &self->refs->rfd[self->rfdi];
@@ -1193,7 +1193,9 @@ static uint32_t _ReferencesData2na(References *self,
             ++self->read_id;
         }
         ++num_read;
-        break;
+
+        if (*status == eVdbBlastNoErr || *status == eVdbBlastCircularSequence)
+            break;
     }
     return num_read;
 }
@@ -1272,6 +1274,9 @@ uint32_t _Core2naDataRef(struct Core2na *self,
         self->eos = true;
     }
 
+    if (r->eos)
+        self->eos = true;
+
     if (self->eos) {
         return 0;
     }

libs/cloud/cloud-cmn.c

@@ -48,22 +48,11 @@ rc_t KNSManager_Read(const struct KNSManager *cself, char *buffer, size_t bsize,
 
     KClientHttpRequest *req = NULL;
 
-    int32_t cmsec = 0;
-    int32_t rmsec = 0;
-    int32_t wmsec = 0;
-
     assert(self);
 
-    /* save existing timeouts */
-    cmsec = self->conn_timeout;
-    rmsec = self->http_read_timeout;
-    wmsec = self->http_write_timeout;
-
     /* minimize timeouts to check cloudy URLs */
-    self -> conn_timeout
-        = self -> http_read_timeout = self->http_write_timeout = 500;
-
-    rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL, url);
+    rc = KNSManagerMakeTimedClientRequest(
+        self, &req, 0x01010000, 500, 500, 500, NULL, url);
 
     if (rc == 0 && hdrName != NULL)
     {
@@ -108,11 +97,6 @@ rc_t KNSManager_Read(const struct KNSManager *cself, char *buffer, size_t bsize,
         RELEASE(KClientHttpResult, rslt);
     }
 
-    /* restore timeouts in KNSManager */
-    self->conn_timeout = cmsec;
-    self->http_read_timeout = rmsec;
-    self->http_write_timeout = wmsec;
-
     RELEASE(KClientHttpRequest, req);
 
     return rc;

libs/kdb/column-base.c

@@ -105,7 +105,7 @@ rc_t KColumnSever ( const KColumn *self )
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
     else                                        \
-        return RC ( rcVDB, rcColumn, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcColumn, rcAccessing, rcSelf, rcNull );
 #define DISPATCH_BOOL(call)  \
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \

libs/kdb/column-cc.c

@@ -145,6 +145,7 @@ rc_t KColumnCheckBlobs(const KColumn *self,
     uint64_t row;
     uint64_t rows;
     rc_t rc;
+    uint64_t nMissingChecksum = 0;
 
     rc = KColumnIdRange(self, &start, &rows);
     if (rc) {
@@ -157,6 +158,7 @@ rc_t KColumnCheckBlobs(const KColumn *self,
         const KColumnBlob *blob;
         int64_t first;
         uint32_t count;
+        bool missingChecksum = false;
 
         rc = KColumnOpenBlobRead(self, &blob, row + start);
         if (rc) {
@@ -181,20 +183,27 @@ rc_t KColumnCheckBlobs(const KColumn *self,
         rc = KColumnBlobValidate(blob);
         KColumnBlobRelease(blob);
         if (rc) {
-            nfo->info.done.rc = rc;
-            nfo->info.done.mesg = "contains bad data";
-            nfo->type = ccrpt_Done;
-            return report(nfo, ctx);
+            if (GetRCState(rc) == (int)rcNotFound && GetRCObject(rc) == (int)rcChecksum) {
+                missingChecksum = true;
+                nMissingChecksum += 1;
+            }
+            else {
+                nfo->type = ccrpt_Done;
+                nfo->info.done.rc = rc;
+                nfo->info.done.mesg = "contains bad data";
+                return report(nfo, ctx);
+            }
         }
         nfo->type = ccrpt_Blob;
         nfo->info.blob.start = first;
         nfo->info.blob.count = count;
+        nfo->info.blob.missingChecksum = missingChecksum;
         rc = report(nfo, ctx);
 
         row += count;
     }
     nfo->info.done.rc = 0;
-    nfo->info.done.mesg = "checksums ok";
+    nfo->info.done.mesg = nMissingChecksum == 0 ? "checksums ok" : "checksums missing";
     nfo->type = ccrpt_Done;
     rc = report(nfo, ctx);
     return rc;

libs/kdb/columnblob-base.c

@@ -71,7 +71,7 @@ rc_t CC KColumnBlobBaseRelease ( const KColumnBlob *cself )
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
     else                                        \
-        return RC ( rcVDB, rcCursor, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcCursor, rcAccessing, rcSelf, rcNull );
 
 LIB_EXPORT rc_t CC KColumnBlobAddRef ( const KColumnBlob *self )
 {

libs/kdb/database-base.c

@@ -108,7 +108,7 @@ rc_t KDatabaseSever ( const KDatabase *self )
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
     else                                        \
-        return RC ( rcVDB, rcDatabase, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcDatabase, rcAccessing, rcSelf, rcNull );
 #define DISPATCH_BOOL(call)  \
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
@@ -202,7 +202,7 @@ LIB_EXPORT rc_t CC KDatabaseOpenTableRead ( const KDatabase *self,
     const struct KTable **tbl, const char *name, ... )
 {
     if ( self == NULL && self -> vt == NULL )
-        return RC ( rcVDB, rcDatabase, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcDatabase, rcAccessing, rcSelf, rcNull );
 
     rc_t rc;
     va_list args;

libs/kdb/database-base.h

@@ -80,7 +80,7 @@ struct KDatabase_vt
 
 struct KDatabaseBase
 {
-    KDatabase_vt * vt;
+    const KDatabase_vt * vt;
 
     KRefcount refcount;
 };

libs/kdb/database-cmn.h

@@ -37,8 +37,8 @@
 
 #define KDATABASE_IMPL KDatabase
 #include "database-base.h"
-#ifdef __cplusplus
 
+#ifdef __cplusplus
 extern "C" {
 #endif
 

libs/kdb/index-base.c

@@ -72,7 +72,7 @@ rc_t CC KIndexBaseRelease ( const KIndex *self )
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
     else                                        \
-        return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull );
 #define DISPATCH_BOOL(call)  \
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \

libs/kdb/index-base.h

@@ -74,7 +74,7 @@ struct KIndex_vt
 
 struct KIndex
 {
-    KIndex_vt * vt;
+    const KIndex_vt * vt;
 
     KRefcount refcount;
 };

libs/kdb/libkdb.vers.h

@@ -26,4 +26,4 @@
 
 #pragma once
 
-#define LIBKDB_VERS 0x0300000A
+#define LIBKDB_VERS 0x03010000

libs/kdb/manager-base.c

@@ -109,7 +109,7 @@ rc_t KDBManagerSever ( const KDBManager *self )
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \
     else                                        \
-        return RC ( rcVDB, rcMgr, rcAccessing, rcSelf, rcNull );
+        return RC ( rcDB, rcMgr, rcAccessing, rcSelf, rcNull );
 #define DISPATCH_BOOL(call)  \
     if ( self != NULL && self -> vt != NULL )   \
         return self -> vt -> call;              \

libs/kdb/manager-base.h

@@ -76,20 +76,9 @@ struct KDBManager_vt
     rc_t ( CC * vPathOpenRemoteDBRead ) ( struct KDBMGR_IMPL const * self, struct KDatabase const ** p_db, struct VPath const * remote, struct VPath const * cache );
 };
 
-// Public write side-only API
-
-// KDB_EXTERN rc_t CC KDBManagerLock ( KDBManager *self, const char *path, ... );
-// KDB_EXTERN rc_t CC KDBManagerVLock ( KDBManager *self, const char *path, va_list args );
-
-// KDB_EXTERN rc_t CC KDBManagerUnlock ( KDBManager *self, const char *path, ... );
-// KDB_EXTERN rc_t CC KDBManagerVUnlock ( KDBManager *self, const char *path, va_list args );
-
-// KDB_EXTERN rc_t CC KDBManagerDrop ( KDBManager *self, uint32_t obj_type, const char *path, ... );
-// KDB_EXTERN rc_t CC KDBManagerVDrop ( KDBManager *self, uint32_t obj_type, const char *path, va_list args );
-
 struct KDBManagerBase
 {
-    KDBManager_vt * vt;
+    const KDBManager_vt * vt;
 
     KRefcount refcount;
 };
@@ -107,9 +96,6 @@ extern rc_t CC KDBManagerBaseRelease ( const KDBMGR_IMPL *self );
 KDBMGR_IMPL *KDBManagerAttach ( const KDBMGR_IMPL *self );
 rc_t KDBManagerSever ( const KDBMGR_IMPL *self );
 
-// write side only public API
-// ...
-
 #ifdef __cplusplus
 }
 #endif