using string-suffix-array-functionality branch instead of master

arkouda/__init__.py

@@ -1,5 +1,6 @@
 from arkouda.client import *
 from arkouda.dtypes import *
+from arkouda.dtypes import npstr as akstr 
 from arkouda.pdarrayclass import *
 from arkouda.sorting import *
 from arkouda.pdarraysetops import * 

arkouda/pdarrayclass.py

@@ -66,6 +66,46 @@ def unescape(s):
         raise ValueError(("unsupported value from server {} {}".\
                               format(mydtype.name, value)))
 
+
+
+
+@typechecked
+def _parse_single_int_array_value(msg : str) -> object:
+    """
+    Attempt to convert a scalar return value from the arkouda server to a
+    numpy string in Python. The user should not call this function directly. 
+
+    Parameters
+    ----------
+    msg : str
+        scalar value in string form to be converted to a numpy string
+
+    Returns
+    -------
+    object numpy scalar         
+    """
+    fields = msg.split(" ",1)
+    dtname, value = msg.split(maxsplit=1)
+    mydtype = dtype(dtname)
+    try:
+        if mydtype == akint64:
+            nfields = value.split("\"")
+#            return nfields[1]
+#            original we return a string include the last ending 0
+
+            _,sastr=nfields[1].split(maxsplit=1)
+            tmpstr=sastr.split()
+            intary  = [int(numeric_string) for numeric_string in tmpstr]
+            return intary
+#           now we return a suffix array and not include the last ending 0
+        else:
+            raise ValueError(("not correct int data type from server {} {}".\
+                              format(mydtype.name, value)))
+    except:
+        raise ValueError(("unsupported value from server {} {}".\
+                              format(mydtype.name, value)))
+
+
 # class for the pdarray
 class pdarray:
     """

arkouda/pdarraycreation.py

@@ -9,13 +9,12 @@
     int_scalars, numeric_scalars
 from arkouda.dtypes import dtype as akdtype
 from arkouda.pdarrayclass import pdarray, create_pdarray
-from arkouda.strings import Strings
+from arkouda.strings import Strings, SArrays
 
 __all__ = ["array", "zeros", "ones", "zeros_like", "ones_like", 
            "arange", "linspace", "randint", "uniform", "standard_normal",
            "random_strings_uniform", "random_strings_lognormal", 
-           "from_series"
-          ]
+           "from_series", "suffix_array","lcp_array","suffix_array_file"]
 
 @typechecked
 def from_series(series : pd.Series, 
@@ -607,7 +606,7 @@ def randint(low : numeric_scalars, high : numeric_scalars,
 
     repMsg = generic_msg(cmd='randint', args='{} {} {} {} {}'.\
                          format(sizestr, dtype.name, lowstr, highstr, seed))
-    return create_pdarray(repMsg)
+    return create_pdarray(cast(str,repMsg))
 
 @typechecked
 def uniform(size : int_scalars, low : numeric_scalars=float(0.0), 
@@ -821,3 +820,137 @@ def random_strings_lognormal(logmean : numeric_scalars, logstd : numeric_scalars
                  NUMBER_FORMAT_STRINGS['float64'].format(logstd),
                  seed))
     return Strings(*(cast(str,repMsg).split('+')))
+
+
+
+@typechecked
+def suffix_array(strings : Strings) -> SArrays:
+        """
+        Return the suffix arrays of given strings. The size/shape of each suffix
+	arrays is the same as the corresponding strings. 
+	A simple example of suffix array is as follow. Given a string "banana$",
+	all the suffixes are as follows. 
+	s[0]="banana$"
+	s[1]="anana$"
+	s[2]="nana$"
+	s[3]="ana$"
+	s[4]="na$"
+	s[5]="a$"
+	s[6]="$"
+	The suffix array of string "banana$"  is the array of indices of sorted suffixes.
+	s[6]="$"
+	s[5]="a$"
+	s[3]="ana$"
+	s[1]="anana$"
+	s[0]="banana$"
+	s[4]="na$"
+	s[2]="nana$"
+	so sa=[6,5,3,1,0,4,2]
+
+        Returns
+        -------
+        pdarray
+            The suffix arrays of the given strings
+
+        See Also
+        --------
+
+        Notes
+        -----
+
+        Raises
+        ------  
+        RuntimeError
+            Raised if there is a server-side error in executing group request or
+            creating the pdarray encapsulating the return message
+        """
+        cmd=  "segmentedSuffixAry"
+        args = "{} {} {}".format( strings.objtype,
+                                                        strings.offsets.name,
+                                                        strings.bytes.name) 
+        repMsg = generic_msg(cmd=cmd,args=args)
+        return SArrays(*(cast(str,repMsg).split('+')))
+
+
+@typechecked
+def lcp_array(suffixarrays : SArrays, strings : Strings) -> SArrays:
+        """
+        Return the longest common prefix of given suffix arrays. The size/shape of each lcp
+	arrays is the same as the corresponding suffix array. 
+        -------
+        SArrays 
+            The LCP arrays of the given suffix arrays
+
+        See Also
+        --------
+
+        Notes
+        -----
+
+        Raises
+        ------  
+        RuntimeError
+            Raised if there is a server-side error in executing group request or
+            creating the pdarray encapsulating the return message
+        """
+        cmd = "segmentedLCP"
+        args = "{} {} {} {} {}".format( suffixarrays.objtype,
+                                                        suffixarrays.offsets.name,
+                                                        suffixarrays.bytes.name, 
+                                                        strings.offsets.name,
+                                                        strings.bytes.name) 
+        repMsg = generic_msg(cmd=cmd,args=args)
+        return SArrays(*(cast(str,repMsg).split('+')))
+
+@typechecked
+def suffix_array_file(filename: str)  -> tuple:
+#def suffix_array_file(filename: str)  -> tuple[SArrays,Strings]:
+        """
+        This function is major used for testing correctness and performance
+        Return the suffix array of given file name's content as a string. 
+	A simple example of suffix array is as follow. Given string "banana$",
+	all the suffixes are as follows. 
+	s[0]="banana$"
+	s[1]="anana$"
+	s[2]="nana$"
+	s[3]="ana$"
+	s[4]="na$"
+	s[5]="a$"
+	s[6]="$"
+	The suffix array of string "banana$"  is the array of indices of sorted suffixes.
+	s[6]="$"
+	s[5]="a$"
+	s[3]="ana$"
+	s[1]="anana$"
+	s[0]="banana$"
+	s[4]="na$"
+	s[2]="nana$"
+	so sa=[6,5,3,1,0,4,2]
+
+        Returns
+        -------
+        pdarray
+            The suffix arrays of the given strings
+
+        See Also
+        --------
+
+        Notes
+        -----
+
+        Raises
+        ------  
+        RuntimeError
+            Raised if there is a server-side error in executing group request or
+            creating the pdarray encapsulating the return message
+        """
+        cmd = "segmentedSAFile"
+        args = "{}".format( filename )
+        repMsg = generic_msg(cmd=cmd,args=args)
+        tmpmsg=cast(str,repMsg).split('+')
+        sastr=tmpmsg[0:2]
+        strstr=tmpmsg[2:4]
+        suffixarray=SArrays(*(cast(str,sastr))) 
+        originalstr=Strings(*(cast(str,strstr))) 
+        return suffixarray,originalstr
+#        return SArrays(*(cast(str,repMsg).split('+')))

arkouda/pdarraysetops.py

@@ -5,7 +5,7 @@
 from arkouda.pdarrayclass import pdarray, create_pdarray
 from arkouda.pdarraycreation import zeros, zeros_like, array
 from arkouda.sorting import argsort
-from arkouda.strings import Strings
+from arkouda.strings import Strings,SArrays
 from arkouda.logger import getArkoudaLogger
 
 Categorical = ForwardRef('Categorical')
@@ -168,6 +168,83 @@ def in1d(pda1 : Union[pdarray,Strings,'Categorical'], pda2 : Union[pdarray,Strin
     else:
         raise TypeError('Both pda1 and pda2 must be pdarray, Strings, or Categorical')
 
+
+
+def in1d_int(pda1 : Union[pdarray,SArrays,'Categorical'], pda2 : Union[pdarray,SArrays,'Categorical'], #type: ignore
+                         invert : bool=False) -> pdarray: #type: ignore
+    """
+    Test whether each element of a 1-D array is also present in a second array.
+
+    Returns a boolean array the same length as `pda1` that is True
+    where an element of `pda1` is in `pda2` and False otherwise.
+
+    Parameters
+    ----------
+    pda1 : pdarray or SArrays or Categorical
+        Input array.
+    pda2 : pdarray or SArrays or Categorical
+        The values against which to test each value of `pda1`. Must be the 
+        same type as `pda1`.
+    invert : bool, optional
+        If True, the values in the returned array are inverted (that is,
+        False where an element of `pda1` is in `pda2` and True otherwise).
+        Default is False. ``ak.in1d(a, b, invert=True)`` is equivalent
+        to (but is faster than) ``~ak.in1d(a, b)``.
+
+    Returns
+    -------
+    pdarray, bool
+        The values `pda1[in1d]` are in `pda2`.
+
+    Raises
+    ------
+    TypeError
+        Raised if either pda1 or pda2 is not a pdarray, Strings, or 
+        Categorical object or if invert is not a bool
+    RuntimeError
+        Raised if the dtype of either array is not supported
+
+    See Also
+    --------
+    unique, intersect1d, union1d
+
+    Notes
+    -----
+    `in1d` can be considered as an element-wise function version of the
+    python keyword `in`, for 1-D sequences. ``in1d(a, b)`` is logically
+    equivalent to ``ak.array([item in b for item in a])``, but is much
+    faster and scales to arbitrarily large ``a``.
+
+    ak.in1d is not supported for bool or float64 pdarrays
+
+    Examples
+    --------
+    >>> ak.in1d(ak.array([-1, 0, 1]), ak.array([-2, 0, 2]))
+    array([False, True, False])    
+
+    >>> ak.in1d(ak.array(['one','two']),ak.array(['two', 'three','four','five']))
+    array([False, True])
+    """
+    from arkouda.categorical import Categorical as Categorical_
+    if hasattr(pda1, 'in1d'):
+        return cast(Categorical_,pda1).in1d(pda2)
+    elif isinstance(pda1, pdarray) and isinstance(pda2, pdarray):
+        repMsg = generic_msg("in1d {} {} {}".\
+                             format(pda1.name, pda2.name, invert))
+        return create_pdarray(cast(str,repMsg))
+    elif isinstance(pda1, SArrays) and isinstance(pda2, SArrays):
+        repMsg = generic_msg("segmentedIn1dInt {} {} {} {} {} {} {}".\
+                                    format(pda1.objtype,
+                                    pda1.offsets.name,
+                                    pda1.bytes.name,
+                                    pda2.objtype,
+                                    pda2.offsets.name,
+                                    pda2.bytes.name,
+                                    invert))
+        return create_pdarray(cast(str,repMsg))
+    else:
+        raise TypeError('Both pda1 and pda2 must be pdarray, SArrays, or Categorical')
+
 @typechecked
 def concatenate(arrays : Sequence[Union[pdarray,Strings,'Categorical']], #type: ignore
                 ordered : bool=True) -> Union[pdarray,Strings,'Categorical']: #type: ignore