From cc4927529bb35557fdd772b8570d3855c68398d7 Mon Sep 17 00:00:00 2001 From: Martijn Dekker Date: Thu, 28 Jan 2021 02:29:55 +0000 Subject: [PATCH] libast: Update cdt(3): Allow empty strings in (dt)trees This backports most of the Cdt (container data types) mechanism from the ksh 93v- beta, based on ground work done by OpenSUSE: https://build.opensuse.org/package/view_file/shells/ksh/ksh93-dttree-crash.dif plus adaptations to match ksh 93u+m and an updated manual page (src/lib/libast/man/cdt.3) added directly from the 93v- sources. | Thu Dec 20 12:48:02 UTC 2012 - werner@suse.de | | - Add ksh93-dttree-crash.dif - Allow empty strings in (dt)trees | (bnc#795324) | | Fri Oct 25 14:07:57 UTC 2013 - werner@suse.de | | - Rework patch ksh93-dttree-crash.dif As usual, precious little information is available because the OpenSUSE bug report is currently closed to the public: https://bugzilla.opensuse.org/show_bug.cgi?id=795324 However, a cursory inspection suggests that this code contains improvements to do with concurrent processing and related robustness. The new cdt.3 manual page adds a lot about that. This has been in production use on OpenSUSE for a long time, so hopefully this will make ksh a little more stable again. Only one way to find out: let's commit and test this... BTW, to get a nice manual, use groff and ghostscript's ps2pdf: $ groff -tman src/lib/libast/man/cdt.3 | ps2pdf - cdt.3.pdf --- src/cmd/ksh93/sh/arith.c | 3 + src/cmd/ksh93/sh/array.c | 30 +- src/cmd/ksh93/sh/init.c | 5 + src/cmd/ksh93/sh/macro.c | 3 + src/cmd/ksh93/sh/main.c | 1 + src/cmd/ksh93/sh/name.c | 8 + src/cmd/ksh93/sh/nvdisc.c | 1 + src/cmd/ksh93/sh/subshell.c | 2 + src/cmd/ksh93/sh/xec.c | 15 +- src/lib/libast/Mamfile | 18 +- src/lib/libast/cdt/cdtlib.h | 4 +- src/lib/libast/cdt/dtcomp.c | 6 - src/lib/libast/cdt/dthash.c | 44 ++- src/lib/libast/cdt/dtlist.c | 13 +- src/lib/libast/cdt/dtopen.c | 22 -- src/lib/libast/cdt/dtstat.c | 54 ++++ src/lib/libast/cdt/dtstrhash.c | 38 ++- src/lib/libast/cdt/dttree.c | 33 +- src/lib/libast/cdt/dtuser.c | 59 ++++ src/lib/libast/include/cdt.h | 10 +- src/lib/libast/man/cdt.3 | 553 ++++++++++++++++++++++++++------- 21 files changed, 710 insertions(+), 212 deletions(-) create mode 100644 src/lib/libast/cdt/dtstat.c create mode 100644 src/lib/libast/cdt/dtuser.c diff --git a/src/cmd/ksh93/sh/arith.c b/src/cmd/ksh93/sh/arith.c index 11c2ff946a6b..93de13cdd969 100644 --- a/src/cmd/ksh93/sh/arith.c +++ b/src/cmd/ksh93/sh/arith.c @@ -184,7 +184,10 @@ static Namval_t *scope(register Namval_t *np,register struct lval *lvalue,int as { ap = nv_arrayptr(np); if(ap && !ap->table) + { ap->table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->table,shp,1); + } if(ap && ap->table && (nq=nv_search(nv_getsub(np),ap->table,NV_ADD))) nq->nvenv = (char*)np; if(nq && nv_isnull(nq)) diff --git a/src/cmd/ksh93/sh/array.c b/src/cmd/ksh93/sh/array.c index f409b2631679..bfc1ba059986 100644 --- a/src/cmd/ksh93/sh/array.c +++ b/src/cmd/ksh93/sh/array.c @@ -95,6 +95,7 @@ static Namarr_t *array_scope(Namval_t *np, Namarr_t *ap, int flags) if(is_associative(aq)) { aq->scope = (void*)dtopen(&_Nvdisc,Dtoset); + dtuserdata(aq->scope,&sh,1); dtview((Dt_t*)aq->scope,aq->table); aq->table = (Dt_t*)aq->scope; return(aq); @@ -373,7 +374,10 @@ static Namval_t *array_find(Namval_t *np,Namarr_t *arp, int flag) { char *cp; if(!ap->header.table) + { ap->header.table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->header.table,&sh,1); + } sfprintf(sh.strbuf,"%d",ap->cur); cp = sfstruse(sh.strbuf); mp = nv_search(cp, ap->header.table, NV_ADD); @@ -410,7 +414,10 @@ int nv_arraysettype(Namval_t *np, Namval_t *tp, const char *sub, int flags) av[1] = 0; sh.last_table = 0; if(!ap->table) + { ap->table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->table,&sh,1); + } if(nq = nv_search(sub, ap->table, NV_ADD)) { if(!nq->nvfun && nq->nvalue.cp && *nq->nvalue.cp==0) @@ -452,7 +459,6 @@ static Namfun_t *array_clone(Namval_t *np, Namval_t *mp, int flags, Namfun_t *fp int nelem, skipped=0; Dt_t *otable=ap->table; struct index_array *aq = (struct index_array*)ap, *ar; - Shell_t *shp = sh_getinterp(); if(flags&NV_MOVE) { if((flags&NV_COMVAR) && nv_putsub(np,NIL(char*),ARRAY_SCAN)) @@ -485,6 +491,7 @@ static Namfun_t *array_clone(Namval_t *np, Namval_t *mp, int flags, Namfun_t *fp if(ap->table) { ap->table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->table,&sh,1); if(ap->scope && !(flags&NV_COMVAR)) { ap->scope = ap->table; @@ -855,6 +862,7 @@ static struct index_array *array_grow(Namval_t *np, register struct index_array if(nv_hasdisc(np,&array_disc) || (nv_type(np) && nv_isvtree(np))) { ap->header.table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->header.table,&sh,1); mp = nv_search("0", ap->header.table,NV_ADD); if(mp && nv_isnull(mp)) { @@ -1180,7 +1188,6 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode) { if(sp) { - Shell_t *shp = sh_getinterp(); if(ap && ap->xp && !strmatch(sp,"+([0-9])")) { Namval_t *mp = nv_namptr(ap->xp,0); @@ -1188,7 +1195,7 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode) size = nv_getnum(mp); } else - size = (int)sh_arith(shp,(char*)sp); + size = (int)sh_arith(&sh,(char*)sp); } if(size <0 && ap) size += array_maxindex(np); @@ -1242,7 +1249,10 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode) char *cp; Namval_t *mp; if(!ap->header.table) + { ap->header.table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->header.table,&sh,1); + } sfprintf(sh.strbuf,"%d",ap->cur); cp = sfstruse(sh.strbuf); mp = nv_search(cp, ap->header.table, NV_ADD); @@ -1379,7 +1389,6 @@ static void array_fixed_setdata(Namval_t *np,Namarr_t* ap,struct fixed_array* fp static int array_fixed_init(Namval_t *np, char *sub, char *cp) { - Shell_t *shp=sh_getinterp(); Namarr_t *ap; struct fixed_array *fp; int n=1,sz; @@ -1403,12 +1412,12 @@ static int array_fixed_init(Namval_t *np, char *sub, char *cp) fp->max = (int*)(fp+1); fp->incr = fp->max+n; fp->cur = fp->incr+n; - fp->max[0] = (int)sh_arith(shp,(char*)sub); + fp->max[0] = (int)sh_arith(&sh,(char*)sub); for(n=1,ep=cp;*ep=='['; ep=cp) { cp = nv_endsubscript(np,ep,0); cp[-1]=0; - fp->max[n++] = sz = (int)sh_arith(shp,(char*)ep+1); + fp->max[n++] = sz = (int)sh_arith(&sh,(char*)ep+1); if(sz<0) { free((void*)ap); @@ -1429,7 +1438,6 @@ static int array_fixed_init(Namval_t *np, char *sub, char *cp) static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode) { - Shell_t *shp=sh_getinterp(); Namarr_t *ap = nv_arrayptr(np); struct fixed_array *fp = (struct fixed_array*)ap->fixed; char *ep; @@ -1447,7 +1455,7 @@ static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode) } else fp->curi = 0; - size = (int)sh_arith(shp,(char*)sub); + size = (int)sh_arith(&sh,(char*)sub); fp->cur[n] = size; if(size >= fp->max[n] || (size < 0)) errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np)); @@ -1459,7 +1467,7 @@ static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode) errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np)); cp = nv_endsubscript(np,ep,0); cp[-1]=0; - size = (int)sh_arith(shp,(char*)ep+1); + size = (int)sh_arith(&sh,(char*)ep+1); if(size >= fp->max[n] || (size < 0)) errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np)); fp->cur[n] = size; @@ -1658,6 +1666,7 @@ void *nv_associative(register Namval_t *np,const char *sp,int mode) if(ap = (struct assoc_array*)calloc(1,sizeof(struct assoc_array))) { ap->header.table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->header.table,&sh,1); ap->cur = 0; ap->pos = 0; ap->header.hdr.disc = &array_disc; @@ -1730,8 +1739,7 @@ void *nv_associative(register Namval_t *np,const char *sp,int mode) case NV_ANAME: if(ap->cur) { - Shell_t *shp = sh_getinterp(); - if(!shp->instance && nv_isnull(ap->cur)) + if(!sh.instance && nv_isnull(ap->cur)) return(NIL(void*)); return((void*)ap->cur->nvname); } diff --git a/src/cmd/ksh93/sh/init.c b/src/cmd/ksh93/sh/init.c index 6c3feec2d2ca..279c6ab613cf 100644 --- a/src/cmd/ksh93/sh/init.c +++ b/src/cmd/ksh93/sh/init.c @@ -1812,9 +1812,13 @@ static Init_t *nv_init(Shell_t *shp) (OPTINDNOD)->nvalue.lp = (&shp->st.optindex); /* set up the seconds clock */ shp->alias_tree = dtopen(&_Nvdisc,Dtoset); + dtuserdata(shp->alias_tree,shp,1); shp->track_tree = dtopen(&_Nvdisc,Dtset); + dtuserdata(shp->track_tree,shp,1); shp->bltin_tree = sh_inittree(shp,(const struct shtable2*)shtab_builtins); + dtuserdata(shp->bltin_tree,shp,1); shp->fun_tree = dtopen(&_Nvdisc,Dtoset); + dtuserdata(shp->fun_tree,shp,1); dtview(shp->fun_tree,shp->bltin_tree); nv_mount(DOTSHNOD, "type", shp->typedict=dtopen(&_Nvdisc,Dtoset)); nv_adddisc(DOTSHNOD, shdiscnames, (Namval_t**)0); @@ -1857,6 +1861,7 @@ Dt_t *sh_inittree(Shell_t *shp,const struct shtable2 *name_vals) nbltins = n; } base_treep = treep = dtopen(&_Nvdisc,Dtoset); + dtuserdata(treep,shp,1); treep->user = (void*)shp; for(tp=name_vals;*tp->sh_name;tp++,np++) { diff --git a/src/cmd/ksh93/sh/macro.c b/src/cmd/ksh93/sh/macro.c index b273a436bde1..9f051befa5e9 100644 --- a/src/cmd/ksh93/sh/macro.c +++ b/src/cmd/ksh93/sh/macro.c @@ -2738,7 +2738,10 @@ static char *sh_tilde(Shell_t *shp,register const char *string) skip: #endif /* _WINIX */ if(!logins_tree) + { logins_tree = dtopen(&_Nvdisc,Dtbag); + dtuserdata(logins_tree,shp,1); + } if(np=nv_search(string,logins_tree,NV_ADD)) { save = shp->subshell; diff --git a/src/cmd/ksh93/sh/main.c b/src/cmd/ksh93/sh/main.c index 021da552d998..4217ea09ce97 100644 --- a/src/cmd/ksh93/sh/main.c +++ b/src/cmd/ksh93/sh/main.c @@ -181,6 +181,7 @@ int sh_main(int ac, char *av[], Shinit_f userinit) /* preset aliases for interactive non-POSIX ksh */ dtclose(shp->alias_tree); shp->alias_tree = sh_inittree(shp,shtab_aliases); + dtuserdata(shp->alias_tree,shp,1); } } #if SHOPT_REMOTE diff --git a/src/cmd/ksh93/sh/name.c b/src/cmd/ksh93/sh/name.c index 81b1fda19f5a..5866ea3393db 100644 --- a/src/cmd/ksh93/sh/name.c +++ b/src/cmd/ksh93/sh/name.c @@ -789,6 +789,7 @@ Namval_t *nv_create(const char *name, Dt_t *root, int flags, Namfun_t *dp) { Dt_t *dp = dtview(shp->var_tree,(Dt_t*)0); rp->sdict = dtopen(&_Nvdisc,Dtoset); + dtuserdata(rp->sdict,shp,1); dtview(rp->sdict,dp); dtview(shp->var_tree,rp->sdict); } @@ -1131,7 +1132,10 @@ Namval_t *nv_create(const char *name, Dt_t *root, int flags, Namfun_t *dp) ap = nv_arrayptr(np); } if(n && ap && !ap->table) + { ap->table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->table,shp,1); + } if(ap && ap->table && (nq=nv_search(sub,ap->table,n))) nq->nvenv = (char*)np; if(nq && nv_isnull(nq)) @@ -2285,6 +2289,7 @@ void sh_scope(Shell_t *shp, struct argnod *envlist, int fun) newroot = nv_dict(shp->namespace); #endif /* SHOPT_NAMESPACE */ newscope = dtopen(&_Nvdisc,Dtoset); + dtuserdata(newscope,shp,1); if(envlist) { dtview(newscope,(Dt_t*)shp->var_tree); @@ -3277,7 +3282,10 @@ int nv_rename(register Namval_t *np, int flags) if(ap=nv_arrayptr(np)) { if(!ap->table) + { ap->table = dtopen(&_Nvdisc,Dtoset); + dtuserdata(ap->table,shp,1); + } if(ap->table) mp = nv_search(nv_getsub(np),ap->table,NV_ADD); nv_arraychild(np,mp,0); diff --git a/src/cmd/ksh93/sh/nvdisc.c b/src/cmd/ksh93/sh/nvdisc.c index 3f8015820081..70e65e2d0a7e 100644 --- a/src/cmd/ksh93/sh/nvdisc.c +++ b/src/cmd/ksh93/sh/nvdisc.c @@ -1299,6 +1299,7 @@ static Namfun_t *clone_table(Namval_t* np, Namval_t *mp, int flags, Namfun_t *fp Dt_t *oroot=tp->dict,*nroot=dtopen(&_Nvdisc,Dtoset); if(!nroot) return(0); + dtuserdata(nroot,dtuserdata(oroot,0,0),1); memcpy((void*)ntp,(void*)fp,sizeof(struct table)); ntp->dict = nroot; ntp->parent = nv_lastdict(); diff --git a/src/cmd/ksh93/sh/subshell.c b/src/cmd/ksh93/sh/subshell.c index 84bd7c0ccb6c..04053f496993 100644 --- a/src/cmd/ksh93/sh/subshell.c +++ b/src/cmd/ksh93/sh/subshell.c @@ -404,6 +404,7 @@ Dt_t *sh_subtracktree(int create) if(create && sh.subshell && !sh.subshare && sp && !sp->strack) { sp->strack = dtopen(&_Nvdisc,Dtset); + dtuserdata(sp->strack,&sh,1); dtview(sp->strack,sh.track_tree); sh.track_tree = sp->strack; } @@ -420,6 +421,7 @@ Dt_t *sh_subfuntree(int create) if(create && sh.subshell && !sh.subshare && sp && !sp->sfun) { sp->sfun = dtopen(&_Nvdisc,Dtoset); + dtuserdata(sp->sfun,&sh,1); dtview(sp->sfun,sh.fun_tree); sh.fun_tree = sp->sfun; } diff --git a/src/cmd/ksh93/sh/xec.c b/src/cmd/ksh93/sh/xec.c index 9ff1b20a6d1b..bb5f5e162a22 100644 --- a/src/cmd/ksh93/sh/xec.c +++ b/src/cmd/ksh93/sh/xec.c @@ -2429,6 +2429,7 @@ int sh_exec(register const Shnode_t *t, int flags) else { root = dtopen(&_Nvdisc,Dtoset); + dtuserdata(root,shp,1); nv_mount(np, (char*)0, root); np->nvalue.cp = Empty; dtview(root,shp->var_base); @@ -2484,11 +2485,6 @@ int sh_exec(register const Shnode_t *t, int flags) slp = (struct slnod*)np->nvenv; sh_funstaks(slp->slchild,-1); stakdelete(slp->slptr); - if(shp->funload) - { - free((void*)np->nvalue.rp); - np->nvalue.rp = 0; - } if(rp->sdict) { Namval_t *mp, *nq; @@ -2502,6 +2498,12 @@ int sh_exec(register const Shnode_t *t, int flags) dtclose(rp->sdict); rp->sdict = 0; } + if(shp->funload) + { + if(!shp->fpathdict) + free((void*)np->nvalue.rp); + np->nvalue.rp = 0; + } } if(!np->nvalue.rp) { @@ -2540,7 +2542,10 @@ int sh_exec(register const Shnode_t *t, int flags) if(!shp->fpathdict) shp->fpathdict = dtopen(&_Rpdisc,Dtobag); if(shp->fpathdict) + { + dtuserdata(shp->fpathdict,shp,1); dtinsert(shp->fpathdict,rp); + } } } else diff --git a/src/lib/libast/Mamfile b/src/lib/libast/Mamfile index 31c995618484..07977f87eb5c 100644 --- a/src/lib/libast/Mamfile +++ b/src/lib/libast/Mamfile @@ -3964,6 +3964,14 @@ make install prev cdt/dtopen.c exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtopen.c done dtopen.o generated + make dtstat.o + make cdt/dtstat.c + prev cdt/dthdr.h implicit + done cdt/dtstat.c + meta dtstat.o %.c>%.o cdt/dtstat.c dtstat + prev cdt/dtstat.c + exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtstat.c + done dtstat.o generated make dtstrhash.o make cdt/dtstrhash.c prev cdt/dthdr.h implicit @@ -3980,6 +3988,14 @@ make install prev cdt/dttree.c exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dttree.c done dttree.o generated + make dtuser.o + make cdt/dtuser.c + prev cdt/dthdr.h implicit + done cdt/dtuser.c + meta dtuser.o %.c>%.o cdt/dtuser.c dtuser + prev cdt/dtuser.c + exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtuser.c + done dtuser.o generated make dtview.o make cdt/dtview.c prev cdt/dthdr.h implicit @@ -6095,7 +6111,7 @@ make install exec - ${AR} rc libast.a streval.o strexpr.o strmatch.o strcopy.o modei.o modex.o strmode.o strlcat.o strlcpy.o strlook.o strncopy.o strsearch.o strpsearch.o stresc.o stropt.o strtape.o strpcmp.o strnpcmp.o strvcmp.o strnvcmp.o tok.o tokline.o tokscan.o pathaccess.o pathcat.o pathcanon.o pathcheck.o pathpath.o pathexists.o pathfind.o pathkey.o pathprobe.o pathrepl.o pathnative.o pathposix.o pathtemp.o pathtmp.o pathstat.o pathgetlink.o pathsetlink.o pathbin.o pathshell.o pathcd.o pathprog.o ftwalk.o ftwflags.o fts.o astintercept.o conformance.o getenv.o setenviron.o optget.o optjoin.o optesc.o optctx.o strsort.o struniq.o magic.o mime.o mimetype.o signal.o sigflag.o systrace.o error.o errorf.o errormsg.o errorx.o localeconv.o setlocale.o translate.o catopen.o iconv.o lc.o lctab.o mc.o base64.o recfmt.o recstr.o reclen.o fmtrec.o fmtbase.o fmtbuf.o fmtclock.o fmtdev.o fmtelapsed.o fmterror.o fmtesc.o fmtfmt.o fmtfs.o fmtident.o fmtint.o fmtip4.o fmtip6.o fmtls.o fmtmatch.o fmtmode.o fmtnum.o fmtperm.o fmtre.o fmttime.o exec - ${AR} rc libast.a fmtuid.o fmtgid.o fmtsignal.o fmtscale.o fmttmx.o fmttv.o fmtversion.o strelapsed.o strperm.o struid.o strgid.o strtoip4.o strtoip6.o stack.o stk.o swapget.o swapmem.o swapop.o swapput.o sigdata.o sigcrit.o sigunblock.o procopen.o procclose.o procrun.o procfree.o tmdate.o tmequiv.o tmfix.o tmfmt.o tmform.o tmgoff.o tminit.o tmleap.o tmlex.o tmlocale.o tmmake.o tmpoff.o tmscan.o tmsleep.o tmtime.o tmtype.o tmweek.o tmword.o tmzone.o tmxdate.o tmxduration.o tmxfmt.o tmxgettime.o tmxleap.o tmxmake.o tmxscan.o tmxsettime.o tmxsleep.o tmxtime.o tmxtouch.o tvcmp.o tvgettime.o tvsettime.o tvsleep.o tvtouch.o cmdarg.o vecargs.o vecfile.o vecfree.o vecload.o vecstring.o univdata.o touch.o mnt.o debug.o memccpy.o memchr.o memcmp.o memcpy.o memdup.o memmove.o memset.o mkdir.o mkfifo.o mknod.o rmdir.o remove.o rename.o link.o unlink.o strdup.o strchr.o strrchr.o strstr.o strtod.o strtold.o strtol.o strtoll.o strtoul.o strtoull.o strton.o strtonll.o strntod.o strntold.o strnton.o exec - ${AR} rc libast.a strntonll.o strntol.o strntoll.o strntoul.o strntoull.o strcasecmp.o strncasecmp.o strerror.o mktemp.o tmpnam.o fsync.o execlp.o execve.o execvp.o execvpe.o spawnveg.o vfork.o killpg.o hsearch.o tsearch.o getlogin.o putenv.o setenv.o unsetenv.o lstat.o statvfs.o eaccess.o gross.o omitted.o readlink.o symlink.o getpgrp.o setpgid.o setsid.o waitpid.o creat64.o fcntl.o open.o atexit.o getdents.o getwd.o dup2.o errno.o getpreroot.o ispreroot.o realopen.o setpreroot.o getgroups.o mount.o system.o iblocks.o modedata.o tmdata.o memfatal.o sfkeyprintf.o sfdcdio.o sfdcdos.o sfdcfilter.o sfdcseekable.o sfdcslow.o sfdcsubstr.o sfdctee.o sfdcunion.o sfdcmore.o sfdcprefix.o wc.o wc2utf8.o basename.o closelog.o dirname.o fmtmsglib.o fnmatch.o ftw.o getdate.o getsubopt.o glob.o nftw.o openlog.o re_comp.o resolvepath.o realpath.o regcmp.o regexp.o setlogmask.o strftime.o strptime.o swab.o syslog.o tempnam.o wordexp.o mktime.o regalloc.o regclass.o regcoll.o regcomp.o regcache.o regdecomp.o regerror.o regexec.o regfatal.o reginit.o - exec - ${AR} rc libast.a regnexec.o regsubcomp.o regsubexec.o regsub.o regrecord.o regrexec.o regstat.o dtclose.o dtdisc.o dthash.o dtlist.o dtmethod.o dtopen.o dtstrhash.o dttree.o dtview.o dtwalk.o dtnew.o dtcomp.o sfclose.o sfclrlock.o sfdisc.o sfdlen.o sfexcept.o sfgetl.o sfgetu.o sfcvt.o sfecvt.o sffcvt.o sfextern.o sffilbuf.o sfflsbuf.o sfprints.o sfgetd.o sfgetr.o sfllen.o sfmode.o sfmove.o sfnew.o sfpkrd.o sfnotify.o sfnputc.o sfopen.o sfpeek.o sfpoll.o sfpool.o sfpopen.o sfprintf.o sfputd.o sfputl.o sfputr.o sfputu.o sfrd.o sfread.o sfreserve.o sfscanf.o sfseek.o sfset.o sfsetbuf.o sfsetfd.o sfsize.o sfsk.o sfstack.o sfstrtod.o sfsync.o sfswap.o sftable.o sftell.o sftmp.o sfungetc.o sfvprintf.o sfvscanf.o sfwr.o sfwrite.o sfpurge.o sfraise.o sfwalk.o sfgetm.o sfmutex.o sfputm.o sfresize.o _sfclrerr.o _sfeof.o _sferror.o _sffileno.o _sfopen.o _sfstacked.o _sfvalue.o _sfgetc.o _sfgetl.o _sfgetl2.o _sfgetu.o _sfgetu2.o _sfdlen.o _sfllen.o _sfslen.o _sfulen.o _sfputc.o _sfputd.o _sfputl.o _sfputm.o + exec - ${AR} rc libast.a regnexec.o regsubcomp.o regsubexec.o regsub.o regrecord.o regrexec.o regstat.o dtclose.o dtdisc.o dthash.o dtlist.o dtmethod.o dtopen.o dtstat.o dtstrhash.o dttree.o dtuser.o dtview.o dtwalk.o dtnew.o dtcomp.o sfclose.o sfclrlock.o sfdisc.o sfdlen.o sfexcept.o sfgetl.o sfgetu.o sfcvt.o sfecvt.o sffcvt.o sfextern.o sffilbuf.o sfflsbuf.o sfprints.o sfgetd.o sfgetr.o sfllen.o sfmode.o sfmove.o sfnew.o sfpkrd.o sfnotify.o sfnputc.o sfopen.o sfpeek.o sfpoll.o sfpool.o sfpopen.o sfprintf.o sfputd.o sfputl.o sfputr.o sfputu.o sfrd.o sfread.o sfreserve.o sfscanf.o sfseek.o sfset.o sfsetbuf.o sfsetfd.o sfsize.o sfsk.o sfstack.o sfstrtod.o sfsync.o sfswap.o sftable.o sftell.o sftmp.o sfungetc.o sfvprintf.o sfvscanf.o sfwr.o sfwrite.o sfpurge.o sfraise.o sfwalk.o sfgetm.o sfmutex.o sfputm.o sfresize.o _sfclrerr.o _sfeof.o _sferror.o _sffileno.o _sfopen.o _sfstacked.o _sfvalue.o _sfgetc.o _sfgetl.o _sfgetl2.o _sfgetu.o _sfgetu2.o _sfdlen.o _sfllen.o _sfslen.o _sfulen.o _sfputc.o _sfputd.o _sfputl.o _sfputm.o exec - ${AR} rc libast.a _sfputu.o clearerr.o fclose.o fdopen.o feof.o ferror.o fflush.o fgetc.o fgetpos.o fgets.o fileno.o fopen.o fprintf.o fpurge.o fputc.o fputs.o fread.o freopen.o fscanf.o fseek.o fseeko.o fsetpos.o ftell.o ftello.o fwrite.o flockfile.o ftrylockfile.o funlockfile.o getc.o getchar.o getw.o pclose.o popen.o printf.o putc.o putchar.o puts.o putw.o rewind.o scanf.o setbuf.o setbuffer.o setlinebuf.o setvbuf.o snprintf.o sprintf.o sscanf.o asprintf.o vasprintf.o tmpfile.o ungetc.o vfprintf.o vfscanf.o vprintf.o vscanf.o vsnprintf.o vsprintf.o vsscanf.o _doprnt.o _doscan.o _filbuf.o _flsbuf.o _stdfun.o _stdopen.o _stdprintf.o _stdscanf.o _stdsprnt.o _stdvbuf.o _stdvsnprnt.o _stdvsprnt.o _stdvsscn.o fgetwc.o fwprintf.o putwchar.o vfwscanf.o wprintf.o fgetws.o fwscanf.o swprintf.o vswprintf.o wscanf.o fputwc.o getwc.o swscanf.o vswscanf.o fputws.o getwchar.o ungetwc.o vwprintf.o fwide.o putwc.o vfwprintf.o vwscanf.o stdio_c99.o fcloseall.o fmemopen.o getdelim.o getline.o frexp.o frexpl.o astcopy.o exec - ${AR} rc libast.a astconf.o astdynamic.o astlicense.o astquery.o astwinsize.o conftab.o aststatic.o getopt.o getoptl.o aso.o asolock.o asometh.o asorelax.o aso-sem.o aso-fcntl.o vmbest.o vmclear.o vmclose.o vmdcheap.o vmdebug.o vmdisc.o vmexit.o vmlast.o vmopen.o vmpool.o vmprivate.o vmprofile.o vmregion.o vmsegment.o vmset.o vmstat.o vmstrdup.o vmtrace.o vmwalk.o vmmopen.o malloc.o vmgetmem.o a64l.o acosh.o asinh.o atanh.o cbrt.o crypt.o erf.o err.o exp.o exp__E.o expm1.o gamma.o getpass.o lgamma.o log.o log1p.o log__L.o rand48.o random.o rcmd.o rint.o support.o sfstrtmp.o spawn.o exec - (ranlib libast.a) >/dev/null 2>&1 || true diff --git a/src/lib/libast/cdt/cdtlib.h b/src/lib/libast/cdt/cdtlib.h index 1972dd24d3cf..2539588b9fa1 100644 --- a/src/lib/libast/cdt/cdtlib.h +++ b/src/lib/libast/cdt/cdtlib.h @@ -58,9 +58,9 @@ /* This struct holds private method data created on DT_OPEN */ struct _dtdata_s { unsigned int lock; /* general dictionary lock */ - Dtuser_t user; /* application's data */ unsigned int type; /* method type, control flags */ ssize_t size; /* number of objects */ + Dtuser_t user; /* application's data */ Dt_t dict; /* when DT_INDATA is requested */ }; @@ -123,7 +123,7 @@ typedef struct _dtlib_s #endif /* _BLD_cdt */ /* these macros lock/unlock dictionaries. DTRETURN substitutes for "return" */ -#define DTSETLOCK(dt) (((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_SPINLOCK) : 0 ) +#define DTSETLOCK(dt) (((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_LOCK) : 0 ) #define DTCLRLOCK(dt) (((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_UNLOCK) : 0 ) #define DTRETURN(ob,rv) do { (ob) = (rv); goto dt_return; } while(0) #define DTERROR(dt, mesg) (!((dt)->disc && (dt)->disc->eventf) ? 0 : \ diff --git a/src/lib/libast/cdt/dtcomp.c b/src/lib/libast/cdt/dtcomp.c index 5308c70ba56c..fe9c4ed6e71e 100644 --- a/src/lib/libast/cdt/dtcomp.c +++ b/src/lib/libast/cdt/dtcomp.c @@ -52,9 +52,3 @@ extern ssize_t dtsize(Dt_t* d) { return (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT); } - -#undef dtstat -extern ssize_t dtstat(Dt_t* d) -{ - return (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT); -} diff --git a/src/lib/libast/cdt/dthash.c b/src/lib/libast/cdt/dthash.c index 61bc84cb490d..8e418a9593cc 100644 --- a/src/lib/libast/cdt/dthash.c +++ b/src/lib/libast/cdt/dthash.c @@ -52,11 +52,13 @@ static int htable(Dt_t* dt) if((n = hash->tblz) > 0 && (hash->type&H_FIXED) ) return 0; /* fixed size table */ - if(n == 0 && disc && disc->eventf) /* let user have input */ + if(disc && disc->eventf) /* let user have input */ { if((*disc->eventf)(dt, DT_HASHSIZE, &n, disc) > 0 ) { if(n < 0) /* fix table size */ { hash->type |= H_FIXED; - n = -n; + n = -n; /* desired table size */ + if(hash->tblz >= n ) /* table size is fixed now */ + return 0; } } } @@ -234,12 +236,13 @@ static Void_t* hstat(Dt_t* dt, Dtstat_t* st) for(endt = (t = hash->htbl) + hash->tblz; t < endt; ++t) { for(n = 0, l = *t; l; l = l->_rght) + { if(n < DT_MAXSIZE) + st->lsize[n] += 1; n += 1; + } st->mlev = n > st->mlev ? n : st->mlev; if(n < DT_MAXSIZE) /* if chain length is small */ - { st->msize = n > st->msize ? n : st->msize; - st->lsize[n] += n; - } + st->msize = n > st->msize ? n : st->msize; } } @@ -310,7 +313,7 @@ int type; hsh = _DTHSH(dt,key,disc); tbl = hash->htbl + (hsh & (hash->tblz-1)); - pp = ll = NIL(Dtlink_t*); + pp = ll = NIL(Dtlink_t*); /* pp is the before, ll is the here */ for(p = NIL(Dtlink_t*), l = *tbl; l; p = l, l = l->_rght) { if(hsh == l->_hash) { o = _DTOBJ(disc,l); k = _DTKEY(disc,o); @@ -342,20 +345,41 @@ int type; _dtfree(dt, ll, type); DTRETURN(obj, _DTOBJ(disc,ll)); } + else if(type & DT_INSTALL ) + { if(dt->meth->type&DT_BAG) + goto do_insert; + else if(!(lnk = _dtmake(dt, obj, type)) ) + DTRETURN(obj, NIL(Void_t*) ); + else /* replace old object with new one */ + { if(pp) /* remove old object */ + pp->_rght = ll->_rght; + else *tbl = ll->_rght; + o = _DTOBJ(disc,ll); + _dtfree(dt, ll, DT_DELETE); + DTANNOUNCE(dt, o, DT_DELETE); + + goto do_insert; + } + } else { /**/DEBUG_ASSERT(type&(DT_INSERT|DT_ATTACH|DT_APPEND|DT_RELINK)); - if(!(dt->meth->type&DT_BAG) ) + if((dt->meth->type&DT_BAG) ) + goto do_insert; + else { if(type&(DT_INSERT|DT_APPEND|DT_ATTACH) ) - type |= DT_SEARCH; /* for announcement */ + type |= DT_MATCH; /* for announcement */ else if(lnk && (type&DT_RELINK) ) + { /* remove a duplicate */ + o = _DTOBJ(disc, lnk); _dtfree(dt, lnk, DT_DELETE); + DTANNOUNCE(dt, o, DT_DELETE); + } DTRETURN(obj, _DTOBJ(disc,ll)); } - else goto do_insert; } } else /* no matching object */ - { if(!(type&(DT_INSERT|DT_APPEND|DT_ATTACH|DT_RELINK)) ) + { if(!(type&(DT_INSERT|DT_INSTALL|DT_APPEND|DT_ATTACH|DT_RELINK)) ) DTRETURN(obj, NIL(Void_t*)); do_insert: /* inserting a new object */ diff --git a/src/lib/libast/cdt/dtlist.c b/src/lib/libast/cdt/dtlist.c index 3496fb5aaad5..44ce73df92d4 100644 --- a/src/lib/libast/cdt/dtlist.c +++ b/src/lib/libast/cdt/dtlist.c @@ -142,9 +142,9 @@ int type; } #if __STD_C -static Void_t* liststat(Dt_t* dt, Dtstat_t* st) +static Void_t* listat(Dt_t* dt, Dtstat_t* st) #else -static Void_t* liststat(dt, st) +static Void_t* listat(dt, st) Dt_t* dt; Dtstat_t* st; #endif @@ -186,7 +186,7 @@ int type; else if(type&DT_CLEAR) DTRETURN(obj, lclear(dt)); else if(type&DT_STAT ) - DTRETURN(obj, liststat(dt, (Dtstat_t*)obj)); + DTRETURN(obj, listat(dt, (Dtstat_t*)obj)); h = list->here; /* save finger to last search object */ list->here = NIL(Dtlink_t*); @@ -202,8 +202,9 @@ int type; { r = (Dtlink_t*)obj; goto do_insert; } - else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH)) - { if(!(r = _dtmake(dt, obj, type)) ) + else if(type&(DT_INSERT|DT_INSTALL|DT_APPEND|DT_ATTACH)) + { dt_insert: + if(!(r = _dtmake(dt, obj, type)) ) DTRETURN(obj, NIL(Void_t*)); dt->data->size += 1; @@ -290,7 +291,7 @@ int type; } r = h ? h : r; } - if(!r) + if(!r) /* not found */ DTRETURN(obj, NIL(Void_t*)); if(type&(DT_DELETE|DT_DETACH|DT_REMOVE)) diff --git a/src/lib/libast/cdt/dtopen.c b/src/lib/libast/cdt/dtopen.c index 64119695a0d3..51097621974e 100644 --- a/src/lib/libast/cdt/dtopen.c +++ b/src/lib/libast/cdt/dtopen.c @@ -153,25 +153,3 @@ void _dtfree(Dt_t* dt, Dtlink_t* l, int type) if(disc->link < 0) /* free holder */ (void)(*dt->memoryf)(dt, (Void_t*)l, 0, disc); } - -int dtuserlock(Dt_t* dt, unsigned int key, int type) -{ - if(type > 0) - return asolock(&dt->data->user.lock, key, ASO_LOCK); - else if(type < 0) - return asolock(&dt->data->user.lock, key, ASO_UNLOCK); - else return asolock(&dt->data->user.lock, key, ASO_TRYLOCK); -} - -Void_t* dtuserdata(Dt_t* dt, Void_t* data, unsigned int key) -{ - if(key == 0) - return dt->data->user.data; - else if(dtuserlock(dt, key, 1) < 0 ) - return NIL(Void_t*); - else - { dt->data->user.data = data; - dtuserlock(dt, key, -1); - return data; - } -} diff --git a/src/lib/libast/cdt/dtstat.c b/src/lib/libast/cdt/dtstat.c new file mode 100644 index 000000000000..b733bad47022 --- /dev/null +++ b/src/lib/libast/cdt/dtstat.c @@ -0,0 +1,54 @@ +/*********************************************************************** +* * +* This software is part of the ast package * +* Copyright (c) 1985-2012 AT&T Intellectual Property * +* and is licensed under the * +* Eclipse Public License, Version 1.0 * +* by AT&T Intellectual Property * +* * +* A copy of the License is available at * +* http://www.eclipse.org/org/documents/epl-v10.html * +* (with md5 checksum b35adb5213ca9657e911e9befb180842) * +* * +* Information and Software Systems Research * +* AT&T Research * +* Florham Park NJ * +* * +* Glenn Fowler * +* David Korn * +* Phong Vo * +* * +***********************************************************************/ +#include "dthdr.h" + +/* Get statistics for a dictionary +** +** Written by Kiem-Phong Vo +*/ + +ssize_t dtstat(Dt_t* dt, Dtstat_t* dtst) +{ + ssize_t sz, k, maxk; + char *str; + char *end; + + sz = (ssize_t)(*dt->meth->searchf)(dt, (Void_t*)dtst, DT_STAT); + + str = dtst->mesg; + end = &dtst->mesg[sizeof(dtst->mesg)] - 1; + str += sfsprintf(str, end - str, "Objects=%d Levels=%d(Largest:", dtst->size, dtst->mlev+1); + + /* print top 3 levels */ + for(k = maxk = 0; k <= dtst->mlev; ++k) + if(dtst->lsize[k] > dtst->lsize[maxk]) + maxk = k; + if(maxk > 0) + maxk -= 1; + for(k = 0; k < 3 && maxk <= dtst->mlev; ++k, ++maxk) + str += sfsprintf(str, end - str, " lev[%d]=%d", maxk, dtst->lsize[maxk] ); + if (str < end) + *str++ = ')'; + *str = 0; + + return sz; +} diff --git a/src/lib/libast/cdt/dtstrhash.c b/src/lib/libast/cdt/dtstrhash.c index 7eaac89e3552..a46c8c27628b 100644 --- a/src/lib/libast/cdt/dtstrhash.c +++ b/src/lib/libast/cdt/dtstrhash.c @@ -22,40 +22,38 @@ #include "dthdr.h" /* Hashing a string into an unsigned integer. -** The basic method is to continuingly accumulate bytes and multiply -** with some given prime. The length n of the string is added last. -** The recurrent equation is like this: -** h[k] = (h[k-1] + bytes)*prime for 0 <= k < n -** h[n] = (h[n-1] + n)*prime -** The prime is chosen to have a good distribution of 1-bits so that -** the multiplication will distribute the bits in the accumulator well. -** The below code accumulates 2 bytes at a time for speed. -** -** Written by Kiem-Phong Vo (02/28/03) +** This is the FNV (Fowler-Noll-Vo) hash function. +** Written by Kiem-Phong Vo (01/10/2012) */ #if __STD_C uint dtstrhash(uint h, Void_t* args, ssize_t n) #else uint dtstrhash(h,args,n) -reg uint h; +uint h; Void_t* args; ssize_t n; #endif { unsigned char *s = (unsigned char*)args; - if(n <= 0) - { for(; *s != 0; s += s[1] ? 2 : 1) - h = (h + (s[0]<<8) + s[1])*DT_PRIME; - n = s - (unsigned char*)args; +#if _ast_sizeof_int == 8 /* 64-bit hash */ +#define FNV_PRIME ((1<<40) + (1<<8) + 0xb3) +#define FNV_OFFSET 14695981039346656037 +#else /* 32-bit hash */ +#define FNV_PRIME ((1<<24) + (1<<8) + 0x93) +#define FNV_OFFSET 2166136261 +#endif + h = (h == 0 || h == ~0) ? FNV_OFFSET : h; + if(n <= 0) /* see discipline key definition for == 0 */ + { for(; *s != 0; ++s ) + h = (h ^ s[0]) * FNV_PRIME; } else { unsigned char* ends; - for(ends = s+n-1; s < ends; s += 2) - h = (h + (s[0]<<8) + s[1])*DT_PRIME; - if(s <= ends) - h = (h + (s[0]<<8))*DT_PRIME; + for(ends = s+n; s < ends; ++s) + h = (h ^ s[0]) * FNV_PRIME; } - return (h+n)*DT_PRIME; + + return h; } diff --git a/src/lib/libast/cdt/dttree.c b/src/lib/libast/cdt/dttree.c index 044e75ae4bd5..a1bf7342bbe9 100644 --- a/src/lib/libast/cdt/dttree.c +++ b/src/lib/libast/cdt/dttree.c @@ -545,7 +545,14 @@ int type; } else goto no_root; } - else if(type&DT_REMOVE) /* remove a particular element in the tree */ + else if(type&(DT_DELETE|DT_DETACH)) + { dt_delete: /* remove an object from the dictionary */ + obj = _DTOBJ(disc,root); + _dtfree(dt, root, type); + dt->data->size -= 1; + goto no_root; + } + else if(type&DT_REMOVE) /* remove a particular object */ { if(_DTOBJ(disc,root) == obj) goto dt_delete; else @@ -555,28 +562,32 @@ int type; DTRETURN(obj, NIL(Void_t*)); } } - else if(type&(DT_DELETE|DT_DETACH)) - { dt_delete: /* remove an object from the dictionary */ - obj = _DTOBJ(disc,root); - _dtfree(dt, root, type); - dt->data->size -= 1; - goto no_root; - } else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH)) { if(dt->meth->type&DT_OSET) - { type |= DT_SEARCH; /* for announcement */ + { type |= DT_MATCH; /* for announcement */ goto has_root; } - else + else /* if(dt->meth->type&DT_OBAG) */ { root->_left = NIL(Dtlink_t*); root->_rght = link._left; link._left = root; goto dt_insert; } } + else if(type&DT_INSTALL) + { /* remove old object before insert new one */ + o = _DTOBJ(disc, root); + _dtfree(dt, root, DT_DELETE); + DTANNOUNCE(dt, o, DT_DELETE); + goto dt_insert; + } else if(type&DT_RELINK) /* a duplicate */ { if(dt->meth->type&DT_OSET) + { /* remove object */ + o = _DTOBJ(disc, me); _dtfree(dt, me, DT_DELETE); + DTANNOUNCE(dt, o, DT_DELETE); + } else { me->_left = NIL(Dtlink_t*); me->_rght = link._left; @@ -612,7 +623,7 @@ int type; { obj = NIL(Void_t*); goto no_root; } - else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH)) + else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH|DT_INSTALL)) { dt_insert: if(!(root = _dtmake(dt, obj, type)) ) { obj = NIL(Void_t*); diff --git a/src/lib/libast/cdt/dtuser.c b/src/lib/libast/cdt/dtuser.c new file mode 100644 index 000000000000..9835c5aeb74f --- /dev/null +++ b/src/lib/libast/cdt/dtuser.c @@ -0,0 +1,59 @@ +/*********************************************************************** +* * +* This software is part of the ast package * +* Copyright (c) 1985-2012 AT&T Intellectual Property * +* and is licensed under the * +* Eclipse Public License, Version 1.0 * +* by AT&T Intellectual Property * +* * +* A copy of the License is available at * +* http://www.eclipse.org/org/documents/epl-v10.html * +* (with md5 checksum b35adb5213ca9657e911e9befb180842) * +* * +* Information and Software Systems Research * +* AT&T Research * +* Florham Park NJ * +* * +* Glenn Fowler * +* David Korn * +* Phong Vo * +* * +***********************************************************************/ +#include "dthdr.h" + +/* Perform various functions on the user's behalf. +** +** Written by Kiem-Phong Vo (01/05/2012) +*/ + +/* managing the lock dt->data->user.lock */ +int dtuserlock(Dt_t* dt, unsigned int key, int type) +{ + if(key == 0) + return -1; + else if(type > 0) + return asolock(&dt->data->user.lock, key, ASO_LOCK); + else if(type < 0) + return asolock(&dt->data->user.lock, key, ASO_UNLOCK); + else return asolock(&dt->data->user.lock, key, ASO_TRYLOCK); +} + +/* managing the user data slot dt->data->user.data */ +Void_t* dtuserdata(Dt_t* dt, Void_t* data, int set) +{ + if(set == 0) /* just return current value */ + return asogetptr(&dt->data->user.data); + else while(1) + { Void_t *current = dt->data->user.data; + if(asocasptr(&dt->data->user.data, current, data) == current) + return current; + } +} + +/* announcing an event on the user's behalf */ +int dtuserevent(Dt_t* dt, int flags, Void_t* data) +{ + if(!dt->disc->eventf) + return 0; + else return (*dt->disc->eventf)(dt, DT_ANNOUNCE|DT_USER|flags, data, dt->disc); +} diff --git a/src/lib/libast/include/cdt.h b/src/lib/libast/include/cdt.h index 8641b2f86215..0ab33c31cf1b 100644 --- a/src/lib/libast/include/cdt.h +++ b/src/lib/libast/include/cdt.h @@ -164,6 +164,7 @@ struct _dtstat_s ssize_t msize; /* max #defined elts in below arrays */ ssize_t lsize[DT_MAXSIZE]; /* #objects by level */ ssize_t tsize[DT_MAXSIZE]; /* #tables by level */ + char mesg[256]; /* digest of top level statistics */ }; /* supported storage methods */ @@ -199,7 +200,8 @@ struct _dtstat_s #define DT_ATLEAST 0000040000 /* find the least elt >= object */ #define DT_ATMOST 0000100000 /* find the biggest elt <= object */ #define DT_REMOVE 0002000000 /* remove a specific object */ -#define DT_TOANNOUNCE (DT_INSERT|DT_DELETE|DT_SEARCH|DT_NEXT|DT_PREV|DT_FIRST|DT_LAST|DT_MATCH|DT_ATTACH|DT_DETACH|DT_APPEND|DT_ATLEAST|DT_ATMOST|DT_REMOVE) +#define DT_INSTALL 0004000000 /* install a new object */ +#define DT_TOANNOUNCE (DT_INSERT|DT_DELETE|DT_SEARCH|DT_NEXT|DT_PREV|DT_FIRST|DT_LAST|DT_MATCH|DT_ATTACH|DT_DETACH|DT_APPEND|DT_ATLEAST|DT_ATMOST|DT_REMOVE|DT_INSTALL) #define DT_RELINK 0000002000 /* re-inserting (dtdisc,dtmethod...) */ #define DT_FLATTEN 0000000040 /* flatten objects into a list */ @@ -216,6 +218,7 @@ struct _dtstat_s /* the actual event will be this bit */ /* combined with the operation bit */ #define DT_OPTIMIZE 0100000000 /* optimizing data structure */ +#define DT_USER 0200000000 /* an announcement on user's behalf */ /* events for discipline and method event-handling functions */ #define DT_OPEN 1 /* a dictionary is being opened */ @@ -277,7 +280,8 @@ extern int dtwalk _ARG_((Dt_t*, int(*)(Dt_t*,Void_t*,Void_t*), Void_t*)); extern int dtcustomize _ARG_((Dt_t*, int, int)); extern unsigned int dtstrhash _ARG_((unsigned int, Void_t*, ssize_t)); extern int dtuserlock _ARG_((Dt_t*, unsigned int, int)); -extern Void_t* dtuserdata _ARG_((Dt_t*, Void_t*, unsigned int)); +extern Void_t* dtuserdata _ARG_((Dt_t*, Void_t*, int)); +extern int dtuserevent _ARG_((Dt_t*, int, Void_t*)); /* deal with upward binary compatibility (operation bit translation, etc.) */ extern Dt_t* _dtopen _ARG_((Dtdisc_t*, Dtmethod_t*, unsigned long)); @@ -334,6 +338,7 @@ _END_EXTERNS_ #define dtsearch(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_SEARCH) #define dtmatch(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_MATCH) #define dtinsert(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_INSERT) +#define dtinstall(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_INSTALL) #define dtappend(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_APPEND) #define dtdelete(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_DELETE) #define dtremove(d,o) (*(_DT(d)->searchf))((d),(Void_t*)(o),DT_REMOVE) @@ -345,7 +350,6 @@ _END_EXTERNS_ #define dtextract(d) (Dtlink_t*)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_EXTRACT) #define dtrestore(d,l) (Dtlink_t*)(*(_DT(d)->searchf))((d),(Void_t*)(l),DT_RESTORE) -#define dtstat(d,s) (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(s),DT_STAT) #define dtsize(d) (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT) #define DT_PRIME 17109811 /* 2#00000001 00000101 00010011 00110011 */ diff --git a/src/lib/libast/man/cdt.3 b/src/lib/libast/man/cdt.3 index 86ed082e44e6..755da41998ac 100644 --- a/src/lib/libast/man/cdt.3 +++ b/src/lib/libast/man/cdt.3 @@ -33,6 +33,7 @@ Dtdisc_t; Dtmethod_t; Dtlink_t; Dtstat_t; +Dtuser_t; .Ce .Ss "DICTIONARY CONTROL" .Cs @@ -42,10 +43,10 @@ void dtclear(dt); Dtdisc_t* dtdisc(Dt_t* dt, const Dtdisc_t* disc, int type); Dtmethod_t* dtmethod(Dt_t* dt, const Dtmethod_t* meth); Dt_t* dtview(Dt_t* dt, Dt_t* view); -int dtcustomize(Dt_t* dt, int type, Void_t* arg); -int dtoptimize(Dt_t* dt); -int dtshare(Dt_t* dt, int type); -int dtlock(Dt_t* dt, unsigned int key, int type); +int dtcustomize(Dt_t* dt, int type, int action); +int dtuserlock(Dt_t* dt, unsigned int key, int action); +Void_t* dtuserdata(Dt_t* dt, Void_t* data, int set); +int dtuserevent(Dt_t* dt, int flags, Void_t* data); .Ce .Ss "STORAGE METHODS" .Cs @@ -74,8 +75,10 @@ typedef int (*Dtevent_f)(Dt_t*, int, Void_t*, Dtdisc_t*); .Ss "OBJECT OPERATIONS" .Cs Void_t* dtinsert(Dt_t* dt, Void_t* obj); +Void_t* dtinstall(Dt_t* dt, Void_t* obj); Void_t* dtappend(Dt_t* dt, Void_t* obj); Void_t* dtdelete(Dt_t* dt, Void_t* obj); +Void_t* dtremove(Dt_t* dt, Void_t* obj); Void_t* dtattach(Dt_t* dt, Void_t* obj); Void_t* dtdetach(Dt_t* dt, Void_t* obj); Void_t* dtsearch(Dt_t* dt, Void_t* obj); @@ -84,9 +87,12 @@ Void_t* dtfirst(Dt_t* dt); Void_t* dtnext(Dt_t* dt, Void_t* obj); Void_t* dtlast(Dt_t* dt); Void_t* dtprev(Dt_t* dt, Void_t* obj); -Void_t* dtleast(Dt_t* dt, Void_t* obj); -Void_t* dtmost(Dt_t* dt, Void_t* obj); +Void_t* dtatleast(Dt_t* dt, Void_t* obj); +Void_t* dtatmost(Dt_t* dt, Void_t* obj); int dtwalk(Dt_t* dt, int (*userf)(Dt_t*, Void_t*, Void_t*), Void_t*); +Void_t* dtstart(Dt_t* dt, Void_t* obj); +Void_t* dtstep(Dt_t* dt, Void_t* path); +Void_t* dtstop(Dt_t* dt, Void_t* path); Dtlink_t* dtflatten(Dt_t* dt); Dtlink_t* dtlink(Dt_t* dt, Dtlink_t* link); Void_t* dtobj(Dt_t* dt, Dtlink_t* link); @@ -101,10 +107,9 @@ Dt_t* dtvhere(Dt_t* dt); ssize_t dtsize(Dt_t* dt); ssize_t dtstat(Dt_t* dt, Dtstat_t* st); .Ce -.Ss "HASH FUNCTIONS" +.Ss "HASH FUNCTION" .Cs unsigned int dtstrhash(unsigned int h, char* str, int n); -unsigned int dtcharhash(unsigned int h, unsigned char c); .Ce .SH DESCRIPTION .PP @@ -120,6 +125,7 @@ and \f5char\fP for older C compilation environments. .PP .Ss " Dt_t" This is the type of a dictionary handle. +It contains a field \f5Dt_t.user\fP of type \f5Dtuser_t*\fP (see below). .PP .Ss " Dtdisc_t" This defines the type of a discipline structure which define the lay-out of @@ -134,6 +140,22 @@ This is the type of a dictionary object holder (see \f5dtdisc()\fP.) .Ss " Dtstat_t" This is the type of a structure to return dictionary statistics (see \f5dtstat()\fP.) .PP +.Ss " Dtuser_t" +This is the type of a structure pointed to by \f5Dt_t.user\fP. +If a discipline function \f5memoryf()\fP was defined, this structure +will reside in memory allocated via \f5memoryf\fP. +Although the structure is intended to be used by an application outside of CDT operations, +the functions \f5dtuserlock()\fP and \f5dtuserdata()\fP +are provided for certain common usages of the defined fields. +It should be emphasized, however, that a particular application can choose +to use this structure in anyway that it sees fit. +.Cs + typedef struct + { unsigned int lock; /* for locking a shared dictionary */ + Void_t* data; /* for application-specific data */ + } Dtuser_t; +.Ce +.PP .Ss "DICTIONARY CONTROL" .PP .Ss " Dt_t* dtopen(const Dtdisc_t* disc, const Dtmethod_t* meth)" @@ -183,50 +205,63 @@ In addition, dictionaries on the same view path should treat objects in a consistent manner with respect to comparison or hashing. If not, undefined behaviors may result. .PP -.Ss " int dtcustomize(Dt_t* dt, int type, Void_t* arg)" +.Ss " int dtcustomize(Dt_t* dt, int type, int action)" This customizes a storage method. The \f5type\fP argument -indicates the type of customization and \f5arg\fP gives additional -information for the operation. Here are the types: +is composed of bits indicating different types of customization. +The \f5action\fP argument, if positive, turns on the desired customization; +else, turning it off. +The return value is a bit vector telling the customization types successfully performed. + +Here are the types: .Tp \f5DT_SHARE\fP: -This turns on/off the share mode for a dictionary. -Concurrent accesses of a dictionary not in share mode -may exhibit undefined behaviors including memory segmentation. - -Share mode allows multiple accessors, threads or processes, to access objects. -Such objects could be in the same directory in the case of threads or shared -memory in the case of processes. +This controls the shared or concurrent mode for a dictionary. +Shared mode allows concurrent threads or processes to safely +access objects in a dictionary. +.Tp +\f5DT_ANNOUNCE\fP: +This requires each dictionary access operation to invoke +the discipline \f5eventf\fP function to announce an object found or constructed +by the operation before returning (See the DISCIPLINE section below). .Tp \f5DT_OPTIMIZE\fP: This causes the underlying method to optimize its internal data structure. For example, the splay tree underlying \f5Dtoset\fP would be balanced. .PP -.Ss " int dtoptimize(Dt_t* dt)" -This is a short-hand for invoking \f5dtcustomize()\fP with the \f5DT_OPTIMIZE\fP event. -.PP -.Ss " int dtshare(Dt_t* dt, int type)" -This turns on or off share mode for dictionary \f5dt\fP depending on whether \f5type\fP -is positive or non-positive. It returns -1 on failure. -.PP -.Ss " int dtlock(Dt_t* dt, unsigned int key, int type)" -This globally locks/unlocks a dictionary using the given \f5key\fP. +.Ss " int dtuserlock(Dt_t* dt, unsigned int key, int action)" +This manipulates the lock \f5dt->user->lock\fP. It returns 0 on success and -1 on failure. -The value of \f5key\fP must not be 0. -The argument \f5type\fP is used as follows: +The value of \f5key\fP must be non-zero. +The argument \f5action\fP is used as follows: .Tp -\f5type < 0\fP: -Unlock the dictionary if it was locked with \f5key\fP. -An error will result if the dictionary was locked with a different key. +\f5action < 0\fP: +Unlock \f5dt->user.lock\fP if it was locked with \f5key\fP. +An error will result if \f5dt->user->lock\fP was locked with a different key. .Tp -\f5type == 0\fP: -Attempt to lock the dictionary with \f5key\fP if it is unlocked. +\f5action == 0\fP: +Attempt to lock \f5dt->user->lock\fP with \f5key\fP if it is unlocked. An error will result if the dictionary was already locked with a different key. .Tp -\f5type > 0\fP: -Attempt to lock the dictionary with \f5key\fP. -If the dictionary is already locked with a different key, -the call will loop and wait until the lock is open to lock it. +\f5action > 0\fP: +Attempt to lock \f5dt->user->lock\fP with \f5key\fP. +If \f5dt->user.lock\fP is already locked with a different key, +the call will block until \f5dt->user->lock\fP can be locked with the given \f5key\fP. + +Note that obtaining or removing a lock with \f5dtuserlock()\fP +is just a service provided to the +application for their own use and has nothing to do with dictionary operations +which may or may not employ their own locking schemes based on the semantics +of the container data structures in use. +.PP +.Ss " Void_t* dtuserdata(Dt_t* dt, Void_t* data, int set)" +This function returns the current value of \f5dt->user->data\fP. +In addition, if \f5set\fP is non-zero, +the value of \f5dt->user->data\fP will be changed to \f5data\fP. +.PP +.Ss " int dtuserevent(Dt_t* dt, int flags, Void_t* data)" +This function invokes the discipline event function +with the event \f5DT_ANNOUNCE|DT_USER|flags\fP and the given data. .PP .Ss "STORAGE METHODS" @@ -252,13 +287,14 @@ The underlying data structure is a hash table with chaining to handle collisions These methods are like \f5Dtset\fP and \f5Dtbag\fP but are based on a recursive hashing data structure that allows table extension without object relocation. The data structure also supports lock-free -concurrent search operations for share dictionaries. +concurrent search operations for shared dictionaries and nearly lock-free +insertions and deletions. .PP .Ss " Dtlist" Objects are kept in a list. \fIA current object\fP is always defined to be either the head of the list or an object resulting from a recent search or insert operation. -The call \f5dtinsert()\fP will insert a new object +The calls \f5dtinsert()\fP and \f5dtinstall()\fP will insert a new object in front of such a current object while the call \f5dtappend()\fP will append in back of it. .PP @@ -313,7 +349,7 @@ i.e., at address \f5(Dtlink_t*)((char*)obj+link)\fP. .PP .Ss " Void_t* (*makef)(Dt_t* dt, Void_t* obj, Dtdisc_t* disc)" If \f5makef\fP is not \f5NULL\fP, -\f5dtinsert(dt,obj)\fP or \f5dtappend()\fP will call it +\f5dtinsert()\fP, \f5dtinstall()\fP or \f5dtappend()\fP will call it to make a copy of \f5obj\fP suitable for insertion into \f5dt\fP. If \f5makef\fP is \f5NULL\fP, \f5obj\fP itself will be inserted into \f5dt\fP. .PP @@ -362,11 +398,11 @@ On a negative return value, \f5dtopen()\fP will return failure. On a zero return value, \f5eventf()\fP may set \f5*(Void_t**)data\fP to some non-\f5NULL\fP value to indicate that the dictionary structure itself should be allocated -along with the \f5Dtdisc_t.data\fP section. +along with the \f5Dt_t.data\fP section. Otherwise, it will be allocated separately with \f5malloc(3)\fP. On a positive return value, the dictionary is being reconstructed -based on existing states of some previous dictionary. +based on the existing states of some previous dictionary. In this case, \f5eventf()\fP should set \f5*(Void_t**)data\fP to point to the field \f5Dt_t.data\fP of the corresponding previous dictionary (see \f5DT_CLOSE\fP below). If the handle of the previous dictionary was created as discussed above @@ -386,7 +422,7 @@ The return value of \f5eventf\fP is significant as follows: On a negative return value, \f5dtclose()\fP will return failure. On a zero return value, all dictionary objects will be deleted and -and all associated memory will be freed. +and associated memory freed. On a positive return value, allocated objects and memory will be kept intact. This means that \f5dt->data\fP remains intact and can be reused in some future @@ -398,31 +434,60 @@ This event is raised at the end of the process to close a dictionary. The return value of \f5eventf()\fP will be ignored. .Tp \f5DT_DISC\fP: -The discipline of \f5dt\fP is being changed to a new one given in +This event indicates that the discipline of \f5dt\fP is being changed to a new one given in \f5(Dtdisc_t*)data\fP. .Tp \f5DT_METH\fP: -The method of \f5dt\fP is being changed to a new one given in +This event indicates that the method of \f5dt\fP is being changed to a new one given in \f5(Dtmethod_t*)data\fP. .Tp \f5DT_HASHSIZE\fP: -This event is applicable to -the methods \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP. -It is typically issued when the respective internal data structure of -a method is about to be initialized. -If the return value of the event handling function is positive, -\f5*(ssize_t*)data\fP is examined for further action; -else, it is ignored. -A positive return value means that the event function wishes to suggest a table size. -It does that by setting \f5*(ssize_t*)data\fP to the desired size. -Then, the actual table size will be the maximum of the absolute value -of \f5*(ssize_t*)data\fP and some predefined value set by the method. -In addition, if \f5*(ssize_t*)data\fP was negative, -the \f5Dtset\fP and \f5Dtbag\fP methods will never resize the hash table. +This event is raised by the methods \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP +to ask an application to suggest a size (measured in objects) for the data structure in use. +This is useful, for example, to set a initial size for a hash table to reduce collisions and rehashing. +On each call, \f5*(ssize_t*)data\fP will initially have the current size +(which should be \f50\fP on the first call). + +The return value of the event handling function indicates actions to be taken. +If non-positive, the method will proceed with its default actions. +Otherwise, the application may set \f5*(ssize_t*)data\fP to suggest a table size. +The actual table size will be based on the absolute value of \f5*(ssize_t*)data\fP +but may be modified to suit for the data structure in use. +Further, if \f5*(ssize_t*)data\fP was negative, the size of the hash table will be fixed going forward. .Tp \f5DT_ERROR\fP: -This event announces an error that occurred during some operations. -The argument \f5(char*)data\fP is a null-terminated string describing the error. +This event states an error that occurred during some operations, e.g., +\f5dtinsert()\fP or \f5dtinstall()\fP failing to create a new object due to a memory allocation error. +The argument \f5(char*)data\fP is a null-terminated string describing the problem. +.Tp +\f5DT_ANNOUNCE\fP: +The event will be a combination of this bit and a bit indicating a successful operation. +For example, \f5DT_ANNOUNCE|DT_SEARCH\fP announces that \f5dtsearch()\fP +found the object that was searched for. The \f5data\fP argument points to the object itself. + +The bits representing operations that can cause an announcement are: +\f5DT_INSERT\fP, +\f5DT_DELETE\fP, +\f5DT_REMOVE\fP, +\f5DT_SEARCH\fP, +\f5DT_NEXT\fP, +\f5DT_PREV\fP, +\f5DT_FIRST\fP, +\f5DT_LAST\fP, +\f5DT_MATCH\fP, +\f5DT_ATTACH\fP, +\f5DT_DETACH\fP, +\f5DT_APPEND\fP, +\f5DT_INSTALL\fP, +\f5DT_LEAST\fP, and +\f5DT_MOST\fP. + +Note that a call to \f5dtinsert()\fP or \f5dtattach()\fP may return +a successfully inserted new object or a found matching object. +For \f5dtinsert()\fP, the former case will be announced as \f5DT_ANNOUNCE|DT_INSERT\fP while +the latter as \f5DT_ANNOUNCE|DT_INSERT|DT_SEARCH\fP. +For \f5dtattach()\fP, the events will be similarly announced as \f5DT_ANNOUNCE|DT_ATTACH\fP +and \f5DT_ANNOUNCE|DT_ATTACH|DT_SEARCH\fP. .PP .Ss "#define DTOFFSET(struct_s,member)" This macro function computes the offset of \f5member\fP from the start @@ -436,25 +501,30 @@ with the given values. .Ss "OBJECT OPERATIONS" .PP .Ss " Void_t* dtinsert(Dt_t* dt, Void_t* obj)" +.Ss " Void_t* dtinstall(Dt_t* dt, Void_t* obj)" .Ss " Void_t* dtappend(Dt_t* dt, Void_t* obj)" These functions add an object prototyped by \f5obj\fP into \f5dt\fP. +See \f5Dtdisc_t.makef\fP for object construction. \f5dtinsert()\fP and \f5dtappend()\fP perform the same function for all methods except for \f5Dtlist\fP (see \f5Dtlist\fP for details). -If there is an existing object in \f5dt\fP matching \f5obj\fP -and the storage method is \f5Dtset\fP, \f5Dtrhset\fP or \f5Dtoset\fP, -\f5dtinsert()\fP and \f5dtappend()\fP will simply return the matching object. -Otherwise, a new object is inserted according to the method in use. -See \f5Dtdisc_t.makef\fP for object construction. -The new object or a matching object as noted will be returned on success -while \f5NULL\fP is returned on error. +For \f5Dtset\fP, \f5Dtrhset\fP or \f5Dtoset\fP, +if there is an object in \f5dt\fP matching \f5obj\fP +\f5dtinsert()\fP and \f5dtappend()\fP will not insert a new object. +On the other hand, \f5dtinstall()\fP remove such a matching +object then insert the new object. + +On failure, \f5dtinsert()\fP and \f5dtinstall()\fP return \f5NULL\fP. +Otherwise, the return value is either the newly inserted object +or the matching object as noted. .PP .Ss " Void_t* dtdelete(Dt_t* dt, Void_t* obj)" -If \f5obj\fP is \f5NULL\fP, methods \f5Dtstack\fP and \f5Dtqueue\fP -delete respectively stack top or queue head while other methods do nothing. -If \f5obj\fP is not \f5NULL\fP, an object matching \f5obj\fP is deleted. +.Ss " Void_t* dtremove(Dt_t* dt, Void_t* obj)" +When \f5obj\fP is not \f5NULL\fP, \f5dtdelete()\fP removes some object \fImatching\fP \f5obj\fP +while \f5dtremove()\fP removes \f5obj\fP itself if it exists. +When \f5obj\fP is \f5NULL\fP, if the method is \f5Dtstack\fP or \f5Dtqueue\fP +then the stack top or queue head is respectively deleted. See \f5Dtdisc_t.freef\fP for object destruction. -\f5dtdelete()\fP returns the deleted object (even if it was deallocated) -or \f5NULL\fP on error. +\f5dtdelete()\fP and \f5dtremove()\fP return the deleted object or \f5NULL\fP. .PP .Ss " Void_t* dtattach(Dt_t* dt, Void_t* obj)" This function is similar to \f5dtinsert()\fP but \f5obj\fP itself @@ -469,13 +539,21 @@ from \f5dt\fP will not be freed (via the discipline \f5freef\fP function). .Ss " Void_t* dtmatch(Dt_t* dt, Void_t* key)" These functions find an object matching \f5obj\fP or \f5key\fP either from \f5dt\fP or from some dictionary accessible from \f5dt\fP via a viewpath (see \f5dtview()\fP.) -\f5dtsearch()\fP and \f5dtmatch()\fP return the matching object or -\f5NULL\fP on failure. +The return value is the matching object or \f5NULL\fP. .PP .Ss " Void_t* dtfirst(Dt_t* dt)" .Ss " Void_t* dtnext(Dt_t* dt, Void_t* obj)" -\f5dtfirst()\fP returns the first object in \f5dt\fP. -\f5dtnext()\fP returns the object that follows an object matching \f5obj\fP. +.Ss " Void_t* dtlast(Dt_t* dt)" +.Ss " Void_t* dtprev(Dt_t* dt, Void_t* obj)" +These functions assume some object ordering (more below) and can be used +to iterate over all objects. +\f5dtfirst()\fP returns the first object in \f5dt\fP or \f5NULL\fP if the +dictionary is empty. +\f5dtnext()\fP returns the object coming after \f5obj\fP +or \f5NULL\fP if there is no such object. +\f5dtlast()\fP and \f5dtprev()\fP are like \f5dtfirst()\fP and \f5dtnext()\fP +but work in reverse order. + Objects are ordered based on the storage method in use. For \f5Dtoset\fP and \f5Dtobag\fP, objects are ordered by object comparisons. For \f5Dtstack\fP, objects are ordered in reverse order of insertion. @@ -484,33 +562,47 @@ For \f5Dtlist\fP, objects are ordered by list position. For \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP, objects are ordered by some internal order defined at the time when these functions are called. +In fact, both forward and reverse orders are defined to be the same +for these methods. + +Objects in a dictionary or a viewpath of dictionaries can be walked using +\f5for(;;)\fP loops as below: -Objects in a dictionary or a viewpath can be walked using -a \f5for(;;)\fP loop as below. .Cs for(obj = dtfirst(dt); obj; obj = dtnext(dt,obj)) .Ce -.PP -.Ss " Void_t* dtlast(Dt_t* dt)" -.Ss " Void_t* dtprev(Dt_t* dt, Void_t* obj)" -\f5dtlast()\fP and \f5dtprev()\fP are like \f5dtfirst()\fP and \f5dtnext()\fP -but work in reverse order. -For \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP, -both reverse and forward orders are the same. -Note that dictionaries on a viewpath are still walked in the order -of the viewpath. -.PP -.Ss " Void_t* dtleast(Dt_t* dt, Void_t* obj)" -.Ss " Void_t* dtmost(Dt_t* dt, Void_t* obj)" -\f5dtleast()\fP returns the smallest object greater or equal to \f5obj\fP. -\f5dtmost()\fP returns the largest object smaller or equal to \f5obj\fP. +or +.Cs + for(obj = dtlast(dt); obj; obj = dtprev(dt,obj)) +.Ce + +The argument \f5obj\fP of \f5dtnext()\fP or \f5dtprev()\fP is treated specially +for a method that allows multiple equal elements such as \f5Dtobag\fP or \f5Dtbag\fP. +If it is in the dictionary, then the returned object will be respectively +immediately before or after it in the implicitly defined object ordering. +If it is not in the dictionary but still matching a group of objects, +then the returned object will be immediately after the last or before the first +of the group respectively. +.PP +.PP +.Ss " Void_t* dtatleast(Dt_t* dt, Void_t* obj)" +.Ss " Void_t* dtatmost(Dt_t* dt, Void_t* obj)" +\f5dtatleast()\fP returns the smallest object greater or equal to \f5obj\fP. +\f5dtatmost()\fP returns the largest object smaller or equal to \f5obj\fP. +In addition, if there are multiple such objects in \f5dt\fP +(i.e., when a bag method was used), then +\f5dtatmost()\fP returns the first instance of such an object while +\f5dtatleast()\fP returns the last one. +Both functions return \f5NULL\fP if the desired object does not exist. + Again, object ordering depends on the storage method in use. -For example, with \f5Dtoset\fP and \f5Dtobag\fP, the ordering of objects -is well-defined and it is possible to call \f5dtleast()\fP or \f5dtmost()\fP +With \f5Dtoset\fP and \f5Dtobag\fP, objects are linearly ordered by +the discipline comparison function. +As such, it is possible to call \f5dtatleast()\fP or \f5dtatmost()\fP on an object not in the dictionary and still get a meaningful result. -On the other hand, with \f5Dtset\fP or \f5Dtrhset\fP, such a call will -essentially be the same as \f5dtsearch()\fP because without matching -an object, it cannot be determined what comes before or after. +Storage methods other than \f5Dtoset\fP and \f5Dtobag\fP do not have +an explicit ordering so \f5dtatmost()\fP +and \f5dtatleast()\fP will return \f5NULL\fP when there are no matching objects. .PP .Ss " dtwalk(Dt_t* dt, int (*userf)(Dt_t*, Void_t*, Void_t*), Void_t* data)" This function calls \f5(*userf)(walk,obj,data)\fP on each object in \f5dt\fP and @@ -528,6 +620,7 @@ to walk a single dictionary can incur significant cost due to function calls. For efficient walking of a single directory (i.e., no viewpathing), \f5dtflatten()\fP and \f5dtlink()\fP can be used. Objects in \f5dt\fP are made into a linked list and walked as follows: + .Cs for(link = dtflatten(dt); link; link = dtlink(dt,link) ) .Ce @@ -537,11 +630,48 @@ not \f5Void_t*\fP. That is, it returns a dictionary holder pointer, not a user object pointer (although both are the same if the discipline field \f5link\fP is zero.) The macro function \f5dtlink()\fP -returns the dictionary holder object following \f5link\fP. -The macro function \f5dtobj(dt,link)\fP +returns the dictionary holder object following \f5link\fP and +the macro function \f5dtobj(dt,link)\fP returns the user object associated with \f5link\fP, -Beware that the flattened object list is unflattened on any -dictionary operations other than \f5dtlink()\fP. +Beware that a flattened object list is not guaranteed to maintain integrity +if any dictionary operation other than \f5dtlink()\fP is performed +(for example, this is important to watch out for +if a dictionary is in \f5DT_SHARE\fP mode). +.PP +.Ss " Void_t* dtstart(Dt_t* dt, Void_t* obj);" +This function starts a path for walking a dictionary. +Note that such a path is restricted to \f5dt\fP only while disregarding +all viewpath dictionaries (see \f5dtview()\fP). +On success, a structure +to be used in \f5dtstep()\fP for walking the path is returned. +Otherwise, \f5NULL\fP is returned. + +If \f5obj\fP is \f5NULL\fP, the path starts at the same object returned by \f5dtfirst()\fP. +If \f5obj\fP is not \f5NULL\fP, it must match some object in the dictionary \f5dt\fP +and the path will start there. No matching object will result in error. +.PP +.Ss " Void_t* dtstop(Dt_t* dt, Void_t* path);" +This function ends a path and releases all memory source associated with it. +.PP +.Ss " Void_t* dtstep(Dt_t* dt, Void_t* path);" +This function returns the object at current position in the given \f5path\fP. +Successive calls move forward one object at a time in the same order that \f5dtnext()\fP +does in the example \f5for(;;)\fP loop above. If there is no more object in the path, +\f5dtstep()\fP returns \f5NULL\fP. + +Below is a code fragment showing how to create and walk a path of objects. +This object walking method is more restricted than the \f5dtfirst()/dtnext()\fP method +since viewpathed dictionaries are ignored. +However, it allows multiple paths to be traversed concurrently in the +most efficient manner possible as supported by the underlying data structures. +.Cs + path = dtstart(dt, firstobj); + for(obj = dtstep(dt, path); obj; obj = dtstep(dt,path)) + { + ... + } + dtstop(dt, path); +.Ce .PP .Ss " Dtlink_t* dtextract(Dt_t* dt)" .Ss " Dtlink_t* dtrestore(Dt_t* dt, Dtlink_t* list)" @@ -579,39 +709,232 @@ This has the number of objects in the dictionary. .Tp \f5ssize_t mlev\fP: This returns the maximum number of levels in the data structure used for object storage, i.e., -the binary tree or the recursive hash table. -For a hash table with chaining (i.e., \f5Dtset\fP and \f5Dtbag\fP), +the binary tree (e.g., \f5Dtoset\fP) or the recursive hash table based on a trie structure (e.g., \f5Dtrhset\fP). +For a hash table with chaining (e.g., \f5Dtset\fP and \f5Dtbag\fP), it gives the length of the longest chain. .Tp \f5ssize_t lsize[]\fP: This gives the object counts at each level. -For a hash table with chaining (i.e., \f5Dtset\fP and \f5Dtbag\fP), +For a hash table with chaining (e.g., \f5Dtset\fP and \f5Dtbag\fP), a level is defined as objects at that position in their chains. -Since chains can be arbitrarily long, the report is limited -to objects at a level less than \f5DT_MAXSIZE\fP. +The reported levels is limited to less than \f5DT_MAXSIZE\fP. .Tp \f5ssize_t tsize[]\fP: -For a hash table using a trie structure, this counts the number of +For a recursive hash table using a trie structure (\f5Dtrehash\fP), this counts the number of sub-tables at each level. For example, \f5tsize[0]\fP should be 1 only for this hash table type. +The reported levels is limited to less than \f5DT_MAXSIZE\fP. +.Tp +\f5char* mesg\fP: +A summary message of some of the statistics. .PP .Ss "HASH FUNCTIONS" .PP -.Ss " unsigned int dtcharhash(unsigned int h, char c)" .Ss " unsigned int dtstrhash(unsigned int h, char* str, int n)" -These functions compute hash values from bytes or strings. -\f5dtcharhash()\fP computes a new hash value from byte \f5c\fP and seed value \f5h\fP. -\f5dtstrhash()\fP computes a new hash value from string \f5str\fP and seed value \f5h\fP. +This function computes a new hash value from string \f5str\fP and seed value \f5h\fP. If \f5n\fP is positive, \f5str\fP is a byte array of length \f5n\fP; otherwise, \f5str\fP is a null-terminated string. .PP +.SH CONCURRENCY PROGRAMMING NOTES +Applications requiring concurrent accesses of a dictionary whether via separate threads +or processes using shared memory should turn on shared mode for the dictionary. +CDT uses locking and lockless data structures to +provide safe concurrent accesses of objects. +Much of this work is based on the atomic scalar operations available in \fIlibaso(3)\fP. + +Even though CDT only considers objects +via the attributes specified in a discipline structure, +practical objects will often have many more attributes germane to the needs of an application. +Thus, beyond safe concurrent dictionary operations, an application must also +protect objects in concurrent computations outside of CDT. +In particular, both \fIobject deletion\fP and \fIobject creation\fP should be handled with care. + +The deletion case is relatively simple. +No object should be destroyed as long as there is a reference to it. +This guarantee is automatic when some garbage collection scheme is in place. +Otherwise, some form of reference counting could be used to make sure +that only objects with no reference would be deleted. +An example to be given below discusses how reference counting could be +done using the \f5DT_ANNOUNCE\fP feature of CDT to ensure correct timing +for object deletion. + +In general, object attributes should be well-defined before they are used. +The simplest way to ensure this is to completely construct an object before +before inserting it into a shared dictionary. +However, an application using complex objects may try +to avoid unnecessary construction work as follows. +First, only a partial object with minimal information needed for dictionary operations +is constructed. +Then, either\f5dtinsert()\fP or \f5dtattach()\fP is called to insert this partial object +into the dictionary. If the call returns this same object, then it was properly inserted and +the rest of its attributes could then be filled in. +If only a matching object is returned, then the new object is simply discarded. +Although this object construction strategy works well in single-threaded code, +it can cause references to uninitialized data in concurrent computations +because objects are accessible by concurrent code +as soon as \f5dtinsert()\fP or \f5dtattach()\fP returns. +A way to solve this problem is to make sure that an incomplete object +is completed before allowing any dictionary operation accessing such an object +to return it to the application. + +Both reference counting for safe objection deletion and ensuring readiness +on object creation can be coordinate with CDT via the event \f5DT_ANNOUNCE\fP. +An example of how to do this is given next. +Objects are assumed to be of type \f5Obj_t\fP and have two +fields: \f5ready\fP to indicate the readiness of an object +and \f5refn\fP for reference counting. +Both fields \f5ready\fP and \f5refn\fP are initialized to zero. +Below are the relevant discipline functions \f5Dtdisc_t.eventf\fP +and \f5Dtdisc_t.freef\fP to handle events and to free an object: + +.Cs + int eventf(Dt_t* dt, int type, Void_t* arg, Dtdisc_t* disc) + { + if(type & DT_ANNOUNCE) + { + if(!(type & DT_DELETE) ) + { + Obj_t *obj = (Obj_t*)arg; + + if(type & ~(DT_ANNOUNCE|DT_INSERT|DT_ATTACH)) + while(asogetchar(&obj->ready) == 0 ) + asorelax(1); + + asoaddint(&obj->refn, 1); + } + + return 0; + } + ... + } + + void freef(Dt_t* dt, Void_t* arg, Dtdisc_t* disc) + { + Obj_t *obj = (Obj_t*)arg; + + while(asogetchar(&obj->ready) == 0 ) + asorelax(1); + + while(asogetint(&obj->refn) > 0 ) + asorelax(1); + + ... destroy the object ... + } +.Ce + +Recall that each operation announcement is composed of \f5DT_ANNOUNCE\fP +and some bits to indicate the operation itself. +The test to exclude \f5dtdelete()\fP (indicated by the bit \f5DT_DELETE\fP) +in \f5eventf()\fP is needed because an announcement always occurs +right before the relevant +CDT operation returns and, in the case of \f5dtdelete()\fP, +the object may/will be already destroyed at that time. + +The \f5while()\fP loops in both \f5eventf()\fP and \f5freef()\fP cause +the relevant operations to wait until the object is \fIready\fP (i.e., +all of its attributes are constructed) before proceeding. +The \f5asorelax(1)\fP call yields control of the processor for 1 nanosecond +so other processes can do their work. +Note that the test for \f5~(DT_ANNOUNCE|DT_INSERT|DT_ATTACH)\fP in \f5eventf()\fP +means that the loop will execute for all CDT operations except for +the \f5dtinsert()\fP or \f5dtattach()\fP call that actually inserts \f5obj\fP +into the dictionary (more on this below). + +When the \f5while\fP loop finished, the construction of object \f5obj\fP is known +to be completed. \f5eventf()\fP increases the reference count \f5obj->refn\fP by one +before the respective operation returns \f5obj\fP to the calling code. +On the other hand, \f5freef()\fP waits for the reference +count to reach zero before proceeding to destroy the object. +Waiting for object readiness in \f5freef()\fP before object destruction is necessary +to avoid any issues with deleting uninitialized data. +Again, it should be emphasized that reference counting +is needed only for a memory management model where objects can be freed +regardless of whether or not there are any references to them. +Applications that use some form of garbage collection in general or +for dictionary objects may ignore doing reference counting as done in this example. + +Next, consider a fragment of code to access +objects concurrently from different threads or processes: + +.Cs + if((obj = dtmatch(dt, "key_string")) != NULL) + { + ...process the object obj... + + asosubint(&obj->refn, 1); + dtdelete(dt, obj); + } +.Ce + +The sequence of activities is as follows. +First, the call \f5dtmatch()\fP retrieves an object \f5obj\fP. +An announcement would be made during the call just before \f5obj\fP is returned +causing the reference count of \f5obj\fP to be increased by one. +After processing \f5obj\fP, the reference count is decreased by one using the +atomic subtraction operator \f5asosubint()\fP. +Then, \f5dtdelete()\fP is called to delete the object. + +A possible danger is that concurrent calls to \f5dtdelete()\fP +may end up causing the same memory to be freed more than once. +Fortunately, this cannot happen. +CDT guarantees that, of all the concurrent calls to \f5dtdelete()\fP on \f5obj\fP, +only one will get far enough to make the \f5freef()\fP call while others do nothing. + +Finally, consider a code fragment to construct and use the object \f5obj\fP: + +.Cs + ... construct a partial object obj ... + if((insobj = dtinsert(dt, obj)) == obj ) + { + ... fully construct obj ... + asocaschar(&obj->ready, 0, 1); + + ... compute based on obj... + asosubint(&obj->refn, 1); + } + else + { ... destroy the partial obj ... + + ... compute based on insobj... + asosubint(&insobj->refn, 1); + } +.Ce + +After the \f5dtinsert()\fP call returns, +all other concurrent computations invoking dictionary operations to access \f5obj\fP +will be blocked in the \f5eventf()\fP function until \f5obj->ready\fP is set to 1 +by the above \f5asocaschar()\fP call. +As this is a concurrent computing application, +the above code fragment itself can be +executed in parallel with different but equivalent versions of \f5obj\fP. +In that case, only one \f5dtinsert()\fP call will succeed in inserting a new object +while the others will report a matching object, i.e., the one actually inserted. +The announcement of the successful case is \f5DT_ANNOUNCE|DT_INSERT\fP +while the announcement of the other cases is \f5DT_ANNOUNCE|DT_INSERT|DT_SEARCH\fP. +The bit \f5DT_SEARCH\fP causes \f5eventf()\fP to +to run the loop waiting for object completion. Thus, overall, except for the single case +of a successful insertion of a new object, all other dictionary accesses that involve +this object will return only when the object is ready. + +Note that, for simplicity, the possibility of failure was ignored in the example. +In both successful outcomes of \f5dtinsert()\fP, the reference count of an +appropriate object will be increased by one. Thus, care must be taken to +reduce that reference count for the object after it is no longer needed. +Else, per this example implementation, a deletion of such an object will +cause an infinite loop in the discipline \f5freef()\fP function. +It is possible to implement a delayed object destruction scheme +that avoids an infinite loop waiting for the reference count to drop to zero. +However, a discussion of that is beyond the scope of this document. +.PP .SH IMPLEMENTATION NOTES -\f5Dtlist\fP, \f5Dtstack\fP and \f5Dtqueue\fP are based on doubly linked list. +\f5Dtlist\fP, \f5Dtstack\fP, \f5Dtdeque\fP and \f5Dtqueue\fP are based on doubly linked list. \f5Dtoset\fP and \f5Dtobag\fP are based on top-down splay trees. -\f5Dtset\fP and \f5Dtbag\fP are based on hash tables with -move-to-front collision chains. +\f5Dtset\fP and \f5Dtbag\fP are based on hash tables with collision chains. \f5Dtrhset\fP and \f5Dtrhbag\fP are based on a recursive hashing data structure that avoids table resizing. .PP +.SH SEE ALSO +libaso(3), libvmalloc(3) +.PP .SH AUTHOR Kiem-Phong Vo, kpv@research.att.com