Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HPCC-30365 Add XREF Sasha service to K8s #18798

Draft
wants to merge 23 commits into
base: candidate-9.6.x
Choose a base branch
from
Draft

HPCC-30365 Add XREF Sasha service to K8s #18798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
62d12fa
HPCC-30365 Add XREF Sasha service to K8s
jackdelv Apr 25, 2024
1f2b539
Add constructPartFilename to CLogicalNameEntry
jackdelv Jul 8, 2024
4cceb25
Check if running on localhost before setting Dafs port to 0.
jackdelv Jul 8, 2024
525f982
Change error message to report missing storage planes rather than clu…
jackdelv Jul 8, 2024
757a618
Move group combination to BM only.
jackdelv Jul 9, 2024
4b0b9c7
Common up XRefNodes code with containerized case
jackdelv Jul 9, 2024
18d94ab
Add call to calcStripeNumber and use makePhysucalPartName
jackdelv Jul 10, 2024
f340ccc
Use lfnHash from file metadata instead of logical filename
jackdelv Jul 11, 2024
0f9ba2e
Remove ifdef _CONTAINERIZED from constructPartFilename
jackdelv Aug 6, 2024
b0561b2
Fix dir-per-part and stripNum masks in parseFileName.
jackdelv Aug 6, 2024
6ec8287
Remove Containerized DBGLOG message from generalized BM code in onDFU…
jackdelv Aug 8, 2024
43d9d89
Co-locate fullname closer to makePhysicalPartName
jackdelv Aug 8, 2024
15eaf5d
Properly check for dirPerPart in file flags.
jackdelv Aug 8, 2024
e0b61ca
Add comment specifying meaning of localhost in xrefRemoteDirectories
jackdelv Aug 8, 2024
7bc4e0d
Remove changes to runXref and add directory getting code to CXRefMana…
jackdelv Aug 9, 2024
3af5870
Merge branch 'candidate-9.6.x' into xrefContainerized
jackdelv Aug 20, 2024
72fe732
Remove use of addUniqueXrefNode
jackdelv Nov 6, 2024
33bb654
Add example options to xref
jackdelv Nov 6, 2024
44fd255
Move constructPartFilename funtion to dadfs.cpp
jackdelv Nov 7, 2024
2a463a8
Change to makeStringException
jackdelv Nov 7, 2024
ded8478
Treat paths as case sensitive
jackdelv Nov 7, 2024
5aa25a9
Match file prefix to storage plane definition
jackdelv Nov 8, 2024
7fe148a
Add isContainerized() checks
jackdelv Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion dali/base/dadfs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ extern da_decl cost_type calcDiskWriteCost(const StringArray & clusters, stat_ty

// JCSMORE - I suspect this function should be removed/deprecated. It does not deal with dirPerPart or striping.
// makePhysicalPartName supports both, but does not deal with groups/endpoints)
RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
{
partno--;
StringBuffer tmp;
Expand All @@ -260,6 +260,41 @@ RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partm
return rfn;
}

RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,IStoragePlane *plane,RemoteFilename &rfn)
{
partNo--;
StringBuffer partName;
if (!pmask)
{
pmask = "!ERROR!._$P$_of_$N$";
IERRLOG("No partmask for constructPartFilename");
}
expandMask(partName, pmask, partNo, max);

// Get stripeNum from storage plane
unsigned stripeNum = calcStripeNumber(partNo+1, lfnHash, plane->numDevices());

StringBuffer fullname;
makePhysicalPartName(lname, partNo+1, max, fullname, 0, DFD_OSdefault, prefix, dirPerPart, stripeNum);

unsigned n = 0;
if (!isContainerized())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

re: #18798 (comment)

can you add a comment here to something like:
"revisit: constructPartFilename should probably be refactored not to deal with replicate directories, by pre-determining the alternate prefix if copy>0"

{
ClusterPartDiskMapSpec mspec;
mspec.replicateOffset = replicateOffset;
unsigned d;
mspec.calcPartLocation(partNo, max, copy, grp?grp->ordinality():max, n, d);
setReplicateFilename(fullname, d);
}

SocketEndpoint ep;
if (grp)
ep = grp->queryNode(n).endpoint();
rfn.setPath(ep, fullname.toLowerCase().str());

return rfn;
}

inline void LOGPTREE(const char *title,IPropertyTree *pt)
{
StringBuffer buf;
Expand Down
7 changes: 4 additions & 3 deletions dali/base/dadfs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,14 +811,15 @@ enum DistributedFileSystemError


// utility routines (used by xref and dfu)
extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn);
extern da_decl RemoteFilename &constructPartFilename(IGroup *grp,unsigned partNo,unsigned copy,unsigned max,unsigned lfnHash,int replicateOffset,bool dirPerPart,const char *lname,const char *prefix,const char *pmask,IStoragePlane *plane,RemoteFilename &rfn);
extern da_decl RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn);
// legacy version
inline RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false)
inline RemoteFilename &deprecatedConstructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,bool replicate,int replicateoffset,RemoteFilename &rfn,bool localmount=false)
{
// local mount ignored!
ClusterPartDiskMapSpec mspec;
mspec.replicateOffset = replicateoffset;
return constructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn);
return deprecatedConstructPartFilename(grp,partno,partmax,name,partmask,partdir,replicate?1:0,mspec,rfn);
}

extern da_decl IDFPartFilter *createPartFilter(const char *filter);
Expand Down
3 changes: 2 additions & 1 deletion dali/datest/datest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ static void addTestFile(const char *name,unsigned n)
StringBuffer path;
for (unsigned m=0; m<n; m++) {
RemoteFilename rfn;
constructPartFilename(group,m+1,n,NULL,partmask.str(),dir.str(),false,1,rfn);
// Add tests for constructPartFilename
deprecatedConstructPartFilename(group,m+1,n,NULL,partmask.str(),dir.str(),false,1,rfn);
rfn.getLocalPath(path.clear());
Owned<IPropertyTree> pp = createPTree("Part");
pp->setPropInt64("@size",1234*(m+1));
Expand Down
Loading
Loading