Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Profiling] Manage indices via Elasticsearch #157949

Merged
merged 33 commits into from
Jun 5, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
9fd5fe9
Manage indices via Elasticsearch
jbcrail May 17, 2023
b314e81
Minor formatting
jbcrail May 26, 2023
0d48c4e
Remove unused code
jbcrail May 25, 2023
2c29c45
Initialize Elasticsearch before other steps
jbcrail May 26, 2023
603fff8
Inline cloud enabled check into setup endpoints
jbcrail May 26, 2023
2919e5e
Refactor how steps are checked
jbcrail May 26, 2023
c17f53b
Refactor how setup steps are executed
jbcrail May 26, 2023
526e24d
Fix check for valid APM policy
jbcrail May 26, 2023
4a60e58
Add interface for response body
jbcrail May 26, 2023
b7735a4
Remove mget interface from client
jbcrail May 26, 2023
84525be
Add profiling status endpoint to client
jbcrail May 26, 2023
396326b
Check profiling status
jbcrail May 26, 2023
d3f9a2c
Refactor how resources are setup and verified
jbcrail May 27, 2023
286604c
Check data existence and setup status at same time
jbcrail May 30, 2023
34aa563
Parallelize resource checks
jbcrail May 31, 2023
cff4196
Return error if cloud not enabled when setting up
jbcrail May 31, 2023
da4a8df
Remove check for existing data when setting up
jbcrail May 31, 2023
dbdabfb
Return early if resources are already created
jbcrail May 31, 2023
20fe917
Use accepted status code when setting up
jbcrail May 31, 2023
3bb6ad6
Enable resource management plugin in Elasticsearch
jbcrail May 31, 2023
bf21024
Rename method
jbcrail May 31, 2023
8cf55c7
Update fixtures
jbcrail May 31, 2023
e6ce7cc
Ensure Fleet setup is complete
jbcrail Jun 1, 2023
6e7ab59
Use stricter typing when modifying APM policy
jbcrail Jun 1, 2023
ee49062
Refactor type for setup state
jbcrail Jun 1, 2023
d28d518
Validate policies for collector and symbolizer
jbcrail Jun 1, 2023
a9a6b8c
Simplify how state is managed when checking status
jbcrail Jun 1, 2023
08c6746
Use type variable to simplify
jbcrail Jun 2, 2023
a2ba531
Remove useless async/await syntax
jbcrail Jun 2, 2023
9e4f020
Add comment
jbcrail Jun 2, 2023
6a3e6bc
Find latest package version when creating policy
jbcrail Jun 2, 2023
fb63bb6
Add explicit type
jbcrail Jun 2, 2023
43f4404
Support recursive partial SetupState
jbcrail Jun 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Support recursive partial SetupState
This allows for more ergonomic namespacing in SetupState and gives us a
natural way to extend the interface with more properties in the future.
  • Loading branch information
jbcrail committed Jun 5, 2023
commit 43f4404a862826e0b5a97dc2664ca27f9e4c80a4
56 changes: 33 additions & 23 deletions x-pack/plugins/profiling/common/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* 2.0.
*/

import { RecursivePartial } from '@kbn/apm-plugin/typings/common';

export interface SetupState {
cloud: {
available: boolean;
Expand All @@ -19,14 +21,16 @@ export interface SetupState {
permissions: {
configured: boolean;
};
apm_policy: {
installed: boolean;
};
collector_policy: {
installed: boolean;
};
symbolizer_policy: {
installed: boolean;
policies: {
apm: {
installed: boolean;
};
collector: {
installed: boolean;
};
symbolizer: {
installed: boolean;
};
};
resource_management: {
enabled: boolean;
Expand All @@ -39,6 +43,8 @@ export interface SetupState {
};
}

export type PartialSetupState = RecursivePartial<SetupState>;

export function createDefaultSetupState(): SetupState {
return {
cloud: {
Expand All @@ -54,14 +60,16 @@ export function createDefaultSetupState(): SetupState {
permissions: {
configured: false,
},
apm_policy: {
installed: false,
},
collector_policy: {
installed: false,
},
symbolizer_policy: {
installed: false,
policies: {
apm: {
installed: false,
},
collector: {
installed: false,
},
symbolizer: {
installed: false,
},
},
resource_management: {
enabled: false,
Expand All @@ -81,18 +89,20 @@ export function areResourcesSetup(state: SetupState): boolean {
state.resources.created &&
state.packages.installed &&
state.permissions.configured &&
state.apm_policy.installed &&
state.collector_policy.installed &&
state.symbolizer_policy.installed &&
state.policies.apm.installed &&
state.policies.collector.installed &&
state.policies.symbolizer.installed &&
state.settings.configured
);
}

function mergeRecursivePartial<T>(base: T, partial: RecursivePartial<T>): T {
return { ...base, ...partial };
}

export function mergePartialSetupStates(
base: SetupState,
partials: Array<Partial<SetupState>>
partials: PartialSetupState[]
): SetupState {
return partials.reduce<SetupState>((previous, current) => {
return { ...previous, ...current };
}, base);
return partials.reduce<SetupState>(mergeRecursivePartial, base);
}
4 changes: 2 additions & 2 deletions x-pack/plugins/profiling/server/lib/setup/apm_package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ import {
pkgToPkgKey,
} from '@kbn/fleet-plugin/server/services/epm/registry';
import { ProfilingSetupOptions } from './types';
import { SetupState } from '../../../common/setup';
import { PartialSetupState } from '../../../common/setup';

export async function isApmPackageInstalled({
soClient,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const installation = await getInstallation({
pkgName: 'apm',
savedObjectsClient: soClient,
Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
*/

import { ProfilingSetupOptions } from './types';
import { SetupState } from '../../../common/setup';
import { PartialSetupState } from '../../../common/setup';

const MAX_BUCKETS = 150000;

export async function validateMaximumBuckets({
client,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const settings = await client.getEsClient().cluster.getSettings({});
const maxBuckets = settings.persistent.search?.max_buckets;
return {
Expand All @@ -34,7 +34,7 @@ export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {

export async function validateResourceManagement({
client,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const statusResponse = await client.profilingStatus();
return {
resource_management: {
Expand Down
26 changes: 16 additions & 10 deletions x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { ElasticsearchClient } from '@kbn/core/server';
import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry';
import { getApmPolicy } from './get_apm_policy';
import { ProfilingSetupOptions } from './types';
import { SetupState } from '../../../common/setup';
import { PartialSetupState } from '../../../common/setup';

async function createIngestAPIKey(esClient: ElasticsearchClient) {
const apiKeyResponse = await esClient.security.createApiKey({
Expand Down Expand Up @@ -43,11 +43,13 @@ async function createIngestAPIKey(esClient: ElasticsearchClient) {
export async function validateApmPolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
return {
apm_policy: {
installed: !!(apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling),
policies: {
apm: {
installed: !!(apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling),
},
},
};
}
Expand Down Expand Up @@ -112,11 +114,13 @@ const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer';
export async function validateCollectorPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const packagePolicies = await packagePolicyClient.list(soClient, {});
return {
collector_policy: {
installed: packagePolicies.items.some((pkg) => pkg.name === COLLECTOR_PACKAGE_POLICY_NAME),
policies: {
collector: {
installed: packagePolicies.items.some((pkg) => pkg.name === COLLECTOR_PACKAGE_POLICY_NAME),
},
},
};
}
Expand Down Expand Up @@ -156,11 +160,13 @@ export async function createCollectorPackagePolicy({
export async function validateSymbolizerPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const packagePolicies = await packagePolicyClient.list(soClient, {});
return {
symbolizer_policy: {
installed: packagePolicies.items.some((pkg) => pkg.name === SYMBOLIZER_PACKAGE_POLICY_NAME),
policies: {
symbolizer: {
installed: packagePolicies.items.some((pkg) => pkg.name === SYMBOLIZER_PACKAGE_POLICY_NAME),
},
},
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
*/

import { ProfilingSetupOptions } from './types';
import { SetupState } from '../../../common/setup';
import { PartialSetupState } from '../../../common/setup';

export async function hasProfilingData({
client,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const hasProfilingDataResponse = await client.search('has_any_profiling_data', {
index: 'profiling*',
size: 0,
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/profiling/server/lib/setup/security_role.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
*/

import { ProfilingSetupOptions } from './types';
import { SetupState } from '../../../common/setup';
import { PartialSetupState } from '../../../common/setup';

const PROFILING_READER_ROLE_NAME = 'profiling-reader';

export async function validateSecurityRole({
client,
}: ProfilingSetupOptions): Promise<Partial<SetupState>> {
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const esClient = client.getEsClient();
const roles = await esClient.security.getRole();
return {
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/profiling/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
"@kbn/share-plugin",
"@kbn/observability-shared-plugin",
"@kbn/licensing-plugin",
"@kbn/apm-plugin",
// add references to other TypeScript projects the plugin depends on

// requiredPlugins from ./kibana.json
Expand Down