Skip to content

Commit

Permalink
15467 bhyve needs TSC frequency control
Browse files Browse the repository at this point in the history
15546 want bhyve interface for altering guest time data
Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Reviewed by: Andy Fiddaman <illumos@fiddaman.net>
Approved by: Richard Lowe <richlowe@richlowe.net>
  • Loading branch information
jordanhendricks authored and pfmooney committed Apr 24, 2023
1 parent d48be21 commit 717646f
Show file tree
Hide file tree
Showing 29 changed files with 2,315 additions and 78 deletions.
1 change: 1 addition & 0 deletions usr/src/contrib/bhyve/x86/specialreg.h
Original file line number Diff line number Diff line change
Expand Up @@ -1147,6 +1147,7 @@
#define MSR_SMM_MASK 0xc0010113 /* SMM TSEG address mask */
#define MSR_VM_CR 0xc0010114 /* SVM: feature control */
#define MSR_VM_HSAVE_PA 0xc0010117 /* SVM: host save area address */
#define MSR_AMD_TSC_RATIO 0xc0000104 /* SVM TSC Ratio */
#define MSR_AMD_CPUID07 0xc0011002 /* CPUID 07 %ebx override */
#define MSR_EXTFEATURES 0xc0011005 /* Extended CPUID Features override */
#define MSR_LS_CFG 0xc0011020
Expand Down
5 changes: 5 additions & 0 deletions usr/src/pkg/manifests/system-bhyve-tests.p5m
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,17 @@ file path=opt/bhyve-tests/tests/inst_emul/rdmsr mode=0555
file path=opt/bhyve-tests/tests/inst_emul/triple_fault mode=0555
file path=opt/bhyve-tests/tests/inst_emul/wrmsr mode=0555
dir path=opt/bhyve-tests/tests/kdev
file path=opt/bhyve-tests/tests/kdev/guest_tsc_adjust mode=0555
file path=opt/bhyve-tests/tests/kdev/rdmsr_tsc mode=0555
file path=opt/bhyve-tests/tests/kdev/tsc_freq_ctrl mode=0555
file path=opt/bhyve-tests/tests/kdev/vatpit_freq mode=0555
file path=opt/bhyve-tests/tests/kdev/vhpet_freq mode=0555
file path=opt/bhyve-tests/tests/kdev/vlapic_freq mode=0555
file path=opt/bhyve-tests/tests/kdev/vlapic_freq_periodic mode=0555
file path=opt/bhyve-tests/tests/kdev/vlapic_mmio_access mode=0555
file path=opt/bhyve-tests/tests/kdev/vlapic_msr_access mode=0555
file path=opt/bhyve-tests/tests/kdev/vpmtmr_freq mode=0555
file path=opt/bhyve-tests/tests/kdev/wrmsr_tsc mode=0555
dir path=opt/bhyve-tests/tests/mevent
file path=opt/bhyve-tests/tests/mevent/lists_delete mode=0555
file path=opt/bhyve-tests/tests/mevent/read_disable mode=0555
Expand All @@ -72,6 +76,7 @@ file path=opt/bhyve-tests/tests/vmm/mem_partial mode=0555
file path=opt/bhyve-tests/tests/vmm/mem_seg_map mode=0555
file path=opt/bhyve-tests/tests/vmm/pause_resume mode=0555
file path=opt/bhyve-tests/tests/vmm/self_destruct mode=0555
file path=opt/bhyve-tests/tests/vmm/time_data mode=0555
dir path=usr/kernel/drv group=sys
dir path=usr/kernel/drv/$(ARCH64) group=sys
file path=usr/kernel/drv/$(ARCH64)/vmm_drv_test
Expand Down
6 changes: 4 additions & 2 deletions usr/src/test/bhyve-tests/runfiles/default.run
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# http://www.illumos.org/license/CDDL.
#

# Copyright 2022 Oxide Computer Company
# Copyright 2023 Oxide Computer Company

[DEFAULT]
pre =
Expand Down Expand Up @@ -46,7 +46,9 @@ tests = [
'vlapic_freq_periodic',
'vlapic_mmio_access',
'vlapic_msr_access',
'vpmtmr_freq'
'vpmtmr_freq',
'wrmsr_tsc',
'rdmsr_tsc'
]

[/opt/bhyve-tests/tests/inst_emul]
Expand Down
21 changes: 20 additions & 1 deletion usr/src/test/bhyve-tests/tests/common/common.c
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
*/

/*
* Copyright 2022 Oxide Computer Company
* Copyright 2023 Oxide Computer Company
*/

#include <stdio.h>
Expand Down Expand Up @@ -146,3 +146,22 @@ destroy_instance(const char *suite_name)
return (0);
}
}

/*
* Returns true if running on AMD
*/
bool
cpu_vendor_amd(void)
{
uint_t regs[4];
char cpu_vendor[13];

do_cpuid(0, regs);
((uint_t *)&cpu_vendor)[0] = regs[1];
((uint_t *)&cpu_vendor)[1] = regs[3];
((uint_t *)&cpu_vendor)[2] = regs[2];
cpu_vendor[12] = '\0';

return (strcmp(cpu_vendor, "AuthenticAMD") == 0 ||
strcmp(cpu_vendor, "HygonGenuine") == 0);
}
3 changes: 2 additions & 1 deletion usr/src/test/bhyve-tests/tests/common/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
*/

/*
* Copyright 2022 Oxide Computer Company
* Copyright 2023 Oxide Computer Company
*/

#ifndef _COMMON_H_
Expand All @@ -23,6 +23,7 @@ int open_drv_test(void);
bool check_instance_usable(const char *);
bool check_instance_exists(const char *);
int destroy_instance(const char *);
bool cpu_vendor_amd(void);

#define PROT_ALL (PROT_READ | PROT_WRITE | PROT_EXEC)

Expand Down
12 changes: 10 additions & 2 deletions usr/src/test/bhyve-tests/tests/common/payload_utils.S
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
*/

/*
* Copyright 2022 Oxide Computer Company
* Copyright 2023 Oxide Computer Company
*/

#include <sys/asm_linkage.h>
Expand Down Expand Up @@ -60,7 +60,7 @@ ENTRY(inl)
ret
SET_SIZE(inl)

/* uint64_t wrmsr(uint32_t msr) */
/* uint64_t rdmsr(uint32_t msr) */
ENTRY(rdmsr)
movl %edi, %ecx
rdmsr
Expand Down Expand Up @@ -93,3 +93,11 @@ ENTRY(cpuid)
popq %rbx
ret
SET_SIZE(cpuid)

/* uint64_t rdtsc(void) */
ENTRY(rdtsc)
rdtsc
shlq $32, %rdx
orq %rdx, %rax
ret
SET_SIZE(rdtsc)
3 changes: 2 additions & 1 deletion usr/src/test/bhyve-tests/tests/common/payload_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
*/

/*
* Copyright 2022 Oxide Computer Company
* Copyright 2023 Oxide Computer Company
*/

#ifndef _PAYLOAD_UTILS_H_
Expand All @@ -28,5 +28,6 @@ uint32_t inl(uint16_t);
uint64_t rdmsr(uint32_t);
void wrmsr(uint32_t, uint64_t);
void cpuid(uint32_t, uint32_t, uint32_t *);
uint64_t rdtsc(void);

#endif /* _PAYLOAD_UTILS_H_ */
15 changes: 11 additions & 4 deletions usr/src/test/bhyve-tests/tests/kdev/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# http://www.illumos.org/license/CDDL.
#

# Copyright 2022 Oxide Computer Company
# Copyright 2023 Oxide Computer Company

include $(SRC)/cmd/Makefile.cmd
include $(SRC)/cmd/Makefile.cmd.64
Expand All @@ -23,10 +23,17 @@ PROG = vpmtmr_freq \
vlapic_msr_access \
vatpit_freq

# These should probably go in the `vmm` tests, but since they depend on
# in-guest payloads, it is easier to build them here.
PROG += guest_tsc_adjust \
tsc_freq_ctrl \
wrmsr_tsc \
rdmsr_tsc

PAYLOADS = $(PROG)
include ../Makefile.in_guest

COMMON_OBJS = in_guest.o
COMMON_OBJS = in_guest.o common.o

CLEANFILES = $(COMMON_OBJS) $(PAYLOAD_CLEANFILES) payload_utils.o
CLOBBERFILES = $(PROG)
Expand All @@ -45,8 +52,8 @@ CPPFLAGS = -I$(COMPAT)/bhyve -I$(CONTRIB)/bhyve \
-I$(SRC)/uts/intel \
-I../common

ASFLAGS += -D_ASM
ASFLAGS64 += -D_ASM
ASFLAGS += -D_ASM
ASFLAGS64 += -D_ASM


CFLAGS = -m64
Expand Down
120 changes: 120 additions & 0 deletions usr/src/test/bhyve-tests/tests/kdev/guest_tsc_adjust.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/

/*
* Copyright 2023 Oxide Computer Company
*/

/*
* Test that adjusting the guest TSC with VMM time data interface is visible
* in guest.
*
* Note: requires `vmm_allow_state_writes` to be set
*/

#include <unistd.h>
#include <stdlib.h>
#include <libgen.h>
#include <errno.h>
#include <err.h>

#include <sys/vmm_data.h>
#include <sys/vmm_dev.h>
#include <vmmapi.h>

#include "in_guest.h"

int
main(int argc, char *argv[])
{
const char *test_suite_name = basename(argv[0]);
struct vmctx *ctx = NULL;
int err;

ctx = test_initialize(test_suite_name);

err = test_setup_vcpu(ctx, 0, MEM_LOC_PAYLOAD, MEM_LOC_STACK);
if (err != 0) {
test_fail_errno(err, "Could not initialize vcpu0");
}

const int vmfd = vm_get_device_fd(ctx);

/* Read time data to get baseline guest time values */
struct vdi_time_info_v1 time_info;
struct vm_data_xfer xfer = {
.vdx_class = VDC_VMM_TIME,
.vdx_version = 1,
.vdx_len = sizeof (struct vdi_time_info_v1),
.vdx_data = &time_info,
};
if (ioctl(vmfd, VM_DATA_READ, &xfer) != 0) {
errx(EXIT_FAILURE, "VMM_DATA_READ of time info failed");
}

/* Change the guest TSC to a much larger value */
uint64_t expect_tsc = 500000000000;
time_info.vt_guest_tsc = expect_tsc;
if (ioctl(vmfd, VM_DATA_WRITE, &xfer) != 0) {
int error;
error = errno;
if (error == EPERM) {
warn("VMM_DATA_WRITE got EPERM: is "
"vmm_allow_state_writes set?");
}
errx(EXIT_FAILURE, "VMM_DATA_WRITE of time info failed");
}

struct vm_entry ventry = { 0 };
struct vm_exit vexit = { 0 };

bool half_read = false;
uint64_t tsc;

do {
const enum vm_exit_kind kind =
test_run_vcpu(ctx, 0, &ventry, &vexit);

if (kind == VEK_REENTR) {
continue;
} else if (kind != VEK_UNHANDLED) {
test_fail_vmexit(&vexit);
}

uint32_t val;
if (vexit_match_inout(&vexit, false, IOP_TEST_VALUE, 4,
&val)) {
if (!half_read) {
/* low 32-bits of TSC first */
tsc = val;
half_read = true;
ventry_fulfill_inout(&vexit, &ventry, 0);
} else {
/* high 32-bits of TSC */
tsc |= ((uint64_t)val << 32);

/*
* Check that the TSC reading is at least the
* absurdly high value it was set to.
*/
if (tsc >= expect_tsc) {
(void) printf("tsc=%ld\n", tsc);
test_pass();
} else {
test_fail_msg("TSC %lu < %lu\n", tsc,
expect_tsc);
}
}
} else {
test_fail_vmexit(&vexit);
}
} while (true);
}
30 changes: 30 additions & 0 deletions usr/src/test/bhyve-tests/tests/kdev/payload_guest_tsc_adjust.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/

/*
* Copyright 2023 Oxide Computer Company
*/

#include "payload_common.h"
#include "payload_utils.h"

#define UINT32_MAX 0xffffffff

void
start(void)
{
/* loop for as long as the host wants */
for (;;) {
uint64_t tsc = rdtsc();
outl(IOP_TEST_VALUE, (uint32_t)(UINT32_MAX & tsc));
outl(IOP_TEST_VALUE, (uint32_t)(UINT32_MAX & (tsc >> 32)));
}
}
35 changes: 35 additions & 0 deletions usr/src/test/bhyve-tests/tests/kdev/payload_rdmsr_tsc.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* This file and its contents are supplied under the terms of the
* Common Development and Distribution License ("CDDL"), version 1.0.
* You may only use this file in accordance with the terms of version
* 1.0 of the CDDL.
*
* A full copy of the text of the CDDL should have accompanied this
* source. A copy of the CDDL is also available via the Internet at
* http://www.illumos.org/license/CDDL.
*/

/*
* Copyright 2023 Oxide Computer Company
*/

#include "payload_common.h"
#include "payload_utils.h"
#include "test_defs.h"

#define UINT32_MAX 0xffffffff
#define MSR_TSC 0x10

void
start(void)
{
/* write a value to the TSC */
wrmsr(MSR_TSC, TSC_TARGET_WRVAL);

/* loop for as long as the host wants */
for (;;) {
uint64_t tsc = rdmsr(MSR_TSC);
outl(IOP_TEST_VALUE, UINT32_MAX & tsc);
outl(IOP_TEST_VALUE, UINT32_MAX & (tsc >> 32));
}
}
Loading

0 comments on commit 717646f

Please sign in to comment.