* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.

/* $NetBSD: cpufunc.h,v 1.42 2020/10/24 07:14:29 mgorny Exp $ */

/*
* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Charles M. Hannum, and by Andrew Doran.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef _X86_CPUFUNC_H_
#define _X86_CPUFUNC_H_

/*
* Functions to provide access to x86-specific instructions.
*/

#include <sys/cdefs.h>
#include <sys/types.h>

#include <machine/segments.h>
#include <machine/specialreg.h>

#ifdef _KERNEL
#if defined(_KERNEL_OPT)
#include "opt_xen.h"
#endif

static inline void
x86_pause(void)
{
__asm volatile ("pause");
}

void x86_lfence(void);
void x86_sfence(void);
void x86_mfence(void);
void x86_flush(void);
void x86_hlt(void);
void x86_stihlt(void);
void tlbflush(void);
void tlbflushg(void);
void invlpg(vaddr_t);
void wbinvd(void);
void breakpoint(void);

#define INVPCID_ADDRESS 0
#define INVPCID_CONTEXT 1
#define INVPCID_ALL 2
#define INVPCID_ALL_NONGLOBAL 3

static inline void
invpcid(register_t op, uint64_t pcid, vaddr_t va)
{
struct {
uint64_t pcid;
uint64_t addr;
} desc = {
.pcid = pcid,
.addr = va
};

__asm volatile (
"invpcid %[desc],%[op]"
:
: [desc] "m" (desc), [op] "r" (op)
: "memory"
);
}

extern uint64_t (*rdtsc)(void);

#define _SERIALIZE_lfence __asm volatile ("lfence")
#define _SERIALIZE_mfence __asm volatile ("mfence")
#define _SERIALIZE_cpuid __asm volatile ("xor %%eax, %%eax;cpuid" ::: \
"eax", "ebx", "ecx", "edx");

#define RDTSCFUNC(fence) \
static inline uint64_t \
rdtsc_##fence(void) \
{ \
uint32_t low, high; \
\
_SERIALIZE_##fence; \
__asm volatile ( \
"rdtsc" \
: "=a" (low), "=d" (high) \
: \
); \
\
return (low | ((uint64_t)high << 32)); \
}

RDTSCFUNC(lfence)
RDTSCFUNC(mfence)
RDTSCFUNC(cpuid)

#undef _SERIALIZE_LFENCE
#undef _SERIALIZE_MFENCE
#undef _SERIALIZE_CPUID

#ifndef XENPV
struct x86_hotpatch_source {
uint8_t *saddr;
uint8_t *eaddr;
};

struct x86_hotpatch_descriptor {
uint8_t name;
uint8_t nsrc;
const struct x86_hotpatch_source *srcs[];
};

void x86_hotpatch(uint8_t, uint8_t);
void x86_patch(bool);
#endif

void x86_monitor(const void *, uint32_t, uint32_t);
void x86_mwait(uint32_t, uint32_t);

static inline void
x86_cpuid2(uint32_t eax, uint32_t ecx, uint32_t *regs)
{
uint32_t ebx, edx;

__asm volatile (
"cpuid"
: "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
: "a" (eax), "c" (ecx)
);

regs[0] = eax;
regs[1] = ebx;
regs[2] = ecx;
regs[3] = edx;
}
#define x86_cpuid(a,b) x86_cpuid2((a), 0, (b))

/* -------------------------------------------------------------------------- */

void lidt(struct region_descriptor *);
void lldt(u_short);
void ltr(u_short);

static inline uint16_t
x86_getss(void)
{
uint16_t val;

__asm volatile (
"mov %%ss,%[val]"
: [val] "=r" (val)
:
);
return val;
}

static inline void
setds(uint16_t val)
{
__asm volatile (
"mov %[val],%%ds"
:
: [val] "r" (val)
);
}

static inline void
setes(uint16_t val)
{
__asm volatile (
"mov %[val],%%es"
:
: [val] "r" (val)
);
}

static inline void
setfs(uint16_t val)
{
__asm volatile (
"mov %[val],%%fs"
:
: [val] "r" (val)
);
}

void setusergs(int);

/* -------------------------------------------------------------------------- */

#define FUNC_CR(crnum) \
static inline void lcr##crnum(register_t val) \
{ \
__asm volatile ( \
"mov %[val],%%cr" #crnum \
: \
: [val] "r" (val) \
: "memory" \
); \
} \
static inline register_t rcr##crnum(void) \
{ \
register_t val; \
__asm volatile ( \
"mov %%cr" #crnum ",%[val]" \
: [val] "=r" (val) \
: \
); \
return val; \
}

#define PROTO_CR(crnum) \
void lcr##crnum(register_t); \
register_t rcr##crnum(void);

#ifndef XENPV
FUNC_CR(0)
FUNC_CR(2)
FUNC_CR(3)
#else
PROTO_CR(0)
PROTO_CR(2)
PROTO_CR(3)
#endif

FUNC_CR(4)
FUNC_CR(8)

/* -------------------------------------------------------------------------- */

#define FUNC_DR(drnum) \
static inline void ldr##drnum(register_t val) \
{ \
__asm volatile ( \
"mov %[val],%%dr" #drnum \
: \
: [val] "r" (val) \
); \
} \
static inline register_t rdr##drnum(void) \
{ \
register_t val; \
__asm volatile ( \
"mov %%dr" #drnum ",%[val]" \
: [val] "=r" (val) \
: \
); \
return val; \
}

#define PROTO_DR(drnum) \
register_t rdr##drnum(void); \
void ldr##drnum(register_t);

#ifndef XENPV
FUNC_DR(0)
FUNC_DR(1)
FUNC_DR(2)
FUNC_DR(3)
FUNC_DR(6)
FUNC_DR(7)
#else
PROTO_DR(0)
PROTO_DR(1)
PROTO_DR(2)
PROTO_DR(3)
PROTO_DR(6)
PROTO_DR(7)
#endif

/* -------------------------------------------------------------------------- */

union savefpu;

static inline void
fninit(void)
{
__asm volatile ("fninit" ::: "memory");
}

static inline void
fnclex(void)
{
__asm volatile ("fnclex");
}

static inline void
fnstcw(uint16_t *val)
{
__asm volatile (
"fnstcw %[val]"
: [val] "=m" (*val)
:
);
}

static inline void
fnstsw(uint16_t *val)
{
__asm volatile (
"fnstsw %[val]"
: [val] "=m" (*val)
:
);
}

static inline void
clts(void)
{
__asm volatile ("clts" ::: "memory");
}

void stts(void);

static inline void
x86_stmxcsr(uint32_t *val)
{
__asm volatile (
"stmxcsr %[val]"
: [val] "=m" (*val)
:
);
}

static inline void
x86_ldmxcsr(uint32_t *val)
{
__asm volatile (
"ldmxcsr %[val]"
:
: [val] "m" (*val)
);
}

void fldummy(void);

static inline uint64_t
rdxcr(uint32_t xcr)
{
uint32_t low, high;

__asm volatile (
"xgetbv"
: "=a" (low), "=d" (high)
: "c" (xcr)
);

return (low | ((uint64_t)high << 32));
}

static inline void
wrxcr(uint32_t xcr, uint64_t val)
{
uint32_t low, high;

low = val;
high = val >> 32;
__asm volatile (
"xsetbv"
:
: "a" (low), "d" (high), "c" (xcr)
);
}

static inline void
fnsave(void *addr)
{
uint8_t *area = addr;

__asm volatile (
"fnsave %[area]"
: [area] "=m" (*area)
:
: "memory"
);
}

static inline void
frstor(const void *addr)
{
const uint8_t *area = addr;

__asm volatile (
"frstor %[area]"
:
: [area] "m" (*area)
: "memory"
);
}

static inline void
fxsave(void *addr)
{
uint8_t *area = addr;

__asm volatile (
"fxsave %[area]"
: [area] "=m" (*area)
:
: "memory"
);
}

static inline void
fxrstor(const void *addr)
{
const uint8_t *area = addr;

__asm volatile (
"fxrstor %[area]"
:
: [area] "m" (*area)
: "memory"
);
}

static inline void
xsave(void *addr, uint64_t mask)
{
uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xsave %[area]"
: [area] "=m" (*area)
: "a" (low), "d" (high)
: "memory"
);
}

static inline void
xsaveopt(void *addr, uint64_t mask)
{
uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xsaveopt %[area]"
: [area] "=m" (*area)
: "a" (low), "d" (high)
: "memory"
);
}

static inline void
xrstor(const void *addr, uint64_t mask)
{
const uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xrstor %[area]"
:
: [area] "m" (*area), "a" (low), "d" (high)
: "memory"
);
}

#ifdef __x86_64__
static inline void
fxsave64(void *addr)
{
uint8_t *area = addr;

__asm volatile (
"fxsave64 %[area]"
: [area] "=m" (*area)
:
: "memory"
);
}

static inline void
fxrstor64(const void *addr)
{
const uint8_t *area = addr;

__asm volatile (
"fxrstor64 %[area]"
:
: [area] "m" (*area)
: "memory"
);
}

static inline void
xsave64(void *addr, uint64_t mask)
{
uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xsave64 %[area]"
: [area] "=m" (*area)
: "a" (low), "d" (high)
: "memory"
);
}

static inline void
xsaveopt64(void *addr, uint64_t mask)
{
uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xsaveopt64 %[area]"
: [area] "=m" (*area)
: "a" (low), "d" (high)
: "memory"
);
}

static inline void
xrstor64(const void *addr, uint64_t mask)
{
const uint8_t *area = addr;
uint32_t low, high;

low = mask;
high = mask >> 32;
__asm volatile (
"xrstor64 %[area]"
:
: [area] "m" (*area), "a" (low), "d" (high)
: "memory"
);
}
#endif

/* -------------------------------------------------------------------------- */

#ifdef XENPV
void x86_disable_intr(void);
void x86_enable_intr(void);
#else
static inline void
x86_disable_intr(void)
{
__asm volatile ("cli" ::: "memory");
}

static inline void
x86_enable_intr(void)
{
__asm volatile ("sti" ::: "memory");
}
#endif /* XENPV */

/* Use read_psl, write_psl when saving and restoring interrupt state. */
u_long x86_read_psl(void);
void x86_write_psl(u_long);

/* Use read_flags, write_flags to adjust other members of %eflags. */
u_long x86_read_flags(void);
void x86_write_flags(u_long);

void x86_reset(void);

/* -------------------------------------------------------------------------- */

/*
* Some of the undocumented AMD64 MSRs need a 'passcode' to access.
* See LinuxBIOSv2: src/cpu/amd/model_fxx/model_fxx_init.c
*/
#define OPTERON_MSR_PASSCODE 0x9c5a203aU

static inline uint64_t
rdmsr(u_int msr)
{
uint32_t low, high;

__asm volatile (
"rdmsr"
: "=a" (low), "=d" (high)
: "c" (msr)
);

return (low | ((uint64_t)high << 32));
}

static inline uint64_t
rdmsr_locked(u_int msr)
{
uint32_t low, high, pass = OPTERON_MSR_PASSCODE;

__asm volatile (
"rdmsr"
: "=a" (low), "=d" (high)
: "c" (msr), "D" (pass)
);

return (low | ((uint64_t)high << 32));
}

int rdmsr_safe(u_int, uint64_t *);

static inline void
wrmsr(u_int msr, uint64_t val)
{
uint32_t low, high;

low = val;
high = val >> 32;
__asm volatile (
"wrmsr"
:
: "a" (low), "d" (high), "c" (msr)
: "memory"
);
}

static inline void
wrmsr_locked(u_int msr, uint64_t val)
{
uint32_t low, high, pass = OPTERON_MSR_PASSCODE;

low = val;
high = val >> 32;
__asm volatile (
"wrmsr"
:
: "a" (low), "d" (high), "c" (msr), "D" (pass)
: "memory"
);
}

#endif /* _KERNEL */

#endif /* !_X86_CPUFUNC_H_ */