diff --git a/hal/halx86/apic/apicsmp.c b/hal/halx86/apic/apicsmp.c index 6379922dda934..10542382f8de1 100644 --- a/hal/halx86/apic/apicsmp.c +++ b/hal/halx86/apic/apicsmp.c @@ -1,18 +1,22 @@ /* * PROJECT: ReactOS HAL * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) - * FILE: hal/halx86/apic/apicsmp.c * PURPOSE: SMP specific APIC code - * PROGRAMMERS: Copyright 2021 Timo Kreuzer (timo.kreuzer@reactos.org) + * COPYRIGHT: Copyright 2021 Timo Kreuzer + * Copyright 2021 Justin Miller */ /* INCLUDES *******************************************************************/ #include #include "apicp.h" +#include #define NDEBUG #include + +extern PPROCESSOR_IDENTITY HalpProcessorIdentity; + /* INTERNAL FUNCTIONS *********************************************************/ /*! @@ -84,7 +88,6 @@ ApicRequestGlobalInterrupt( /* SMP SUPPORT FUNCTIONS ******************************************************/ -// Should be called by SMP version of HalRequestIpi VOID NTAPI HalpRequestIpi(KAFFINITY TargetProcessors) @@ -93,4 +96,17 @@ HalpRequestIpi(KAFFINITY TargetProcessors) __debugbreak(); } -// APIC specific SMP code here +VOID +ApicStartApplicationProcessor(ULONG NTProcessorNumber, PHYSICAL_ADDRESS StartupLoc) +{ + /* Init IPI */ + ApicRequestGlobalInterrupt(HalpProcessorIdentity[NTProcessorNumber].LapicId, 0, + APIC_MT_INIT, APIC_TGM_Edge, APIC_DSH_Destination); + + /* Stall execution for a bit to give APIC time */ + KeStallExecutionProcessor(1000); + + /* Startup IPI */ + ApicRequestGlobalInterrupt(NTProcessorNumber, (StartupLoc.LowPart) >> 12, + APIC_MT_Startup, APIC_TGM_Edge, APIC_DSH_Destination); +} diff --git a/hal/halx86/apic/processor.c b/hal/halx86/apic/processor.c index c9f370f745924..412f231b5a527 100644 --- a/hal/halx86/apic/processor.c +++ b/hal/halx86/apic/processor.c @@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID) return TRUE; } -/* - * @implemented - */ -BOOLEAN -NTAPI -HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock, - IN PKPROCESSOR_STATE ProcessorState) -{ - /* Ready to start */ - return FALSE; -} - /* * @implemented */ @@ -62,15 +50,4 @@ HalProcessorIdle(VOID) __halt(); } -/* - * @implemented - */ -VOID -NTAPI -HalRequestIpi(KAFFINITY TargetProcessors) -{ - UNIMPLEMENTED; - __debugbreak(); -} - /* EOF */ diff --git a/hal/halx86/generic/up.c b/hal/halx86/generic/up.c new file mode 100644 index 0000000000000..2dcfe30f51dd1 --- /dev/null +++ b/hal/halx86/generic/up.c @@ -0,0 +1,32 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Core source file for UP alternative functions + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include +#define NDEBUG +#include + +/* FUNCTIONS *****************************************************************/ + +VOID +NTAPI +HalRequestIpi(KAFFINITY TargetProcessors) +{ + /* This should never be called in UP mode */ + __debugbreak(); +} + +BOOLEAN +NTAPI +HalStartNextProcessor( + IN PLOADER_PARAMETER_BLOCK LoaderBlock, + IN PKPROCESSOR_STATE ProcessorState) +{ + /* Always return false on UP systems */ + return FALSE; +} diff --git a/hal/halx86/include/smp.h b/hal/halx86/include/smp.h index 06ec42ec6dba3..1709b58d15aa0 100644 --- a/hal/halx86/include/smp.h +++ b/hal/halx86/include/smp.h @@ -41,3 +41,12 @@ HalpSetupProcessorsTable( VOID HalpPrintApicTables(VOID); + +/* APIC specific functions inside apic/apicsmp.c */ + +VOID +ApicStartApplicationProcessor(ULONG NTProcessorNumber, PHYSICAL_ADDRESS StartupLoc); + +VOID +NTAPI +HalpRequestIpi(KAFFINITY TargetProcessors); diff --git a/hal/halx86/pic/processor.c b/hal/halx86/pic/processor.c index 2ea03b57326e4..6b38073ee7cfa 100644 --- a/hal/halx86/pic/processor.c +++ b/hal/halx86/pic/processor.c @@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID) return TRUE; } -/* - * @implemented - */ -BOOLEAN -NTAPI -HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock, - IN PKPROCESSOR_STATE ProcessorState) -{ - /* Ready to start */ - return FALSE; -} - /* * @implemented */ @@ -62,15 +50,4 @@ HalProcessorIdle(VOID) __halt(); } -/* - * @implemented - */ -VOID -NTAPI -HalRequestIpi(KAFFINITY TargetProcessors) -{ - /* Not implemented on UP */ - __debugbreak(); -} - /* EOF */ diff --git a/hal/halx86/smp.cmake b/hal/halx86/smp.cmake index 0f3021338350b..ec89bed72b6ef 100644 --- a/hal/halx86/smp.cmake +++ b/hal/halx86/smp.cmake @@ -2,8 +2,25 @@ list(APPEND HAL_SMP_SOURCE generic/buildtype.c generic/spinlock.c - smp/smp.c) + smp/smp.c + smp/ipi.c) -add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE}) +if(ARCH STREQUAL "i386") + list(APPEND HAL_SMP_ASM_SOURCE + smp/i386/apentry.S) + list(APPEND HAL_SMP_SOURCE + smp/i386/spinup.c) +endif() + + +if(ARCH STREQUAL "amd64") + list(APPEND HAL_SMP_ASM_SOURCE + smp/amd64/apentry.S) + list(APPEND HAL_SMP_SOURCE + smp/amd64/spinup.c) +endif() + +add_asm_files(lib_hal_smp_asm ${HAL_SMP_ASM_SOURCE}) +add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE} ${lib_hal_smp_asm}) add_dependencies(lib_hal_smp bugcodes xdk) target_compile_definitions(lib_hal_smp PRIVATE CONFIG_SMP) diff --git a/hal/halx86/smp/amd64/apentry.S b/hal/halx86/smp/amd64/apentry.S new file mode 100644 index 0000000000000..0f9d34f09b0e7 --- /dev/null +++ b/hal/halx86/smp/amd64/apentry.S @@ -0,0 +1,43 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Assembly file for real mode AP code + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +#include + +PUBLIC APEntry16 +PUBLIC APEntry16End +PUBLIC APEntry32 +PUBLIC APEntryJump32Offset +PUBLIC APEntryJump32Segment +PUBLIC TempPageTableAddr +PUBLIC APEntryCpuState + +.code +APEntry16: + cli + + xor ax, ax + mov ds, ax + mov ss, ax + mov fs, ax + mov gs, ax + + hlt + +APEntry16End: +.long HEX(0) +APEntry32: +.long HEX(0) +APEntryJump32Offset: +.long HEX(0) +APEntryJump32Segment: +.long HEX(0) +TempPageTableAddr: +.long HEX(0) +APEntryCpuState: +.long HEX(0) + +END diff --git a/hal/halx86/smp/amd64/spinup.c b/hal/halx86/smp/amd64/spinup.c new file mode 100644 index 0000000000000..7fb44f7a7202b --- /dev/null +++ b/hal/halx86/smp/amd64/spinup.c @@ -0,0 +1,23 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: and64 AP spinup setup + * COPYRIGHT: Copyright 2023 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include +#include +#define NDEBUG +#include + +BOOLEAN +NTAPI +HalStartNextProcessor( + _In_ PLOADER_PARAMETER_BLOCK LoaderBlock, + _In_ PKPROCESSOR_STATE ProcessorState) +{ + //TODO: + return FALSE; +} diff --git a/hal/halx86/smp/i386/apentry.S b/hal/halx86/smp/i386/apentry.S new file mode 100644 index 0000000000000..4eb25e6dcd239 --- /dev/null +++ b/hal/halx86/smp/i386/apentry.S @@ -0,0 +1,181 @@ +/* + * PROJECT: ReactOS HAL + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Application processor startup code for i386 + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +#include +#include + +#define ZERO_OFFSET(f) (f - _APEntry16) +#define PS(f) (f - _APEntryCpuState) + +PUBLIC _APEntry16 +PUBLIC _APEntry16End +PUBLIC _APEntryJump32Offset +PUBLIC _APEntryJump32Segment +PUBLIC _TempPageTableAddr +PUBLIC _APEntryCpuState +PUBLIC _APEntry32 + +.code16 +_APEntry16: + cli + + /* Load final descriptor tables */ +#ifdef _USE_ML + data32 lgdt fword ptr cs:[ZERO_OFFSET(PS_Gdtr)] + data32 lidt fword ptr cs:[ZERO_OFFSET(PS_Idtr)] +#else + data32 lgdt cs:[ZERO_OFFSET(PS_Gdtr)] + data32 lidt cs:[ZERO_OFFSET(PS_Idtr)] +#endif + + /* Processor state base address */ + mov ebx, cs:[ZERO_OFFSET(PS_SelfPtr)] + mov esi, cs:[ZERO_OFFSET(PS_SegDs)] + + /* Load temp page table */ + mov eax, cs:[ZERO_OFFSET(_TempPageTableAddr)] + mov cr3, eax + + /* Enable paging and protected mode */ + mov eax, cs:[ZERO_OFFSET(PS_Cr0)] + mov cr0, eax + + /* Long jump, 32bit address */ + .byte HEX(66) + .byte HEX(EA) +_APEntryJump32Offset: + .long 0 +_APEntryJump32Segment: + .word 0 + +_APEntry16End: + +.align 4 +_TempPageTableAddr: + .long 0 +// Processor state +_APEntryCpuState: +PS_SelfPtr: + .long 0 // self pointer (for use in protected mode) +PS_Esp: + .long 0 +PS_Eip: + .long 0 +PS_Eflags: + .long 0 +PS_SegCs: + .long 0 +PS_SegDs: + .long 0 +PS_SegEs: + .long 0 +PS_SegSs: + .long 0 +PS_SegFs: + .long 0 +PS_SegGs: + .long 0 +// KSPECIAL_REGISTERS +PS_Cr0: + .long 0 +PS_Cr2: + .long 0 +PS_Cr3: + .long 0 +PS_Cr4: + .long 0 +PS_KernelDr0: + .long 0 +PS_KernelDr1: + .long 0 +PS_KernelDr2: + .long 0 +PS_KernelDr3: + .long 0 +PS_KernelDr6: + .long 0 +PS_KernelDr7: + .long 0 + .space 2 +PS_Gdtr: + .word 0 + .long 0 + .space 2 +PS_Idtr: + .word 0 + .long 0 +PS_Tr: + .word 0 +PS_Ldtr: + .word 0 + .space 4*6 // reserved +.endcode16 + +.code32 +_APEntry32: + cli + + /* Load segment registers from ProcessorState values */ + /* DS should be the first one */ + mov ds, esi + + mov eax, [ebx + PS(PS_SegEs)] + mov es, eax + mov eax, [ebx + PS(PS_SegSs)] + mov ss, eax + mov eax, [ebx + PS(PS_SegFs)] + mov fs, eax + mov eax, [ebx + PS(PS_SegGs)] + mov gs, eax + + /* Write CR registers with ProcessorState values */ + mov eax, [ebx + PS(PS_Cr3)] + mov cr3, eax + mov eax, [ebx + PS(PS_Cr4)] + mov cr4, eax + + /* Load debug registers */ + mov eax, [ebx + PS(PS_KernelDr0)] + mov dr0, eax + mov eax, [ebx + PS(PS_KernelDr1)] + mov dr1, eax + mov eax, [ebx + PS(PS_KernelDr2)] + mov dr2, eax + mov eax, [ebx + PS(PS_KernelDr3)] + mov dr3, eax + mov eax, [ebx + PS(PS_KernelDr6)] + mov dr6, eax + mov eax, [ebx + PS(PS_KernelDr7)] + mov dr7, eax + + /* Load TSS */ + ltr word ptr [ebx + PS(PS_Tr)] + + /* Load AP Stack*/ + mov esp, [ebx + PS(PS_Esp)] + + /* Load Eip and push it as a "return" address */ + mov eax, [ebx + PS(PS_Eip)] + push eax + + /* Load flags */ + mov eax, [ebx + PS(PS_Eflags)] + sahf + + /* Set up all GP registers */ + xor edi, edi + xor esi, esi + xor ebp, ebp + xor ebx, ebx + xor edx, edx + xor ecx, ecx + xor eax, eax + + /* Jump into the kernel */ + ret +END diff --git a/hal/halx86/smp/i386/spinup.c b/hal/halx86/smp/i386/spinup.c new file mode 100644 index 0000000000000..072fa82bd942a --- /dev/null +++ b/hal/halx86/smp/i386/spinup.c @@ -0,0 +1,173 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: i386 AP spinup setup + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +/* INCLUDES ******************************************************************/ + +#include +#include +#define NDEBUG +#include + +/* GLOBALS *******************************************************************/ + +extern PPROCESSOR_IDENTITY HalpProcessorIdentity; +extern PHYSICAL_ADDRESS HalpLowStubPhysicalAddress; +extern PVOID HalpLowStub; + +// The data necessary for a boot (stored inside HalpLowStub) +extern PVOID APEntry16; +extern PVOID APEntry16End; +extern PVOID APEntry32; +extern PVOID APEntryJump32Offset; +extern PVOID APEntryJump32Segment; +extern PVOID TempPageTableAddr; +extern PVOID APEntryCpuState; +extern HALP_APIC_INFO_TABLE HalpApicInfoTable; + +ULONG StartedProcessorCount = 1; + +typedef struct _AP_ENTRY_CPU_STATE +{ + PVOID SelfPtr; + UINT32 Esp; + UINT32 Eip; + UINT32 Eflags; + UINT32 SegCs; + UINT32 SegDs; + UINT32 SegEs; + UINT32 SegSs; + UINT32 SegFs; + UINT32 SegGs; + KSPECIAL_REGISTERS SpecialRegisters; +} AP_ENTRY_CPU_STATE, *PAP_ENTRY_CPU_STATE; + +/* FUNCTIONS *****************************************************************/ + +static +VOID +HalpMapAddressFlat( + _Inout_ PMMPDE PageDirectory, + _In_ PVOID VirtAddress, + _In_ PVOID TargetVirtAddress) +{ + if (TargetVirtAddress == NULL) + TargetVirtAddress = VirtAddress; + + PMMPDE currentPde; + + currentPde = &PageDirectory[MiAddressToPdeOffset(TargetVirtAddress)]; + + // Allocate a Page Table if there is no one for this address + if (currentPde->u.Long == 0) + { + PMMPTE pageTable = ExAllocatePoolZero(NonPagedPool, PAGE_SIZE, TAG_HAL); + ASSERT(pageTable); + + currentPde->u.Hard.PageFrameNumber = MmGetPhysicalAddress(pageTable).QuadPart >> PAGE_SHIFT; + currentPde->u.Hard.Valid = TRUE; + currentPde->u.Hard.Write = TRUE; + } + + // Map the Page Table so we can add our VirtAddress there (hack around I/O memory mapper for that) + PHYSICAL_ADDRESS b = {.QuadPart = (ULONG_PTR)currentPde->u.Hard.PageFrameNumber << PAGE_SHIFT}; + + PMMPTE pageTable = MmMapIoSpace(b, PAGE_SIZE, MmCached); + + PMMPTE currentPte = &pageTable[MiAddressToPteOffset(TargetVirtAddress)]; + currentPte->u.Hard.PageFrameNumber = MmGetPhysicalAddress(VirtAddress).QuadPart >> PAGE_SHIFT; + currentPte->u.Hard.Valid = TRUE; + currentPte->u.Hard.Write = TRUE; + + MmUnmapIoSpace(pageTable, PAGE_SIZE); + + DPRINT("Map %p -> %p, PDE %u PTE %u\n", + TargetVirtAddress, + MmGetPhysicalAddress(VirtAddress).LowPart, + MiAddressToPdeOffset(TargetVirtAddress), + MiAddressToPteOffset(TargetVirtAddress)); +} + +static +PHYSICAL_ADDRESS +HalpInitTempPageTable( + _In_ PKPROCESSOR_STATE ProcessorState) +{ + PMMPDE pageDirectory = ExAllocatePoolZero(NonPagedPool, PAGE_SIZE, TAG_HAL); + ASSERT(pageDirectory); + + // Map the low stub + HalpMapAddressFlat(pageDirectory, HalpLowStub, (PVOID)(ULONG_PTR)HalpLowStubPhysicalAddress.QuadPart); + HalpMapAddressFlat(pageDirectory, HalpLowStub, NULL); + + // Map 32bit mode entry point + HalpMapAddressFlat(pageDirectory, &APEntry32, NULL); + + // Map GDT + HalpMapAddressFlat(pageDirectory, (PVOID)ProcessorState->SpecialRegisters.Gdtr.Base, NULL); + + // Map IDT + HalpMapAddressFlat(pageDirectory, (PVOID)ProcessorState->SpecialRegisters.Idtr.Base, NULL); + + return MmGetPhysicalAddress(pageDirectory); +} + +BOOLEAN +NTAPI +HalStartNextProcessor( + _In_ PLOADER_PARAMETER_BLOCK LoaderBlock, + _In_ PKPROCESSOR_STATE ProcessorState) +{ + if (HalpApicInfoTable.ProcessorCount > StartedProcessorCount) + { + // Initalize the temporary page table + // TODO: clean it up after an AP boots successfully + ULONG_PTR initialCr3 = HalpInitTempPageTable(ProcessorState).QuadPart; + + // Put the bootstrap code into low memory + RtlCopyMemory(HalpLowStub, &APEntry16, ((ULONG_PTR)&APEntry16End - (ULONG_PTR)&APEntry16)); + + // Set the data for 16bit entry code + PUINT32 APEntryJump32OffsetPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&APEntryJump32Offset - (ULONG_PTR)&APEntry16); + PUINT32 APEntryJump32SegmentPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&APEntryJump32Segment - (ULONG_PTR)&APEntry16); + PUINT32 TempPageTableAddrPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&TempPageTableAddr - (ULONG_PTR)&APEntry16); + + PVOID buf = &APEntry32; + + RtlCopyMemory(APEntryJump32OffsetPtr, &buf, sizeof(buf)); + RtlCopyMemory(APEntryJump32SegmentPtr, &ProcessorState->ContextFrame.SegCs, sizeof(UINT16)); + RtlCopyMemory(TempPageTableAddrPtr, &initialCr3, sizeof(initialCr3)); + + // Write processor state stuff + PAP_ENTRY_CPU_STATE apCpuState = (PVOID)((ULONG_PTR)HalpLowStub + ((ULONG_PTR)&APEntryCpuState - (ULONG_PTR)&APEntry16)); + + *apCpuState = (AP_ENTRY_CPU_STATE){ + .SelfPtr = apCpuState, + .Esp = ProcessorState->ContextFrame.Esp, + .Eip = ProcessorState->ContextFrame.Eip, + .Eflags = ProcessorState->ContextFrame.EFlags, + .SegCs = ProcessorState->ContextFrame.SegCs, + .SegDs = ProcessorState->ContextFrame.SegDs, + .SegEs = ProcessorState->ContextFrame.SegEs, + .SegSs = ProcessorState->ContextFrame.SegSs, + .SegFs = ProcessorState->ContextFrame.SegFs, + .SegGs = ProcessorState->ContextFrame.SegGs, + .SpecialRegisters = ProcessorState->SpecialRegisters, + }; + + ApicStartApplicationProcessor(StartedProcessorCount, HalpLowStubPhysicalAddress); + + StartedProcessorCount++; + + return TRUE; + } + else + { + return FALSE; + } +} + diff --git a/hal/halx86/smp/ipi.c b/hal/halx86/smp/ipi.c new file mode 100644 index 0000000000000..90f98c88f24ed --- /dev/null +++ b/hal/halx86/smp/ipi.c @@ -0,0 +1,22 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Source file for IPI management + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include +#include +#define NDEBUG +#include + +/* GLOBALS *******************************************************************/ + +VOID +NTAPI +HalRequestIpi(KAFFINITY TargetProcessors) +{ + HalpRequestIpi(TargetProcessors); +} diff --git a/hal/halx86/smp/smp.c b/hal/halx86/smp/smp.c index f8e2f9e4cdf5b..b9f72e65012ce 100644 --- a/hal/halx86/smp/smp.c +++ b/hal/halx86/smp/smp.c @@ -3,6 +3,7 @@ * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) * PURPOSE: Core source file for SMP management * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin */ /* INCLUDES ******************************************************************/ diff --git a/hal/halx86/up.cmake b/hal/halx86/up.cmake index 24b8103cefee7..47378b83a0135 100644 --- a/hal/halx86/up.cmake +++ b/hal/halx86/up.cmake @@ -1,7 +1,8 @@ list(APPEND HAL_UP_SOURCE generic/buildtype.c - generic/spinlock.c) + generic/spinlock.c + generic/up.c) add_library(lib_hal_up OBJECT ${HAL_UP_SOURCE}) add_dependencies(lib_hal_up bugcodes xdk) diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 517661dfd4ccc..734960a206a86 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1558,6 +1558,11 @@ Phase1InitializationDiscard(IN PVOID Context) KeBootTimeBias = 0; } +#ifdef CONFIG_SMP + /* Start Application Processors */ + KeStartAllProcessors(); +#endif + /* Initialize all processors */ if (!HalAllProcessorsStarted()) KeBugCheck(HAL1_INITIALIZATION_FAILED); diff --git a/ntoskrnl/include/internal/amd64/ke.h b/ntoskrnl/include/internal/amd64/ke.h index 8424669063810..b44cffbfd94c8 100644 --- a/ntoskrnl/include/internal/amd64/ke.h +++ b/ntoskrnl/include/internal/amd64/ke.h @@ -476,6 +476,13 @@ KiSetTrapContext( _In_ PCONTEXT Context, _In_ KPROCESSOR_MODE RequestorMode); +VOID +NTAPI +KiInitializePcr(IN PKIPCR Pcr, + IN ULONG ProcessorNumber, + IN PKTHREAD IdleThread, + IN PVOID DpcStack); + #ifdef __cplusplus } // extern "C" #endif diff --git a/ntoskrnl/include/internal/i386/ke.h b/ntoskrnl/include/internal/i386/ke.h index 30509248a2b9a..f8ddf1349f8e3 100644 --- a/ntoskrnl/include/internal/i386/ke.h +++ b/ntoskrnl/include/internal/i386/ke.h @@ -399,6 +399,17 @@ KiRundownThread(IN PKTHREAD Thread) #endif } +CODE_SEG("INIT") +VOID +NTAPI +KiInitializePcr(IN ULONG ProcessorNumber, + IN PKIPCR Pcr, + IN PKIDTENTRY Idt, + IN PKGDTENTRY Gdt, + IN PKTSS Tss, + IN PKTHREAD IdleThread, + IN PVOID DpcStack); + FORCEINLINE VOID Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) diff --git a/ntoskrnl/include/internal/ke.h b/ntoskrnl/include/internal/ke.h index ae255961388cb..10a352a09811a 100644 --- a/ntoskrnl/include/internal/ke.h +++ b/ntoskrnl/include/internal/ke.h @@ -304,6 +304,13 @@ KiCompleteTimer( IN PKSPIN_LOCK_QUEUE LockQueue ); +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors( + VOID +); + /* gmutex.c ********************************************************************/ VOID diff --git a/ntoskrnl/ke/amd64/mproc.c b/ntoskrnl/ke/amd64/mproc.c new file mode 100644 index 0000000000000..05b5e8a096b0c --- /dev/null +++ b/ntoskrnl/ke/amd64/mproc.c @@ -0,0 +1,22 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Architecture specific source file to hold multiprocessor functions + * COPYRIGHT: Copyright 2022 Justin Miller + */ + +/* INCLUDES *****************************************************************/ + +#include +// #define NDEBUG +#include + +/* FUNCTIONS *****************************************************************/ + +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors() +{ + UNIMPLEMENTED; +} diff --git a/ntoskrnl/ke/i386/kiinit.c b/ntoskrnl/ke/i386/kiinit.c index 9699d11b03c3a..6a737a2a609db 100644 --- a/ntoskrnl/ke/i386/kiinit.c +++ b/ntoskrnl/ke/i386/kiinit.c @@ -536,7 +536,8 @@ KiInitializeKernel(IN PKPROCESS InitProcess, else { /* FIXME */ - DPRINT1("SMP Boot support not yet present\n"); + DPRINT1("Starting CPU#%u - you are brave!\n", Number); + KeLowerIrql(DISPATCH_LEVEL); } /* Setup the Idle Thread */ @@ -824,6 +825,19 @@ KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock) __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu); __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu); + //TODO: We don't setup IPIs yet so freeze other processors here. + if (Cpu) + { + KeMemoryBarrier(); + LoaderBlock->Prcb = 0; + + for(;;) + { + KeMemoryBarrier(); + YieldProcessor(); + } + } + KiVerifyCpuFeatures(Pcr->Prcb); /* Initialize the Processor with HAL */ diff --git a/ntoskrnl/ke/i386/mproc.c b/ntoskrnl/ke/i386/mproc.c new file mode 100644 index 0000000000000..19bd9c5f7edeb --- /dev/null +++ b/ntoskrnl/ke/i386/mproc.c @@ -0,0 +1,195 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Architecture specific source file to hold multiprocessor functions + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +/* INCLUDES *****************************************************************/ + +#include +// #define NDEBUG +#include + +typedef struct _APINFO +{ + DECLSPEC_ALIGN(PAGE_SIZE) KGDTENTRY Gdt[128]; + DECLSPEC_ALIGN(PAGE_SIZE) KIDTENTRY Idt[256]; + DECLSPEC_ALIGN(16) UINT8 NMIStackData[DOUBLE_FAULT_STACK_SIZE]; + KIPCR Pcr; + ETHREAD Thread; + KTSS Tss; + KTSS TssDoubleFault; + KTSS TssNMI; +} APINFO, *PAPINFO; + +/* FUNCTIONS *****************************************************************/ + +FORCEINLINE +PKGDTENTRY +KiGetGdtEntry( + IN PVOID pGdt, + IN USHORT Selector) +{ + return (PKGDTENTRY)((ULONG_PTR)pGdt + (Selector & ~RPL_MASK)); +} + +FORCEINLINE +VOID +KiSetGdtDescriptorBase( + IN OUT PKGDTENTRY Entry, + IN UINT32 Base) +{ + Entry->BaseLow = (UINT16)(Base & 0xffff); + Entry->HighWord.Bytes.BaseMid = (UINT8)((Base >> 16) & 0xff); + Entry->HighWord.Bytes.BaseHi = (UINT8)((Base >> 24) & 0xff); +} + +FORCEINLINE +VOID +KiSetGdtDescriptorLimit( + IN OUT PKGDTENTRY Entry, + IN UINT32 Limit) +{ + if (Limit < 0x100000) + { + Entry->HighWord.Bits.Granularity = 0; + } + else + { + Limit >>= 12; + Entry->HighWord.Bits.Granularity = 1; + } + Entry->LimitLow = (UINT16)(Limit & 0xffff); + Entry->HighWord.Bits.LimitHi = ((Limit >> 16) & 0x0f); +} + +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors(VOID) +{ + PVOID KernelStack, DPCStack; + SIZE_T ProcessorCount = 0; + PAPINFO APInfo; + + while (TRUE) + { + ProcessorCount++; + KernelStack = NULL; + DPCStack = NULL; + + // Allocate structures for a new CPU. + APInfo = ExAllocatePoolZero(NonPagedPool, sizeof(APINFO), ' eK'); + ASSERT(ALIGN_DOWN_POINTER_BY(APInfo, PAGE_SIZE) == APInfo); + if (!APInfo) + { + break; + } + KernelStack = MmCreateKernelStack(FALSE, 0); + if (!KernelStack) + { + break; + } + DPCStack = MmCreateKernelStack(FALSE, 0); + if (!DPCStack) + { + break; + } + + /* Initalize a new PCR for the specific AP */ + KiInitializePcr(ProcessorCount, + &APInfo->Pcr, + &APInfo->Idt[0], + &APInfo->Gdt[0], + &APInfo->Tss, + (PKTHREAD)&APInfo->Thread, + DPCStack); + + // Prepare descriptor tables + KDESCRIPTOR bspGdt, bspIdt; + __sgdt(&bspGdt.Limit); + __sidt(&bspIdt.Limit); + RtlCopyMemory(&APInfo->Gdt, (PVOID)bspGdt.Base, bspGdt.Limit + 1); + RtlCopyMemory(&APInfo->Idt, (PVOID)bspIdt.Base, bspIdt.Limit + 1); + + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_R0_PCR), (ULONG_PTR)&APInfo->Pcr); + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_DF_TSS), (ULONG_PTR)&APInfo->TssDoubleFault); + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_NMI_TSS), (ULONG_PTR)&APInfo->TssNMI); + + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS), (ULONG_PTR)&APInfo->Tss); + // Clear TSS Busy flag (aka set the type to "TSS (Available)") + KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS)->HighWord.Bits.Type = 0b1001; + + APInfo->TssDoubleFault.Esp0 = (ULONG_PTR)&APInfo->NMIStackData; + APInfo->TssDoubleFault.Esp = (ULONG_PTR)&APInfo->NMIStackData; + + APInfo->TssNMI.Esp0 = (ULONG_PTR)&APInfo->NMIStackData; + APInfo->TssNMI.Esp = (ULONG_PTR)&APInfo->NMIStackData; + + // Push LOADER_BLOCK on stack as a parameter + KernelStack = (PVOID)((ULONG_PTR)KernelStack - sizeof(PVOID)); + *(PVOID *)KernelStack = KeLoaderBlock; + + // Push NULL address on stack as a "return" addr + KernelStack = (PVOID)((ULONG_PTR)KernelStack - sizeof(PVOID)); + *(PVOID *)KernelStack = NULL; + + // Fill the processor state + PKPROCESSOR_STATE ProcessorState = &APInfo->Pcr.Prcb->ProcessorState; + RtlZeroMemory(ProcessorState, sizeof(*ProcessorState)); + + ProcessorState->SpecialRegisters.Cr0 = __readcr0(); + ProcessorState->SpecialRegisters.Cr3 = __readcr3(); + ProcessorState->SpecialRegisters.Cr4 = __readcr4(); + + ProcessorState->ContextFrame.SegCs = KGDT_R0_CODE; + ProcessorState->ContextFrame.SegDs = KGDT_R3_DATA; + ProcessorState->ContextFrame.SegEs = KGDT_R3_DATA; + ProcessorState->ContextFrame.SegSs = KGDT_R0_DATA; + ProcessorState->ContextFrame.SegFs = KGDT_R0_PCR; + + ProcessorState->SpecialRegisters.Gdtr.Base = (ULONG_PTR)APInfo->Gdt; + ProcessorState->SpecialRegisters.Gdtr.Limit = sizeof(APInfo->Gdt) - 1; + ProcessorState->SpecialRegisters.Idtr.Base = (ULONG_PTR)APInfo->Idt; + ProcessorState->SpecialRegisters.Idtr.Limit = sizeof(APInfo->Idt) - 1; + + ProcessorState->SpecialRegisters.Tr = KGDT_TSS; + + ProcessorState->ContextFrame.Esp = (ULONG_PTR)KernelStack; + ProcessorState->ContextFrame.Eip = (ULONG_PTR)KiSystemStartup; + ProcessorState->ContextFrame.EFlags = __readeflags() & ~EFLAGS_INTERRUPT_MASK; + + // Prepare the LOADER_PARAMETER_BLOCK structure + KeLoaderBlock->KernelStack = (ULONG_PTR)KernelStack; + KeLoaderBlock->Prcb = (ULONG_PTR)&APInfo->Pcr.Prcb; + KeLoaderBlock->Thread = (ULONG_PTR)&APInfo->Pcr.Prcb->IdleThread; + + // Start the CPU + DPRINT("Starting CPU: #%u\n", ProcessorCount); + if (!HalStartNextProcessor(KeLoaderBlock, ProcessorState)) + { + break; + } + + while (KeLoaderBlock->Prcb != 0) + { + KeMemoryBarrier(); + YieldProcessor(); + } + DPRINT1("Escape\n"); + } + + // The last CPU didn't start - clean the data + ProcessorCount--; + + if (APInfo) + ExFreePoolWithTag(APInfo, ' eK'); + if (KernelStack) + MmDeleteKernelStack(KernelStack, FALSE); + if (DPCStack) + MmDeleteKernelStack(DPCStack, FALSE); + + DPRINT1("KeStartAllProcessors: Sucessful AP startup count is %u\n", ProcessorCount); +} diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake index a85cb19e39a99..30ac08b64397c 100644 --- a/ntoskrnl/ntos.cmake +++ b/ntoskrnl/ntos.cmake @@ -333,6 +333,10 @@ if(ARCH STREQUAL "i386") ${REACTOS_SOURCE_DIR}/ntoskrnl/ps/i386/psldt.c ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmmain.c ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmexec.c) + if(BUILD_MP) + list(APPEND SOURCE + ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/i386/mproc.c) + endif() elseif(ARCH STREQUAL "amd64") list(APPEND ASM_SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/boot.S @@ -358,6 +362,10 @@ elseif(ARCH STREQUAL "amd64") ${REACTOS_SOURCE_DIR}/ntoskrnl/ps/amd64/psctx.c ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/stubs.c ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/usercall.c) + if(BUILD_MP) + list(APPEND SOURCE + ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/mproc.c) + endif() elseif(ARCH STREQUAL "arm") list(APPEND ASM_SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/ex/arm/ioport.s