diff --git a/boot/freeldr/freeldr/ntldr/arch/i386/winldr.c b/boot/freeldr/freeldr/ntldr/arch/i386/winldr.c index ec91b7313a159..2fccbf9225dea 100644 --- a/boot/freeldr/freeldr/ntldr/arch/i386/winldr.c +++ b/boot/freeldr/freeldr/ntldr/arch/i386/winldr.c @@ -64,81 +64,6 @@ typedef struct #define TYPE_CODE (0x10 | DESCRIPTOR_CODE | DESCRIPTOR_EXECUTE_READ) #define TYPE_DATA (0x10 | DESCRIPTOR_READ_WRITE) -FORCEINLINE -PKGDTENTRY -KiGetGdtEntry( - IN PVOID pGdt, - IN USHORT Selector) -{ - return (PKGDTENTRY)((ULONG_PTR)pGdt + (Selector & ~RPL_MASK)); -} - -FORCEINLINE -VOID -KiSetGdtDescriptorBase( - IN OUT PKGDTENTRY Entry, - IN ULONG32 Base) -{ - Entry->BaseLow = (USHORT)(Base & 0xffff); - Entry->HighWord.Bytes.BaseMid = (UCHAR)((Base >> 16) & 0xff); - Entry->HighWord.Bytes.BaseHi = (UCHAR)((Base >> 24) & 0xff); - // Entry->BaseUpper = (ULONG)(Base >> 32); -} - -FORCEINLINE -VOID -KiSetGdtDescriptorLimit( - IN OUT PKGDTENTRY Entry, - IN ULONG Limit) -{ - if (Limit < 0x100000) - { - Entry->HighWord.Bits.Granularity = 0; - } - else - { - Limit >>= 12; - Entry->HighWord.Bits.Granularity = 1; - } - Entry->LimitLow = (USHORT)(Limit & 0xffff); - Entry->HighWord.Bits.LimitHi = ((Limit >> 16) & 0x0f); -} - -VOID -KiSetGdtEntryEx( - IN OUT PKGDTENTRY Entry, - IN ULONG32 Base, - IN ULONG Limit, - IN UCHAR Type, - IN UCHAR Dpl, - IN BOOLEAN Granularity, - IN UCHAR SegMode) // 0: 16-bit, 1: 32-bit, 2: 64-bit -{ - KiSetGdtDescriptorBase(Entry, Base); - KiSetGdtDescriptorLimit(Entry, Limit); - Entry->HighWord.Bits.Type = (Type & 0x1f); - Entry->HighWord.Bits.Dpl = (Dpl & 0x3); - Entry->HighWord.Bits.Pres = (Type != 0); // Present, must be 1 when the GDT entry is valid. - Entry->HighWord.Bits.Sys = 0; // System - Entry->HighWord.Bits.Reserved_0 = 0; // LongMode = !!(SegMode & 1); - Entry->HighWord.Bits.Default_Big = !!(SegMode & 2); - Entry->HighWord.Bits.Granularity |= !!Granularity; // The flag may have been already set by KiSetGdtDescriptorLimit(). - // Entry->MustBeZero = 0; -} - -FORCEINLINE -VOID -KiSetGdtEntry( - IN OUT PKGDTENTRY Entry, - IN ULONG32 Base, - IN ULONG Limit, - IN UCHAR Type, - IN UCHAR Dpl, - IN UCHAR SegMode) // 0: 16-bit, 1: 32-bit, 2: 64-bit -{ - KiSetGdtEntryEx(Entry, Base, Limit, Type, Dpl, FALSE, SegMode); -} - #if 0 VOID DumpGDTEntry(ULONG_PTR Base, ULONG Selector) diff --git a/hal/halx86/acpi/halacpi.c b/hal/halx86/acpi/halacpi.c index da46365b04902..36a5d474843b9 100644 --- a/hal/halx86/acpi/halacpi.c +++ b/hal/halx86/acpi/halacpi.c @@ -851,12 +851,12 @@ HalpSetupAcpiPhase0(IN PLOADER_PARAMETER_BLOCK LoaderBlock) /* Allocate it */ HalpLowStubPhysicalAddress.QuadPart = HalpAllocPhysicalMemory(LoaderBlock, 0x100000, - 1, + HALP_LOW_STUB_SIZE_IN_PAGES, FALSE); if (HalpLowStubPhysicalAddress.QuadPart) { /* Map it */ - HalpLowStub = HalpMapPhysicalMemory64(HalpLowStubPhysicalAddress, 1); + HalpLowStub = HalpMapPhysicalMemory64(HalpLowStubPhysicalAddress, HALP_LOW_STUB_SIZE_IN_PAGES); } } diff --git a/hal/halx86/apic/apicsmp.c b/hal/halx86/apic/apicsmp.c index 6379922dda934..18ff2fe679355 100644 --- a/hal/halx86/apic/apicsmp.c +++ b/hal/halx86/apic/apicsmp.c @@ -1,18 +1,23 @@ /* * PROJECT: ReactOS HAL * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) - * FILE: hal/halx86/apic/apicsmp.c * PURPOSE: SMP specific APIC code - * PROGRAMMERS: Copyright 2021 Timo Kreuzer (timo.kreuzer@reactos.org) + * COPYRIGHT: Copyright 2021 Timo Kreuzer + * Copyright 2023 Justin Miller */ /* INCLUDES *******************************************************************/ #include #include "apicp.h" +#include + #define NDEBUG #include + +extern PPROCESSOR_IDENTITY HalpProcessorIdentity; + /* INTERNAL FUNCTIONS *********************************************************/ /*! @@ -36,7 +41,7 @@ local APIC(s) specified in Destination field. Vector specifies the startup address. APIC_MT_ExtInt - Delivers an external interrupt to the target local - APIC specified in Destination field. + APIC specified in Destination field. \param TriggerMode - The trigger mode of the interrupt. Can be: APIC_TGM_Edge - The interrupt is edge triggered. @@ -64,6 +69,12 @@ ApicRequestGlobalInterrupt( { APIC_INTERRUPT_COMMAND_REGISTER Icr; + /* Wait for the APIC to be idle */ + do + { + Icr.Long0 = ApicRead(APIC_ICR0); + } while (Icr.DeliveryStatus); + /* Setup the command register */ Icr.LongLong = 0; Icr.Vector = Vector; @@ -84,13 +95,35 @@ ApicRequestGlobalInterrupt( /* SMP SUPPORT FUNCTIONS ******************************************************/ -// Should be called by SMP version of HalRequestIpi VOID NTAPI -HalpRequestIpi(KAFFINITY TargetProcessors) +HalpRequestIpi(_In_ KAFFINITY TargetProcessors) { UNIMPLEMENTED; __debugbreak(); } -// APIC specific SMP code here +VOID +ApicStartApplicationProcessor( + _In_ ULONG NTProcessorNumber, + _In_ PHYSICAL_ADDRESS StartupLoc) +{ + ASSERT(StartupLoc.HighPart == 0); + ASSERT((StartupLoc.QuadPart & 0xFFF) == 0); + ASSERT((StartupLoc.QuadPart & 0xFFF00FFF) == 0); + + /* Init IPI */ + ApicRequestGlobalInterrupt(HalpProcessorIdentity[NTProcessorNumber].LapicId, 0, + APIC_MT_INIT, APIC_TGM_Edge, APIC_DSH_Destination); + + /* De-Assert Init IPI */ + ApicRequestGlobalInterrupt(HalpProcessorIdentity[NTProcessorNumber].LapicId, 0, + APIC_MT_INIT, APIC_TGM_Level, APIC_DSH_Destination); + + /* Stall execution for a bit to give APIC time: MPS Spec - B.4 */ + KeStallExecutionProcessor(200); + + /* Startup IPI */ + ApicRequestGlobalInterrupt(HalpProcessorIdentity[NTProcessorNumber].LapicId, (StartupLoc.LowPart) >> 12, + APIC_MT_Startup, APIC_TGM_Edge, APIC_DSH_Destination); +} diff --git a/hal/halx86/apic/processor.c b/hal/halx86/apic/processor.c index c9f370f745924..412f231b5a527 100644 --- a/hal/halx86/apic/processor.c +++ b/hal/halx86/apic/processor.c @@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID) return TRUE; } -/* - * @implemented - */ -BOOLEAN -NTAPI -HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock, - IN PKPROCESSOR_STATE ProcessorState) -{ - /* Ready to start */ - return FALSE; -} - /* * @implemented */ @@ -62,15 +50,4 @@ HalProcessorIdle(VOID) __halt(); } -/* - * @implemented - */ -VOID -NTAPI -HalRequestIpi(KAFFINITY TargetProcessors) -{ - UNIMPLEMENTED; - __debugbreak(); -} - /* EOF */ diff --git a/hal/halx86/generic/up.c b/hal/halx86/generic/up.c new file mode 100644 index 0000000000000..a7162774d89f3 --- /dev/null +++ b/hal/halx86/generic/up.c @@ -0,0 +1,34 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Core source file for Uniprocessor (UP) alternative functions + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include + +#define NDEBUG +#include + +/* FUNCTIONS *****************************************************************/ + +VOID +NTAPI +HalRequestIpi( + _In_ KAFFINITY TargetProcessors) +{ + /* This should never be called in UP mode */ + __debugbreak(); +} + +BOOLEAN +NTAPI +HalStartNextProcessor( + _In_ PLOADER_PARAMETER_BLOCK LoaderBlock, + _In_ PKPROCESSOR_STATE ProcessorState) +{ + /* Always return false on UP systems */ + return FALSE; +} diff --git a/hal/halx86/include/halp.h b/hal/halx86/include/halp.h index 9ea22e8c3c559..21ae397621ed2 100644 --- a/hal/halx86/include/halp.h +++ b/hal/halx86/include/halp.h @@ -53,6 +53,12 @@ VOID #define IDT_INTERNAL 0x11 #define IDT_DEVICE 0x21 +#if _M_AMD64 +#define HALP_LOW_STUB_SIZE_IN_PAGES 5 +#else +#define HALP_LOW_STUB_SIZE_IN_PAGES 3 +#endif + /* Conversion functions */ #define BCD_INT(bcd) \ (((bcd & 0xF0) >> 4) * 10 + (bcd & 0x0F)) diff --git a/hal/halx86/include/smp.h b/hal/halx86/include/smp.h index 06ec42ec6dba3..86b1ded5f775e 100644 --- a/hal/halx86/include/smp.h +++ b/hal/halx86/include/smp.h @@ -41,3 +41,15 @@ HalpSetupProcessorsTable( VOID HalpPrintApicTables(VOID); + +/* APIC specific functions inside apic/apicsmp.c */ + +VOID +ApicStartApplicationProcessor( + _In_ ULONG NTProcessorNumber, + _In_ PHYSICAL_ADDRESS StartupLoc); + +VOID +NTAPI +HalpRequestIpi( + _In_ KAFFINITY TargetProcessors); diff --git a/hal/halx86/pic/processor.c b/hal/halx86/pic/processor.c index 2ea03b57326e4..6b38073ee7cfa 100644 --- a/hal/halx86/pic/processor.c +++ b/hal/halx86/pic/processor.c @@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID) return TRUE; } -/* - * @implemented - */ -BOOLEAN -NTAPI -HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock, - IN PKPROCESSOR_STATE ProcessorState) -{ - /* Ready to start */ - return FALSE; -} - /* * @implemented */ @@ -62,15 +50,4 @@ HalProcessorIdle(VOID) __halt(); } -/* - * @implemented - */ -VOID -NTAPI -HalRequestIpi(KAFFINITY TargetProcessors) -{ - /* Not implemented on UP */ - __debugbreak(); -} - /* EOF */ diff --git a/hal/halx86/smp.cmake b/hal/halx86/smp.cmake index 0f3021338350b..f24116a23533f 100644 --- a/hal/halx86/smp.cmake +++ b/hal/halx86/smp.cmake @@ -2,8 +2,22 @@ list(APPEND HAL_SMP_SOURCE generic/buildtype.c generic/spinlock.c - smp/smp.c) + smp/smp.c + smp/ipi.c) -add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE}) +if(ARCH STREQUAL "i386") + list(APPEND HAL_SMP_ASM_SOURCE + smp/i386/apentry.S) + list(APPEND HAL_SMP_SOURCE + smp/i386/spinup.c) +elseif(ARCH STREQUAL "amd64") + list(APPEND HAL_SMP_ASM_SOURCE + smp/amd64/apentry.S) + list(APPEND HAL_SMP_SOURCE + smp/amd64/spinup.c) +endif() + +add_asm_files(lib_hal_smp_asm ${HAL_SMP_ASM_SOURCE}) +add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE} ${lib_hal_smp_asm}) add_dependencies(lib_hal_smp bugcodes xdk) target_compile_definitions(lib_hal_smp PRIVATE CONFIG_SMP) diff --git a/hal/halx86/smp/amd64/apentry.S b/hal/halx86/smp/amd64/apentry.S new file mode 100644 index 0000000000000..6b3b2c2c1b40e --- /dev/null +++ b/hal/halx86/smp/amd64/apentry.S @@ -0,0 +1,43 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: AMD64 Application Processor (AP) spinup setup + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +#include + +PUBLIC APEntry16 +PUBLIC APEntry16End +PUBLIC APEntry32 +PUBLIC APEntryJump32Offset +PUBLIC APEntryJump32Segment +PUBLIC TempPageTableAddr +PUBLIC APEntryCpuState + +.code +APEntry16: + cli + + xor ax, ax + mov ds, ax + mov ss, ax + mov fs, ax + mov gs, ax + + hlt + +APEntry16End: +.long HEX(0) +APEntry32: +.long HEX(0) +APEntryJump32Offset: +.long HEX(0) +APEntryJump32Segment: +.long HEX(0) +TempPageTableAddr: +.long HEX(0) +APEntryCpuState: +.long HEX(0) + +END diff --git a/hal/halx86/smp/amd64/spinup.c b/hal/halx86/smp/amd64/spinup.c new file mode 100644 index 0000000000000..058cafbd7d92a --- /dev/null +++ b/hal/halx86/smp/amd64/spinup.c @@ -0,0 +1,24 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: and64 AP spinup setup + * COPYRIGHT: Copyright 2023 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include +#include + +#define NDEBUG +#include + +BOOLEAN +NTAPI +HalStartNextProcessor( + _In_ PLOADER_PARAMETER_BLOCK LoaderBlock, + _In_ PKPROCESSOR_STATE ProcessorState) +{ + //TODO: + return FALSE; +} diff --git a/hal/halx86/smp/i386/apentry.S b/hal/halx86/smp/i386/apentry.S new file mode 100644 index 0000000000000..c5cbd2c56f67e --- /dev/null +++ b/hal/halx86/smp/i386/apentry.S @@ -0,0 +1,137 @@ +/* + * PROJECT: ReactOS HAL + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: i386 Application Processor (AP) spinup setup + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +#include +#include + +#define ZERO_OFFSET(f) (f - _HalpAPEntry16) +#define PS(f) (f - _HalpAPEntryData) + +PUBLIC _HalpAPEntry16 +PUBLIC _HalpAPEntryData +PUBLIC _HalpAPEntry32 +PUBLIC _HalpAPEntry16End + +.code16 +_HalpAPEntry16: + cli + + /* Calculate the flat base address */ + mov ebp, cs + shl ebp, 4 + + /* Use flat addressing */ + xor eax, eax + mov ds, eax + +#ifdef _USE_ML + data32 lgdt fword ptr cs:[ZERO_OFFSET(Gdtr)] +#else + data32 lgdt cs:[ZERO_OFFSET(Gdtr)] +#endif + + /* Load temp page table */ + mov eax, cs:[ZERO_OFFSET(PageTableRoot)] + mov cr3, eax + + mov eax, cr0 + or eax, HEX(80000001) + mov cr0, eax + +.align 4 + /* Long jump, 32bit address */ + .byte HEX(66) + .byte HEX(EA) +_HalpAPEntryData: +_APEntryJump32Offset: + .long 0 +_APEntryJump32Segment: + .long 8 +SelfPtr: + .long 0 +PageTableRoot: + .long 0 +ProcessorState: + .long 0 +Gdtr_Pad: + .short 0 // Pad +Gdtr: + .short 0 // Limit + .long 0 // Base +_HalpAPEntry16End: +.endcode16 + +.code32 +_HalpAPEntry32: + /* Set the Ring 0 DS/ES/SS Segment */ + mov ax, HEX(10) + mov ds, ax + mov es, ax + mov ss, ax + mov gs, ax + + /* Load ProcessorState pointer */ + mov esi, [ebp + ZERO_OFFSET(ProcessorState)] + + mov eax, [esi + PsContextFrame + CsSegDs] + mov ds, eax + mov eax, [esi + PsContextFrame + CsSegEs] + mov es, eax + mov eax, [esi + PsContextFrame + CsSegSs] + mov ss, eax + mov eax, [esi + PsContextFrame + CsSegFs] + mov fs, eax + mov eax, [esi + PsContextFrame + CsSegGs] + mov gs, eax + + /* Write CR registers with ProcessorState values */ + mov eax, [esi + PsSpecialRegisters + SrCr3] + mov cr3, eax + mov eax, [esi + PsSpecialRegisters + SrCr4] + mov cr4, eax + + /* Load debug registers */ + mov eax, [esi + PsSpecialRegisters + SrKernelDr0] + mov dr0, eax + mov eax, [esi + PsSpecialRegisters + SrKernelDr1] + mov dr1, eax + mov eax, [esi + PsSpecialRegisters + SrKernelDr2] + mov dr2, eax + mov eax, [esi + PsSpecialRegisters + SrKernelDr3] + mov dr3, eax + mov eax, [esi + PsSpecialRegisters + SrKernelDr6] + mov dr6, eax + mov eax, [esi + PsSpecialRegisters + SrKernelDr7] + mov dr7, eax + + /* Load TSS */ + ltr word ptr [esi + PsSpecialRegisters + SrTr] + + /* Load AP Stack */ + mov esp, [esi + PsContextFrame + CsEsp] + + /* Load Eip and push it as a "return" address */ + mov eax, [esi + PsContextFrame + CsEip] + push eax + + /* Load flags */ + mov eax, [esi + PsContextFrame + EFlags] + sahf + + /* Set up all GP registers */ + xor edi, edi + xor esi, esi + xor ebp, ebp + xor ebx, ebx + xor edx, edx + xor ecx, ecx + xor eax, eax + + /* Jump into the kernel */ + ret +END diff --git a/hal/halx86/smp/i386/spinup.c b/hal/halx86/smp/i386/spinup.c new file mode 100644 index 0000000000000..32d268ec8f4fa --- /dev/null +++ b/hal/halx86/smp/i386/spinup.c @@ -0,0 +1,120 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: i386 Application Processor (AP) spinup setup + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +/* INCLUDES ******************************************************************/ + +#include +#include + +#define NDEBUG +#include + +/* GLOBALS *******************************************************************/ + +extern PPROCESSOR_IDENTITY HalpProcessorIdentity; +extern PHYSICAL_ADDRESS HalpLowStubPhysicalAddress; +extern PVOID HalpLowStub; + +// The data necessary for a boot (stored inside HalpLowStub) +extern PVOID HalpAPEntry16; +extern PVOID HalpAPEntryData; +extern PVOID HalpAPEntry32; +extern PVOID HalpAPEntry16End; +extern HALP_APIC_INFO_TABLE HalpApicInfoTable; + +ULONG HalpStartedProcessorCount = 1; + +#ifndef Add2Ptr +#define Add2Ptr(P,I) ((PVOID)((PUCHAR)(P) + (I))) +#endif +#ifndef PtrOffset +#define PtrOffset(B,O) ((ULONG)((ULONG_PTR)(O) - (ULONG_PTR)(B))) +#endif + +typedef struct _AP_ENTRY_DATA +{ + UINT32 Jump32Offset; + ULONG Jump32Segment; + PVOID SelfPtr; + ULONG PageTableRoot; + PKPROCESSOR_STATE ProcessorState; + KDESCRIPTOR Gdtr; +} AP_ENTRY_DATA, *PAP_ENTRY_DATA; + +/* FUNCTIONS *****************************************************************/ + +static +ULONG +HalpSetupTemporaryMappings( + _In_ PKPROCESSOR_STATE ProcessorState) +{ + PMMPDE RootPageTable = Add2Ptr(HalpLowStub, PAGE_SIZE); + PMMPDE LowMapPde = Add2Ptr(HalpLowStub, 2 * PAGE_SIZE); + PMMPTE LowStubPte = MiAddressToPte(HalpLowStub); + PHYSICAL_ADDRESS PhysicalAddress; + ULONG StartPti; + + /* Copy current mappings */ + RtlCopyMemory(RootPageTable, MiAddressToPde(NULL), PAGE_SIZE); + + /* Set up low PDE */ + PhysicalAddress = MmGetPhysicalAddress(LowMapPde); + RootPageTable[0].u.Hard.PageFrameNumber = PhysicalAddress.QuadPart >> PAGE_SHIFT; + RootPageTable[0].u.Hard.Valid = 1; + RootPageTable[0].u.Hard.Write = 1; + + /* Copy low stub PTEs */ + StartPti = MiAddressToPteOffset(HalpLowStubPhysicalAddress.QuadPart); + ASSERT(StartPti + HALP_LOW_STUB_SIZE_IN_PAGES < 1024); + for (ULONG i = 0; i < HALP_LOW_STUB_SIZE_IN_PAGES; i++) + { + LowMapPde[StartPti + i] = LowStubPte[i]; + } + + PhysicalAddress = MmGetPhysicalAddress(RootPageTable); + ASSERT(PhysicalAddress.QuadPart < 0x100000000); + return (ULONG)PhysicalAddress.QuadPart; +} + +BOOLEAN +NTAPI +HalStartNextProcessor( + _In_ PLOADER_PARAMETER_BLOCK LoaderBlock, + _In_ PKPROCESSOR_STATE ProcessorState) +{ + if (HalpStartedProcessorCount == HalpApicInfoTable.ProcessorCount) + return FALSE; + + // Initalize the temporary page table + // TODO: clean it up after an AP boots successfully + ULONG initialCr3 = HalpSetupTemporaryMappings(ProcessorState); + if (!initialCr3) + return FALSE; + + // Put the bootstrap code into low memory + RtlCopyMemory(HalpLowStub, &HalpAPEntry16, (ULONG_PTR)&HalpAPEntry16End - (ULONG_PTR)&HalpAPEntry16); + + // Get a pointer to apEntryData + PAP_ENTRY_DATA apEntryData = (PVOID)((ULONG_PTR)HalpLowStub + ((ULONG_PTR)&HalpAPEntryData - (ULONG_PTR)&HalpAPEntry16)); + + *apEntryData = (AP_ENTRY_DATA){ + .Jump32Offset = (ULONG)&HalpAPEntry32, + .Jump32Segment = (ULONG)ProcessorState->ContextFrame.SegCs, + .SelfPtr = (PVOID)apEntryData, + .PageTableRoot = initialCr3, + .ProcessorState = ProcessorState, + .Gdtr = ProcessorState->SpecialRegisters.Gdtr, + }; + + ApicStartApplicationProcessor(HalpStartedProcessorCount, HalpLowStubPhysicalAddress); + + HalpStartedProcessorCount++; + + return TRUE; +} + diff --git a/hal/halx86/smp/ipi.c b/hal/halx86/smp/ipi.c new file mode 100644 index 0000000000000..9051386d23daa --- /dev/null +++ b/hal/halx86/smp/ipi.c @@ -0,0 +1,24 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Source file for Inter-Processor Interrupts management + * COPYRIGHT: Copyright 2021 Justin Miller + */ + +/* INCLUDES ******************************************************************/ + +#include +#include + +#define NDEBUG +#include + +/* GLOBALS *******************************************************************/ + +VOID +NTAPI +HalRequestIpi( + _In_ KAFFINITY TargetProcessors) +{ + HalpRequestIpi(TargetProcessors); +} diff --git a/hal/halx86/smp/smp.c b/hal/halx86/smp/smp.c index f8e2f9e4cdf5b..bcf0d4bef7b75 100644 --- a/hal/halx86/smp/smp.c +++ b/hal/halx86/smp/smp.c @@ -3,12 +3,14 @@ * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) * PURPOSE: Core source file for SMP management * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin */ /* INCLUDES ******************************************************************/ #include #include + #define NDEBUG #include diff --git a/hal/halx86/up.cmake b/hal/halx86/up.cmake index 24b8103cefee7..47378b83a0135 100644 --- a/hal/halx86/up.cmake +++ b/hal/halx86/up.cmake @@ -1,7 +1,8 @@ list(APPEND HAL_UP_SOURCE generic/buildtype.c - generic/spinlock.c) + generic/spinlock.c + generic/up.c) add_library(lib_hal_up OBJECT ${HAL_UP_SOURCE}) add_dependencies(lib_hal_up bugcodes xdk) diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 517661dfd4ccc..734960a206a86 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1558,6 +1558,11 @@ Phase1InitializationDiscard(IN PVOID Context) KeBootTimeBias = 0; } +#ifdef CONFIG_SMP + /* Start Application Processors */ + KeStartAllProcessors(); +#endif + /* Initialize all processors */ if (!HalAllProcessorsStarted()) KeBugCheck(HAL1_INITIALIZATION_FAILED); diff --git a/ntoskrnl/include/internal/amd64/ke.h b/ntoskrnl/include/internal/amd64/ke.h index 8424669063810..b44cffbfd94c8 100644 --- a/ntoskrnl/include/internal/amd64/ke.h +++ b/ntoskrnl/include/internal/amd64/ke.h @@ -476,6 +476,13 @@ KiSetTrapContext( _In_ PCONTEXT Context, _In_ KPROCESSOR_MODE RequestorMode); +VOID +NTAPI +KiInitializePcr(IN PKIPCR Pcr, + IN ULONG ProcessorNumber, + IN PKTHREAD IdleThread, + IN PVOID DpcStack); + #ifdef __cplusplus } // extern "C" #endif diff --git a/ntoskrnl/include/internal/i386/intrin_i.h b/ntoskrnl/include/internal/i386/intrin_i.h index d937057fd0caf..7b354d9182db9 100644 --- a/ntoskrnl/include/internal/i386/intrin_i.h +++ b/ntoskrnl/include/internal/i386/intrin_i.h @@ -5,6 +5,82 @@ extern "C" { #endif +FORCEINLINE +PKGDTENTRY +KiGetGdtEntry( + IN PVOID pGdt, + IN USHORT Selector) +{ + return (PKGDTENTRY)((ULONG_PTR)pGdt + (Selector & ~RPL_MASK)); +} + +FORCEINLINE +VOID +KiSetGdtDescriptorBase( + IN OUT PKGDTENTRY Entry, + IN ULONG32 Base) +{ + Entry->BaseLow = (USHORT)(Base & 0xffff); + Entry->HighWord.Bytes.BaseMid = (UCHAR)((Base >> 16) & 0xff); + Entry->HighWord.Bytes.BaseHi = (UCHAR)((Base >> 24) & 0xff); + // Entry->BaseUpper = (ULONG)(Base >> 32); +} + +FORCEINLINE +VOID +KiSetGdtDescriptorLimit( + _Inout_ PKGDTENTRY Entry, + _In_ ULONG Limit) +{ + if (Limit < 0x100000) + { + Entry->HighWord.Bits.Granularity = 0; + } + else + { + Limit >>= 12; + Entry->HighWord.Bits.Granularity = 1; + } + Entry->LimitLow = (USHORT)(Limit & 0xffff); + Entry->HighWord.Bits.LimitHi = ((Limit >> 16) & 0x0f); +} + +FORCEINLINE +VOID +KiSetGdtEntryEx( + _Inout_ PKGDTENTRY Entry, + _In_ ULONG32 Base, + _In_ ULONG Limit, + _In_ UCHAR Type, + _In_ UCHAR Dpl, + _In_ BOOLEAN Granularity, + _In_ UCHAR SegMode) // 0: 16-bit, 1: 32-bit, 2: 64-bit +{ + KiSetGdtDescriptorBase(Entry, Base); + KiSetGdtDescriptorLimit(Entry, Limit); + Entry->HighWord.Bits.Type = (Type & 0x1f); + Entry->HighWord.Bits.Dpl = (Dpl & 0x3); + Entry->HighWord.Bits.Pres = (Type != 0); // Present, must be 1 when the GDT entry is valid. + Entry->HighWord.Bits.Sys = 0; // System + Entry->HighWord.Bits.Reserved_0 = 0; // LongMode = !!(SegMode & 1); + Entry->HighWord.Bits.Default_Big = !!(SegMode & 2); + Entry->HighWord.Bits.Granularity |= !!Granularity; // The flag may have been already set by KiSetGdtDescriptorLimit(). + // Entry->MustBeZero = 0; +} + +FORCEINLINE +VOID +KiSetGdtEntry( + _Inout_ PKGDTENTRY Entry, + _In_ ULONG32 Base, + _In_ ULONG Limit, + _In_ UCHAR Type, + _In_ UCHAR Dpl, + _In_ UCHAR SegMode) // 0: 16-bit, 1: 32-bit, 2: 64-bit +{ + KiSetGdtEntryEx(Entry, Base, Limit, Type, Dpl, FALSE, SegMode); +} + #if defined(__GNUC__) FORCEINLINE diff --git a/ntoskrnl/include/internal/i386/ke.h b/ntoskrnl/include/internal/i386/ke.h index 30509248a2b9a..f8ddf1349f8e3 100644 --- a/ntoskrnl/include/internal/i386/ke.h +++ b/ntoskrnl/include/internal/i386/ke.h @@ -399,6 +399,17 @@ KiRundownThread(IN PKTHREAD Thread) #endif } +CODE_SEG("INIT") +VOID +NTAPI +KiInitializePcr(IN ULONG ProcessorNumber, + IN PKIPCR Pcr, + IN PKIDTENTRY Idt, + IN PKGDTENTRY Gdt, + IN PKTSS Tss, + IN PKTHREAD IdleThread, + IN PVOID DpcStack); + FORCEINLINE VOID Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) diff --git a/ntoskrnl/include/internal/ke.h b/ntoskrnl/include/internal/ke.h index ae255961388cb..10a352a09811a 100644 --- a/ntoskrnl/include/internal/ke.h +++ b/ntoskrnl/include/internal/ke.h @@ -304,6 +304,13 @@ KiCompleteTimer( IN PKSPIN_LOCK_QUEUE LockQueue ); +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors( + VOID +); + /* gmutex.c ********************************************************************/ VOID diff --git a/ntoskrnl/ke/amd64/mproc.c b/ntoskrnl/ke/amd64/mproc.c new file mode 100644 index 0000000000000..14b6dadf35b3a --- /dev/null +++ b/ntoskrnl/ke/amd64/mproc.c @@ -0,0 +1,23 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Architecture specific source file to hold multiprocessor functions + * COPYRIGHT: Copyright 2022 Justin Miller + */ + +/* INCLUDES *****************************************************************/ + +#include + +#define NDEBUG +#include + +/* FUNCTIONS *****************************************************************/ + +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors(VOID) +{ + UNIMPLEMENTED; +} diff --git a/ntoskrnl/ke/i386/kiinit.c b/ntoskrnl/ke/i386/kiinit.c index 9699d11b03c3a..97b525f5a9d73 100644 --- a/ntoskrnl/ke/i386/kiinit.c +++ b/ntoskrnl/ke/i386/kiinit.c @@ -536,7 +536,7 @@ KiInitializeKernel(IN PKPROCESS InitProcess, else { /* FIXME */ - DPRINT1("SMP Boot support not yet present\n"); + DPRINT1("Starting CPU#%u - you are brave!\n", Number); } /* Setup the Idle Thread */ @@ -824,6 +824,18 @@ KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock) __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu); __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu); + //TODO: We don't setup IPIs yet so freeze other processors here. + if (Cpu) + { + KeMemoryBarrier(); + LoaderBlock->Prcb = 0; + + for (;;) + { + YieldProcessor(); + } + } + KiVerifyCpuFeatures(Pcr->Prcb); /* Initialize the Processor with HAL */ diff --git a/ntoskrnl/ke/i386/mproc.c b/ntoskrnl/ke/i386/mproc.c new file mode 100644 index 0000000000000..01a90d0163610 --- /dev/null +++ b/ntoskrnl/ke/i386/mproc.c @@ -0,0 +1,162 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) + * PURPOSE: Architecture specific source file to hold multiprocessor functions + * COPYRIGHT: Copyright 2021 Justin Miller + * Copyright 2021 Victor Perevertkin + */ + +/* INCLUDES *****************************************************************/ + +#include +#define NDEBUG +#include + +typedef struct _APINFO +{ + DECLSPEC_ALIGN(PAGE_SIZE) KGDTENTRY Gdt[128]; + DECLSPEC_ALIGN(PAGE_SIZE) KIDTENTRY Idt[256]; + DECLSPEC_ALIGN(16) UINT8 NMIStackData[DOUBLE_FAULT_STACK_SIZE]; + KIPCR Pcr; + ETHREAD Thread; + KTSS Tss; + KTSS TssDoubleFault; + KTSS TssNMI; +} APINFO, *PAPINFO; + + +typedef struct _AP_SETUP_STACK +{ + PVOID ReturnAddr; + PVOID KxLoaderBlock; +} AP_SETUP_STACK, *PAP_SETUP_STACK; + +/* FUNCTIONS *****************************************************************/ + +CODE_SEG("INIT") +VOID +NTAPI +KeStartAllProcessors(VOID) +{ + PVOID KernelStack, DPCStack; + SIZE_T ProcessorCount = 0; + PAPINFO APInfo; + + while (TRUE) + { + ProcessorCount++; + KernelStack = NULL; + DPCStack = NULL; + + // Allocate structures for a new CPU. + APInfo = ExAllocatePoolZero(NonPagedPool, sizeof(APINFO), ' eK'); + ASSERT(ALIGN_DOWN_POINTER_BY(APInfo, PAGE_SIZE) == APInfo); + if (!APInfo) + { + break; + } + KernelStack = MmCreateKernelStack(FALSE, 0); + if (!KernelStack) + { + break; + } + DPCStack = MmCreateKernelStack(FALSE, 0); + if (!DPCStack) + { + break; + } + + /* Initalize a new PCR for the specific AP */ + KiInitializePcr(ProcessorCount, + &APInfo->Pcr, + &APInfo->Idt[0], + &APInfo->Gdt[0], + &APInfo->Tss, + (PKTHREAD)&APInfo->Thread, + DPCStack); + + // Prepare descriptor tables + KDESCRIPTOR bspGdt, bspIdt; + __sgdt(&bspGdt.Limit); + __sidt(&bspIdt.Limit); + RtlCopyMemory(&APInfo->Gdt, (PVOID)bspGdt.Base, bspGdt.Limit + 1); + RtlCopyMemory(&APInfo->Idt, (PVOID)bspIdt.Base, bspIdt.Limit + 1); + + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_R0_PCR), (ULONG_PTR)&APInfo->Pcr); + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_DF_TSS), (ULONG_PTR)&APInfo->TssDoubleFault); + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_NMI_TSS), (ULONG_PTR)&APInfo->TssNMI); + + KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS), (ULONG_PTR)&APInfo->Tss); + // Clear TSS Busy flag (aka set the type to "TSS (Available)") + KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS)->HighWord.Bits.Type = 0b1001; + + APInfo->TssDoubleFault.Esp0 = (ULONG_PTR)&APInfo->NMIStackData; + APInfo->TssDoubleFault.Esp = (ULONG_PTR)&APInfo->NMIStackData; + + APInfo->TssNMI.Esp0 = (ULONG_PTR)&APInfo->NMIStackData; + APInfo->TssNMI.Esp = (ULONG_PTR)&APInfo->NMIStackData; + + // Fill the processor state + PKPROCESSOR_STATE ProcessorState = &APInfo->Pcr.Prcb->ProcessorState; + RtlZeroMemory(ProcessorState, sizeof(*ProcessorState)); + + ProcessorState->SpecialRegisters.Cr0 = __readcr0(); + ProcessorState->SpecialRegisters.Cr3 = __readcr3(); + ProcessorState->SpecialRegisters.Cr4 = __readcr4(); + + ProcessorState->ContextFrame.SegCs = KGDT_R0_CODE; + ProcessorState->ContextFrame.SegDs = KGDT_R3_DATA; + ProcessorState->ContextFrame.SegEs = KGDT_R3_DATA; + ProcessorState->ContextFrame.SegSs = KGDT_R0_DATA; + ProcessorState->ContextFrame.SegFs = KGDT_R0_PCR; + + ProcessorState->SpecialRegisters.Gdtr.Base = (ULONG_PTR)APInfo->Gdt; + ProcessorState->SpecialRegisters.Gdtr.Limit = sizeof(APInfo->Gdt) - 1; + ProcessorState->SpecialRegisters.Idtr.Base = (ULONG_PTR)APInfo->Idt; + ProcessorState->SpecialRegisters.Idtr.Limit = sizeof(APInfo->Idt) - 1; + + ProcessorState->SpecialRegisters.Tr = KGDT_TSS; + + ProcessorState->ContextFrame.Esp = (ULONG_PTR)KernelStack; + ProcessorState->ContextFrame.Eip = (ULONG_PTR)KiSystemStartup; + ProcessorState->ContextFrame.EFlags = __readeflags() & ~EFLAGS_INTERRUPT_MASK; + + /* Write KeLoaderBlock into Stack */ + ProcessorState->ContextFrame.Esp = (ULONG)((ULONG_PTR)ProcessorState->ContextFrame.Esp - sizeof(AP_SETUP_STACK)); + PAP_SETUP_STACK ApStack = (PAP_SETUP_STACK)ProcessorState->ContextFrame.Esp; + ApStack->KxLoaderBlock = KeLoaderBlock; + ApStack->ReturnAddr = NULL; + + // Update the LOADER_PARAMETER_BLOCK structure for the new processor + KeLoaderBlock->KernelStack = (ULONG_PTR)KernelStack; + KeLoaderBlock->Prcb = (ULONG_PTR)&APInfo->Pcr.Prcb; + KeLoaderBlock->Thread = (ULONG_PTR)&APInfo->Pcr.Prcb->IdleThread; + + // Start the CPU + DPRINT("Attempting to Start a CPU with number: %u\n", ProcessorCount); + if (!HalStartNextProcessor(KeLoaderBlock, ProcessorState)) + { + break; + } + + // And wait for it to start + while (KeLoaderBlock->Prcb != 0) + { + //TODO: Add a time out so we don't wait forever + KeMemoryBarrier(); + YieldProcessor(); + } + } + + // The last CPU didn't start - clean the data + ProcessorCount--; + + if (APInfo) + ExFreePoolWithTag(APInfo, ' eK'); + if (KernelStack) + MmDeleteKernelStack(KernelStack, FALSE); + if (DPCStack) + MmDeleteKernelStack(DPCStack, FALSE); + + DPRINT1("KeStartAllProcessors: Sucessful AP startup count is %u\n", ProcessorCount); +} diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake index a85cb19e39a99..30ac08b64397c 100644 --- a/ntoskrnl/ntos.cmake +++ b/ntoskrnl/ntos.cmake @@ -333,6 +333,10 @@ if(ARCH STREQUAL "i386") ${REACTOS_SOURCE_DIR}/ntoskrnl/ps/i386/psldt.c ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmmain.c ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmexec.c) + if(BUILD_MP) + list(APPEND SOURCE + ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/i386/mproc.c) + endif() elseif(ARCH STREQUAL "amd64") list(APPEND ASM_SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/boot.S @@ -358,6 +362,10 @@ elseif(ARCH STREQUAL "amd64") ${REACTOS_SOURCE_DIR}/ntoskrnl/ps/amd64/psctx.c ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/stubs.c ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/usercall.c) + if(BUILD_MP) + list(APPEND SOURCE + ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/mproc.c) + endif() elseif(ARCH STREQUAL "arm") list(APPEND ASM_SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/ex/arm/ioport.s diff --git a/sdk/include/asm/ksx.template.h b/sdk/include/asm/ksx.template.h index 019232bb43995..855320bbafa7e 100644 --- a/sdk/include/asm/ksx.template.h +++ b/sdk/include/asm/ksx.template.h @@ -285,7 +285,7 @@ HEADER("Stack sizes"), CONSTANT(KERNEL_STACK_SIZE), /// FIXME: Obsolete CONSTANT(KERNEL_LARGE_STACK_SIZE), CONSTANT(KERNEL_LARGE_STACK_COMMIT), -//CONSTANT(DOUBLE_FAULT_STACK_SIZE), +CONSTANT(DOUBLE_FAULT_STACK_SIZE), #ifdef _M_AMD64 CONSTANT(KERNEL_MCA_EXCEPTION_STACK_SIZE), CONSTANT(NMI_STACK_SIZE), diff --git a/sdk/include/ndk/amd64/ketypes.h b/sdk/include/ndk/amd64/ketypes.h index 81219a73f828a..710c371178807 100644 --- a/sdk/include/ndk/amd64/ketypes.h +++ b/sdk/include/ndk/amd64/ketypes.h @@ -91,6 +91,8 @@ Header Name: //#define KeGetPcr() ((volatile KPCR * const)__readfsdword(0x1C)) //#endif +#define DOUBLE_FAULT_STACK_SIZE 0x2000 + // // CPU Vendors // diff --git a/sdk/include/ndk/arm/ketypes.h b/sdk/include/ndk/arm/ketypes.h index 5bcac60f7d713..44b7e5f326863 100644 --- a/sdk/include/ndk/arm/ketypes.h +++ b/sdk/include/ndk/arm/ketypes.h @@ -305,6 +305,8 @@ typedef enum // #define SYNCH_LEVEL DISPATCH_LEVEL +#define DOUBLE_FAULT_STACK_SIZE 0x3000 + // // Number of pool lookaside lists per pool in the PRCB // diff --git a/sdk/include/ndk/arm64/ketypes.h b/sdk/include/ndk/arm64/ketypes.h index 2536ecc6c74f6..dbb5200875249 100644 --- a/sdk/include/ndk/arm64/ketypes.h +++ b/sdk/include/ndk/arm64/ketypes.h @@ -49,6 +49,8 @@ extern "C" { #define MM_HAL_VA_START 0xFFFFFFFFFFC00000ULL #define MM_HAL_VA_END 0xFFFFFFFFFFFFFFFFULL +#define DOUBLE_FAULT_STACK_SIZE 0x8000 + // // Structure for CPUID info // diff --git a/sdk/include/ndk/i386/ketypes.h b/sdk/include/ndk/i386/ketypes.h index 6eb97d81ba0a4..4d77e015b0228 100644 --- a/sdk/include/ndk/i386/ketypes.h +++ b/sdk/include/ndk/i386/ketypes.h @@ -260,6 +260,11 @@ typedef KIO_ACCESS_MAP *PKIO_ACCESS_MAP; #endif #endif +// +// Double fault stack size +// +#define DOUBLE_FAULT_STACK_SIZE 0x3000 + // // Number of pool lookaside lists per pool in the PRCB //