diff options
Diffstat (limited to 'drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c')
| -rw-r--r-- | drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c | 4509 |
1 files changed, 4509 insertions, 0 deletions
diff --git a/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c new file mode 100644 index 000000000000..324b945d2aa1 --- /dev/null +++ b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c @@ -0,0 +1,4509 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2020 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2020 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_COMMAND + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ + +/******************************************************************************* +** +** _NewQueue +** +** Allocate a new command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL Stalled +** Indicate if hardware is stalled already. +** +** OUTPUT: +** +** gckCOMMAND Command +** gckCOMMAND object has been updated with a new command queue. +*/ +static gceSTATUS +_NewQueue( + IN OUT gckCOMMAND Command, + IN gctBOOL Stalled + ) +{ + gceSTATUS status; + gctINT currentIndex, newIndex; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Switch to the next command buffer. */ + currentIndex = Command->index; + newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; + + /* Wait for availability. */ + gcmkDUMP(Command->os, "#[kernel.waitsignal]"); + + gcmkONERROR(gckOS_WaitSignal( + Command->os, + Command->queues[newIndex].signal, + gcvFALSE, + gcvINFINITE + )); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + if (newIndex < currentIndex) + { + Command->wrapCount += 1; + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 2 * 4, + "%s(%d): queue array wrapped around.\n", + __FUNCTION__, __LINE__ + ); + } + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): total queue wrap arounds %d.\n", + __FUNCTION__, __LINE__, Command->wrapCount + ); + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): switched to queue %d.\n", + __FUNCTION__, __LINE__, newIndex + ); +#endif + + /* Update gckCOMMAND object with new command queue. */ + Command->index = newIndex; + Command->newQueue = gcvTRUE; + Command->videoMem = Command->queues[newIndex].videoMem; + Command->logical = Command->queues[newIndex].logical; + Command->address = Command->queues[newIndex].address; + Command->offset = 0; + + if (currentIndex != -1) + { + if (Stalled) + { + gckOS_Signal( + Command->os, + Command->queues[currentIndex].signal, + gcvTRUE + ); + } + else + { + /* Mark the command queue as available. */ + gcmkONERROR(gckEVENT_Signal( + Command->kernel->eventObj, + Command->queues[currentIndex].signal, + gcvKERNEL_COMMAND + )); + } + } + + /* Success. */ + gcmkFOOTER_ARG("Command->index=%d", Command->index); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_IncrementCommitAtom( + IN gckCOMMAND Command, + IN gctBOOL Increment + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctINT32 atomValue; + gctBOOL powerAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Grab the power mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, hardware->powerMutex, gcvINFINITE + )); + powerAcquired = gcvTRUE; + + /* Increment the commit atom. */ + if (Increment) + { + gcmkONERROR(gckOS_AtomIncrement( + Command->os, Command->atomCommit, &atomValue + )); + } + else + { + gcmkONERROR(gckOS_AtomDecrement( + Command->os, Command->atomCommit, &atomValue + )); + } + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + powerAcquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (powerAcquired) + { + /* Release the power mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CheckFlushMMU( + IN gckCOMMAND Command, + IN gckHARDWARE Hardware + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gctUINT32 oldValue; + gctBOOL pause = gcvFALSE; + + gctUINT8_PTR pointer; + gctUINT32 address; + gctUINT32 eventBytes; + gctUINT32 endBytes; + gctUINT32 bufferSize; + gctUINT32 executeBytes; + + gckOS_AtomicExchange(Command->os, + Hardware->pageTableDirty[gcvENGINE_RENDER], + 0, + &oldValue); + + if (oldValue) + { + /* Page Table is upated, flush mmu before commit. */ + gctUINT32 flushBytes; + + gcmkONERROR(gckHARDWARE_FlushMMU( + Hardware, + gcvNULL, + gcvINVALID_ADDRESS, + 0, + &flushBytes + )); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + flushBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + /* Pointer to reserved address. */ + address = Command->address + Command->offset; + + /* + * subsequent 8 bytes are wait-link commands. + * Set more existed bytes for now. + */ + gcmkONERROR(gckHARDWARE_FlushMMU( + Hardware, + pointer, + address, + (bufferSize - flushBytes), + &flushBytes + )); + + gcmkONERROR(gckCOMMAND_Execute(Command, flushBytes)); + + if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) + && (!Hardware->stallFEPrefetch) + ) + { + pause = gcvTRUE; + } + } + + if (pause) + { + /* Query size. */ + gcmkONERROR(gckWLFE_Event(Hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); + gcmkONERROR(gckWLFE_End(Hardware, gcvNULL, ~0U, &endBytes)); + + executeBytes = eventBytes + endBytes; + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + executeBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + /* Pointer to reserved address. */ + address = Command->address + Command->offset; + + /* Append EVENT(29). */ + gcmkONERROR(gckWLFE_Event( + Hardware, + pointer, + 29, + gcvKERNEL_PIXEL, + &eventBytes + )); + + /* Append END. */ + pointer += eventBytes; + address += eventBytes; + + gcmkONERROR(gckWLFE_End(Hardware, pointer, address, &endBytes)); + + gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +/* WaitLink FE only. */ +static gceSTATUS +_DummyDraw( + IN gckCOMMAND Command + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + + gctUINT8_PTR pointer; + gctUINT32 bufferSize; + + gctUINT32 dummyDrawBytes; + gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) + { + dummyDrawType = gcvDUMMY_DRAW_GC400; + } + + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && + gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) + { + dummyDrawType = gcvDUMMY_DRAW_V60; + } + + if (dummyDrawType != gcvDUMMY_DRAW_INVALID) + { + gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + dummyDrawBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +/* + * Wait pending semaphores. + * + * next == free: full, no more semaphores. + * (free + 1) = next: empty + */ +static gcmINLINE gceSTATUS +_WaitPendingMcfeSema( + gckCOMMAND Command + ) +{ + gceSTATUS status; + const gctUINT count = gcmCOUNTOF(Command->pendingSema); + gctUINT32 nextFreePos; + gctUINT32 timeout = gcvINFINITE; + + gcmkHEADER_ARG("freePendingPos=%u nextPendingPos=%u", + Command->freePendingPos, Command->nextPendingPos); + + nextFreePos = (Command->freePendingPos + 1) % count; + + if (nextFreePos == Command->nextPendingPos) + { + /* No pendings. */ + gcmkONERROR(gcvSTATUS_NOT_FOUND); + } + + while (nextFreePos != Command->nextPendingPos) + { + /* Timeout is infinite in the first to at least free one slot. */ + status = gckOS_WaitSignal(Command->os, + Command->pendingSema[nextFreePos].signal, + gcvFALSE, + timeout); + + if (status == gcvSTATUS_TIMEOUT) + { + /* Timeout out is OK for later pendings. */ + break; + } + + gcmkONERROR(status); + + /* Do not wait for the later slots. */ + timeout = 0; + + /* More free semaphores can be used. */ + Command->freeSemaId = Command->pendingSema[nextFreePos].semaId; + + /* Advance free pos. */ + Command->freePendingPos = nextFreePos; + nextFreePos = (nextFreePos + 1) % count; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gctUINT32 +_GetFreeMcfeSemaNum( + gckCOMMAND Command + ) +{ + gctUINT32 num = 0; + + if (Command->nextSemaId <= Command->freeSemaId) + { + num = Command->freeSemaId - Command->nextSemaId; + } + else + { + num = Command->totalSemaId - Command->nextSemaId + Command->freeSemaId; + } + + return num; +} + +/* + * Get next semaphore id in semaphore ring. + * + * There are semaMinThreshhold semaphores are reserved for system operations, + * such as _SyncToSystemChannel etc. + * The rest semaphores are regular ones. + */ +static gcmINLINE gceSTATUS +_GetNextMcfeSemaId( + gckCOMMAND Command, + gctBOOL regularSema, + gctUINT32 * SemaId + ) +{ + gctUINT32 freeSemaNum = 0; + + gceSTATUS status = gcvSTATUS_OK; + + /* + * See the comments in struct definition. + * wait when run out of semaphores. + */ + freeSemaNum = _GetFreeMcfeSemaNum(Command); + + if ((regularSema && (freeSemaNum <= Command->semaMinThreshhold)) || + (!regularSema && (freeSemaNum == 0))) + { + gcmkONERROR(_WaitPendingMcfeSema(Command)); + } + + gcmkASSERT(Command->nextSemaId != Command->freeSemaId); + + /* Output the semaphore ID. */ + *SemaId = Command->nextSemaId; + + /* Advance to next. */ + if (++Command->nextSemaId == Command->totalSemaId) + { + Command->nextSemaId = 0; + } + +OnError: + return status; +} + +/* + * Get next pending pos in pending semaphore tracking ring. + */ +static gcmINLINE gceSTATUS +_GetNextPendingPos( + gckCOMMAND Command, + gctUINT32 * Pos + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + /* Wait when out of pending ring. */ + if (Command->nextPendingPos == Command->freePendingPos) + { + /* Run out of pending semaphore tracking ring. */ + gcmkONERROR(_WaitPendingMcfeSema(Command)); + } + + gcmkASSERT(Command->nextPendingPos != Command->freePendingPos); + + *Pos = Command->nextPendingPos; + + /* Advance to next. */ + if (++Command->nextPendingPos == gcmCOUNTOF(Command->pendingSema)) + { + Command->nextPendingPos = 0; + } + +OnError: + return status; +} + +/* + * Sync specific channels to system channel. + * Record semaphores to pendingSema structure. + * 'SyncChannel' is cleared upon function return. + */ +static gceSTATUS +_SyncToSystemChannel( + gckCOMMAND Command, + gctUINT64 SyncChannel[2] + ) +{ + gceSTATUS status; + gckKERNEL kernel = Command->kernel; + gckHARDWARE hardware = kernel->hardware; + gctUINT8 semaId[128]; + gctUINT32 semaCount = 0; + gctUINT32 reqBytes = 0; + gctUINT32 bytes = 0; + gctUINT8_PTR buffer; + gctUINT32 i; + gctUINT32 pri; + + /* Ignore system channel. */ + SyncChannel[0] &= ~((gctUINT64)1ull); + SyncChannel[1] &= ~((gctUINT64)1ull); + + if (!SyncChannel[0] && !SyncChannel[1]) + { + return gcvSTATUS_OK; + } + + /* Query SendSemaphore command size. */ + gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); + + for (pri = 0; pri < 2; pri++) + { + for (i = 1; i < 64 && SyncChannel[pri]; i++) + { + gctUINT32 id; + + if (!(SyncChannel[pri] & (1ull << i))) + { + continue; + } + + /* Get a free semaphore id. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + semaId[semaCount++] = (gctUINT8)id; + + gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bytes)); + + /* Send semaphore executed in specified channel. */ + gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, pri, i, reqBytes)); + + /* Remove the sync'ed channel. */ + SyncChannel[pri] &= ~(1ull << i); + } + } + + if (semaCount > 0) + { + gctUINT32 pos = 0; + gckEVENT eventObj = kernel->eventObj; + gctUINT32 bufferLen = 0; + + /* Query WaitSemaphore command size. */ + gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); + reqBytes *= semaCount; + + gcmkONERROR(gckCOMMAND_Reserve(Command, reqBytes, (gctPOINTER *)&buffer, &bufferLen)); + + for (i = 0; i < semaCount; i++) + { + bytes = bufferLen; + + /* Wait semaphores executed in fixed system channel. */ + gckMCFE_WaitSemaphore(hardware, buffer, semaId[i], &bytes); + + buffer += bytes; + bufferLen -= bytes; + } + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); + + /* Now upload the pending semaphore tracking ring. */ + gcmkONERROR(_GetNextPendingPos(Command, &pos)); + + /* Update latest pending semaphore id. */ + Command->pendingSema[pos].semaId = (gctUINT32)semaId[semaCount - 1]; + + /* Send the signal by event. */ + gcmkONERROR(gckEVENT_Signal( + eventObj, + Command->pendingSema[pos].signal, + gcvKERNEL_PIXEL + )); + + gcmkONERROR(gckEVENT_Submit( + eventObj, + gcvTRUE, + gcvFALSE + )); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gcmINLINE gceSTATUS +_SyncFromSystemChannel( + gckCOMMAND Command, + gctBOOL Priority, + gctUINT32 ChannelId + ) +{ + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + gctUINT32 reqBytes = 0; + gctUINT32 bytes = 0; + gctUINT8_PTR buffer; + gctUINT32 id; + + if (!(Command->syncChannel[Priority ? 1 : 0] & (1ull << ChannelId))) + { + /* No need to sync. */ + return gcvSTATUS_OK; + } + + /* Get a semaphore. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + + /* Send the semaphore in system channel. */ + gckMCFE_SendSemaphore(hardware, gcvNULL, 0, &reqBytes); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + gckMCFE_SendSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, gcvFALSE, 0, reqBytes)); + + /* Wait the semaphore in specific channel. */ + gckMCFE_WaitSemaphore(hardware, gcvNULL, 0, &reqBytes); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + gckMCFE_WaitSemaphore(hardware, buffer, id, &bytes); + + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel( + Command, + Priority, + ChannelId, + reqBytes + )); + + /* Clear the sync flag. */ + Command->syncChannel[Priority ? 1 : 0] &= ~(1ull << ChannelId); + + /* Can not track the semaphore here. */ + return gcvSTATUS_OK; + +OnError: + return status; +} + +static gceSTATUS +_CheckFlushMcfeMMU( + IN gckCOMMAND Command, + IN gckHARDWARE Hardware + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctUINT32 oldValue; + gctUINT32 reqBytes; + gctUINT32 bytes; + gctUINT8_PTR buffer; + gctUINT32 id = 0; + + gckOS_AtomicExchange(Command->os, + Hardware->pageTableDirty[gcvENGINE_RENDER], + 0, + &oldValue); + + if (!oldValue) + { + return gcvSTATUS_OK; + } + + /* + * We had sync'ed to system channel in every commit, see comments in Commit. + * It should not run into sync again here, unless there's some other place + * causes channels dirty. Let's check it here. + */ + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + + /* Query flush Mcfe MMU cache command bytes. */ + gcmkONERROR(gckHARDWARE_FlushMcfeMMU(Hardware, gcvNULL, &reqBytes)); + + /* Query semaphore command bytes. */ + gcmkONERROR( + gckMCFE_SendSemaphore(Hardware, gcvNULL, 0, &bytes)); + reqBytes += bytes; + + gcmkONERROR( + gckMCFE_WaitSemaphore(Hardware, gcvNULL, 0, &bytes)); + reqBytes += bytes; + + /* Get a semaphore. */ + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvFALSE, &id)); + + /* Request command buffer for system channel. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + reqBytes, + (gctPOINTER *)&buffer, + &bytes + )); + + /* Do flush mmu. */ + gckHARDWARE_FlushMcfeMMU(Hardware, buffer, &bytes); + buffer += bytes; + + /* Send and wait semaphore in the system channel itself. */ + gcmkONERROR(gckMCFE_SendSemaphore(Hardware, buffer, id, &bytes)); + buffer += bytes; + + gcmkONERROR(gckMCFE_WaitSemaphore(Hardware, buffer, id, &bytes)); + + /* Execute flush mmu and send semaphores. */ + gcmkONERROR(gckCOMMAND_ExecuteMultiChannel(Command, 0, 0, reqBytes)); + + /* Need sync from system channel. */ + Command->syncChannel[0] = ~1ull; + Command->syncChannel[1] = ~1ull; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/* + * Find sema id from the map. + * Returns semaphore Id, ie the array index of semaHandleMap. + * -1 if not found. + */ +static gcmINLINE gctINT32 +_FindSemaIdFromMap( + IN gckCOMMAND Command, + IN gctUINT32 SemaHandle + ) +{ + gctUINT32 semaId = Command->nextSemaId; + + do + { + /* + * Only need to check semaId between signaledId (inclusive) and + * nextSemaId (exclusive). + */ + semaId = (semaId == 0) ? (Command->totalSemaId - 1) : (semaId - 1); + + if (Command->semaHandleMap[semaId] == SemaHandle) + { + return (gctINT32)semaId; + } + } + while (semaId != Command->freeSemaId); + + return -1; +} + +/* Put together patch list handling variables. */ +typedef struct _gcsPATCH_LIST_VARIABLE +{ + /* gcvHAL_PATCH_VIDMEM_TIMESTAMP. */ + gctUINT64 maxAsyncTimestamp; + + /* gcvHAL_PATCH_MCFE_SEMAPHORE. */ + gctBOOL semaUsed; +} +gcsPATCH_LIST_VARIABLE; + +/* Patch item handler typedef. */ +typedef gceSTATUS +(* PATCH_ITEM_HANDLER)( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ); + +static const gctUINT32 _PatchItemSize[] = +{ + 0, + (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_ADDRESS), + (gctUINT32)sizeof(gcsHAL_PATCH_MCFE_SEMAPHORE), + (gctUINT32)sizeof(gcsHAL_PATCH_VIDMEM_TIMESTAMP), +}; + +static gceSTATUS +_HandleVidmemAddressPatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gcsHAL_PATCH_VIDMEM_ADDRESS * patch = Patch; + + gcmkHEADER_ARG("Command=%p location=0x%x node=0x%x offset=%x", + Command, patch->location, patch->node, patch->offset); + + (void)status; + (void)patch; + + gcmkFOOTER(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_HandleMCFESemaphorePatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gckHARDWARE hardware = Command->kernel->hardware; + gctINT32 index; + gctUINT32 semaId; + gceSTATUS status; + gctUINT32 bytes = 8; + gctUINT32 buffer[2]; + gctUINT8_PTR location; + gcsHAL_PATCH_MCFE_SEMAPHORE * patch = (gcsHAL_PATCH_MCFE_SEMAPHORE *)Patch; + + gcmkHEADER_ARG("Command=%p location=0x%x semaHandle=%d", + Command, patch->location, patch->semaHandle); + + index = _FindSemaIdFromMap(Command, patch->semaHandle); + + if (index < 0) + { + status = _GetNextMcfeSemaId(Command, gcvTRUE, &semaId); + + if (gcmIS_ERROR(status)) + { + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + gcmkONERROR(_GetNextMcfeSemaId(Command, gcvTRUE, &semaId)); + } + + Command->semaHandleMap[semaId] = patch->semaHandle; + } + else + { + semaId = (gctUINT32)index; + + /* One send must match one wait, will assign new id next time. */ + Command->semaHandleMap[semaId] = 0; + } + + if (patch->sendSema) + { + gcmkONERROR(gckMCFE_SendSemaphore(hardware, buffer, semaId, &bytes)); + } + else + { + gcmkONERROR(gckMCFE_WaitSemaphore(hardware, buffer, semaId, &bytes)); + } + + gcmkASSERT(bytes == 8); + + location = gcmUINT64_TO_PTR(CommandBuffer->logical + patch->location); + + /* Patch the command buffer. */ + gckOS_WriteMemory(Command->os, location, buffer[0]); + gckOS_WriteMemory(Command->os, location + 4, buffer[1]); + + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_HandleTimestampPatch( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gctPOINTER Patch, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + gctUINT32 processID; + gckVIDMEM_NODE videoMem = gcvNULL; + gcsHAL_PATCH_VIDMEM_TIMESTAMP * patch = Patch; + gceENGINE engine = Command->feType == gcvHW_FE_ASYNC ? gcvENGINE_BLT + : gcvENGINE_RENDER; + + gcmkHEADER_ARG("Command=%p node=0x%x", Command, patch->handle); + + /* Get the current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + gcmkONERROR( + gckVIDMEM_HANDLE_Lookup(Command->kernel, + processID, + patch->handle, + &videoMem)); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Reference(Command->kernel, videoMem)); + + /* Touch video memory node. */ + gcmkVERIFY_OK( + gckVIDMEM_NODE_SetCommitStamp(Command->kernel, + engine, + videoMem, + Command->commitStamp)); + + if ((engine == gcvENGINE_RENDER) && Command->kernel->asyncCommand) + { + /* Find the latest timestamp of the nodes used in async FE. */ + gctUINT64 stamp = 0; + + /* Get stamp touched async command buffer. */ + gcmkVERIFY_OK( + gckVIDMEM_NODE_GetCommitStamp(Command->kernel, + gcvENGINE_BLT, + videoMem, + &stamp)); + + /* Find latest one. */ + if (PatchListVar->maxAsyncTimestamp < stamp) + { + PatchListVar->maxAsyncTimestamp = stamp; + } + } + +OnError: + if (videoMem) + { + gckVIDMEM_NODE_Dereference(Command->kernel, videoMem); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_HandlePatchListSingle( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gcsHAL_PATCH_LIST * PatchList, + IN gctBOOL NeedCopy, + IN gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + /* 256 bytes for storage. */ + gctUINT64 storage[32]; + gctPOINTER kArray = gcvNULL; + gctPOINTER userPtr = gcvNULL; + gctUINT32 index = 0; + gctUINT32 count = 0; + gctUINT32 itemSize = 0; + gctUINT32 batchCount = 0; + + static const PATCH_ITEM_HANDLER patchHandler[] = + { + gcvNULL, + _HandleVidmemAddressPatch, + _HandleMCFESemaphorePatch, + _HandleTimestampPatch, + }; + PATCH_ITEM_HANDLER handler; + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p PatchList=%p type=%d", + Command, CommandBuffer, PatchList, PatchList->type); + + if (PatchList->type >= gcmCOUNTOF(_PatchItemSize) || PatchList->type >= gcmCOUNTOF(patchHandler)) + { + /* Exceeds buffer max size. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + itemSize = _PatchItemSize[PatchList->type]; + + batchCount = (gctUINT32)(sizeof(storage) / itemSize); + + handler = patchHandler[PatchList->type]; + + while (index < PatchList->count) + { + gctUINT i; + gctUINT8_PTR ptr; + + /* Determine batch count, don't handle too many in one batch. */ + count = PatchList->count - index; + + if (count > batchCount) + { + count = batchCount; + } + + userPtr = gcmUINT64_TO_PTR(PatchList->patchArray + itemSize * index); + + /* Copy/map a patch array batch from user. */ + if (NeedCopy) + { + kArray = storage; + + status = gckOS_CopyFromUserData( + Command->os, + kArray, + userPtr, + itemSize * count + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + itemSize * count, + (gctPOINTER *)&kArray + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + gcmkONERROR(status); + } + + /* Advance to next batch. */ + index += count; + + ptr = (gctUINT8_PTR)kArray; + + for (i = 0; i < count; i++) + { + /* Call handler. */ + gcmkONERROR( + handler(Command, CommandBuffer, ptr, PatchListVar)); + + /* Advance to next patch. */ + ptr += itemSize; + } + + /* Unmap user pointer if mapped. */ + if (!NeedCopy) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + itemSize * count, + kArray + )); + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!NeedCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + itemSize * count, + kArray + )); + + userPtr = gcvNULL; + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_HandlePatchList( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + OUT gcsPATCH_LIST_VARIABLE * PatchListVar + ) +{ + gceSTATUS status; + gctBOOL needCopy = gcvFALSE; + gcsHAL_PATCH_LIST storage; + gcsHAL_PATCH_LIST * kPatchList = gcvNULL; + gctPOINTER userPtr = gcmUINT64_TO_PTR(CommandBuffer->patchHead); + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p", Command, CommandBuffer); + + /* Check wehther we need to copy the structures or not. */ + gcmkONERROR(gckOS_QueryNeedCopy(Command->os, 0, &needCopy)); + + while (userPtr) + { + gctUINT64 next; + + /* Copy/map a patch from user. */ + if (needCopy) + { + kPatchList = &storage; + + status = gckOS_CopyFromUserData( + Command->os, + kPatchList, + userPtr, + sizeof(gcsHAL_PATCH_LIST) + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + (gctPOINTER *)&kPatchList + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + gcmkONERROR(status); + } + + /* Do handle patch. */ + gcmkASSERT(kPatchList->type < gcvHAL_PATCH_TYPE_COUNT); + + gcmkONERROR( + _HandlePatchListSingle(Command, + CommandBuffer, + kPatchList, + needCopy, + PatchListVar)); + + next = kPatchList->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + kPatchList + )); + } + + /* Advance to next patch from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + sizeof(gcsHAL_PATCH_LIST), + kPatchList + )); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_WaitForAsyncCommandStamp( + IN gckCOMMAND Command, + IN gctUINT64 Stamp + ) +{ + gctUINT32 bytes; + gceSTATUS status; + gctUINT32 fenceAddress; + gctUINT32 bufferSize; + gctPOINTER pointer; + gckCOMMAND asyncCommand = Command->kernel->asyncCommand; + + gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); + + if (*(gctUINT64 *)asyncCommand->fence->logical >= Stamp) + { + /* Already satisfied, skip. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + fenceAddress = asyncCommand->fence->address; + + gcmkONERROR(gckHARDWARE_WaitFence( + Command->kernel->hardware, + gcvNULL, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + bytes, + &pointer, + &bufferSize + )); + + gcmkONERROR(gckHARDWARE_WaitFence( + Command->kernel->hardware, + pointer, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************\ +****************************** gckCOMMAND API Code ****************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckCOMMAND_Construct +** +** Construct a new gckCOMMAND object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckCOMMAND * Command +** Pointer to a variable that will hold the pointer to the gckCOMMAND +** object. +*/ +gceSTATUS +gckCOMMAND_Construct( + IN gckKERNEL Kernel, + IN gceHW_FE_TYPE FeType, + OUT gckCOMMAND * Command + ) +{ + gckOS os; + gckCOMMAND command = gcvNULL; + gceSTATUS status; + gctINT i; + gctPOINTER pointer = gcvNULL; + gctSIZE_T pageSize; + + gcmkHEADER_ARG("Kernel=%p", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Command != gcvNULL); + + /* Extract the gckOS object. */ + os = Kernel->os; + + /* Allocate the gckCOMMAND structure. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); + command = pointer; + + /* Reset the entire object. */ + gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); + + /* Initialize the gckCOMMAND object.*/ + command->object.type = gcvOBJ_COMMAND; + command->kernel = Kernel; + command->os = os; + + command->feType = FeType; + + /* Get the command buffer requirements. */ + gcmkONERROR(gckHARDWARE_QueryCommandBuffer( + Kernel->hardware, + gcvENGINE_RENDER, + &command->alignment, + gcvNULL, + gcvNULL + )); + + /* Create the command queue mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); + + /* Create the power management semaphore. */ + gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); + + /* Create the commit atom. */ + gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); + + /* Get the page size from teh OS. */ + gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); + + gcmkSAFECASTSIZET(command->pageSize, pageSize); + + /* Get process ID. */ + gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); + + /* Set hardware to pipe 0. */ + command->pipeSelect = gcvPIPE_INVALID; + + /* Pre-allocate the command queues. */ + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { +#if !gcdCAPTURE_ONLY_MODE + gcePOOL pool = gcvPOOL_DEFAULT; +#else + gcePOOL pool = gcvPOOL_VIRTUAL; +#endif + + gctSIZE_T size = pageSize; + gckVIDMEM_NODE videoMem = gcvNULL; + gctUINT32 allocFlag = 0; + +#if gcdENABLE_CACHEABLE_COMMAND_BUFFER + allocFlag = gcvALLOC_FLAG_CACHEABLE; +#endif + + /* Allocate video memory node for command buffers. */ + gcmkONERROR(gckKERNEL_AllocateVideoMemory( + Kernel, + 64, + gcvVIDMEM_TYPE_COMMAND, + allocFlag, + &size, + &pool, + &videoMem + )); + + command->queues[i].videoMem = videoMem; + + /* Lock for GPU access. */ + gcmkONERROR(gckVIDMEM_NODE_Lock( + Kernel, + videoMem, + &command->queues[i].address + )); + + /* Lock for kernel side CPU access. */ + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Kernel, + videoMem, + gcvFALSE, + gcvFALSE, + &command->queues[i].logical + )); + + gcmkONERROR(gckOS_CreateSignal( + os, gcvFALSE, &command->queues[i].signal + )); + + gcmkONERROR(gckOS_Signal( + os, command->queues[i].signal, gcvTRUE + )); + } + +#if gcdRECORD_COMMAND + gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); +#endif + + gcmkONERROR(gckFENCE_Create( + os, Kernel, &command->fence + )); + + /* No command queue in use yet. */ + command->index = -1; + command->logical = gcvNULL; + command->newQueue = gcvFALSE; + + /* Query mcfe semaphore count. */ + if (FeType == gcvHW_FE_MULTI_CHANNEL) + { + command->totalSemaId = 128; + + /* Empty sema id ring. */ + command->nextSemaId = 0; + command->freeSemaId = command->totalSemaId - 1; + + command->semaMinThreshhold = 16; + + /* Create signals. */ + for (i = 0; i < (gctINT)gcmCOUNTOF(command->pendingSema); i++) + { + gcmkONERROR(gckOS_CreateSignal( + os, + gcvFALSE, + &command->pendingSema[i].signal + )); + } + + /* Empty pending sema tracking ring. */ + command->nextPendingPos = 0; + command->freePendingPos = gcmCOUNTOF(command->pendingSema) - 1; + + /* Allocate sema handle mapping. */ + gcmkONERROR(gckOS_Allocate( + os, + command->totalSemaId * sizeof(gctUINT32), + &pointer + )); + + command->semaHandleMap = (gctUINT32 *)pointer; + + gcmkVERIFY_OK(gckOS_ZeroMemory( + command->semaHandleMap, + command->totalSemaId * sizeof(gctUINT32) + )); + } + + /* Command is not yet running. */ + command->running = gcvFALSE; + + /* Command queue is idle. */ + command->idle = gcvTRUE; + + /* Commit stamp start from 1. */ + command->commitStamp = 1; + + command->dummyDraw = gcvTRUE; + + /* Return pointer to the gckCOMMAND object. */ + *Command = command; + + /* Success. */ + gcmkFOOTER_ARG("*Command=0x%x", *Command); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (command != gcvNULL) + { + gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Destroy +** +** Destroy an gckCOMMAND object. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Destroy( + IN gckCOMMAND Command + ) +{ + gctINT i; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Stop the command queue. */ + gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); + + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { + if (Command->queues[i].signal) + { + gcmkVERIFY_OK(gckOS_DestroySignal( + Command->os, Command->queues[i].signal + )); + } + + if (Command->queues[i].logical) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Command->kernel, + Command->queues[i].videoMem, + 0, + gcvFALSE, + gcvFALSE + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + Command->kernel, + Command->queues[i].videoMem, + 0, + gcvNULL + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_Dereference( + Command->kernel, + Command->queues[i].videoMem + )); + + Command->queues[i].videoMem = gcvNULL; + Command->queues[i].logical = gcvNULL; + } + } + + if (Command->mutexContext) + { + /* Delete the context switching mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); + } + + if (Command->mutexContextSeq != gcvNULL) + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); + + if (Command->mutexQueue) + { + /* Delete the command queue mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); + } + + if (Command->powerSemaphore) + { + /* Destroy the power management semaphore. */ + gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); + } + + if (Command->atomCommit) + { + /* Destroy the commit atom. */ + gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); + } + +#if gcdRECORD_COMMAND + gckRECORDER_Destory(Command->os, Command->recorder); +#endif + + if (Command->stateMap) + { + gcmkOS_SAFE_FREE(Command->os, Command->stateMap); + } + + if (Command->semaHandleMap) + { + gcmkOS_SAFE_FREE(Command->os, Command->semaHandleMap); + } + + if (Command->fence) + { + gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); + } + + /* Mark object as unknown. */ + Command->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckCOMMAND object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckCOMMAND_EnterCommit +** +** Acquire command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_EnterCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctBOOL atomIncremented = gcvFALSE; + gctBOOL semaAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (!FromPower) + { + /* Increment COMMIT atom to let power management know that a commit is + ** in progress. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); + atomIncremented = gcvTRUE; + + /* Notify the system the GPU has a commit. */ + gcmkONERROR(gckOS_Broadcast(Command->os, + hardware, + gcvBROADCAST_GPU_COMMIT)); + + /* Acquire the power management semaphore. */ + gcmkONERROR(gckOS_AcquireSemaphore(Command->os, + Command->powerSemaphore)); + semaAcquired = gcvTRUE; + } + + /* Grab the conmmand queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Command->os, + Command->mutexQueue, + gcvINFINITE)); + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (semaAcquired) + { + /* Release the power management semaphore. */ + gcmkVERIFY_OK(gckOS_ReleaseSemaphore( + Command->os, Command->powerSemaphore + )); + } + + if (atomIncremented) + { + /* Decrement the commit atom. */ + gcmkVERIFY_OK(_IncrementCommitAtom( + Command, gcvFALSE + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_ExitCommit +** +** Release command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_ExitCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); + + if (!FromPower) + { + /* Release the power management semaphore. */ + gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, + Command->powerSemaphore)); + + /* Decrement the commit atom. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); + } + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StartWaitLinkFE( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctUINT32 waitOffset = 0; + gctUINT32 waitLinkBytes; + gctPOINTER logical; + gctUINT32 address; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (Command->running) + { + /* Command queue already running. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + gcvNULL, + ~0U, + Command->offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + if ((Command->pageSize - Command->offset < waitLinkBytes) + || (Command->logical == gcvNULL) + ) + { + /* Start at beginning of a new queue. */ + gcmkONERROR(_NewQueue(Command, gcvTRUE)); + } + + logical = (gctUINT8_PTR) Command->logical + Command->offset; + address = Command->address + Command->offset; + + /* Append WAIT/LINK. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + logical, + address, + 0, + &waitLinkBytes, + &waitOffset, + &Command->waitPos.size + )); + + /* Update wait command position. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = Command->offset + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR) logical + waitOffset; + Command->waitPos.address = address + waitOffset; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + logical, + waitLinkBytes + )); + + /* Adjust offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + + gcmkDUMP(Command->os, "#[wait-link: fe start]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + logical, + address, + waitLinkBytes + ); + +#if gcdSECURITY + /* Start FE by calling security service. */ + gckKERNEL_SecurityStartCommand( + Command->kernel + ); +#else + +#if !gcdCAPTURE_ONLY_MODE + /* Enable command processor. */ + gcmkONERROR(gckWLFE_Execute( + hardware, + address, + waitLinkBytes + )); +#endif + +#endif + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StartAsyncFE( + IN gckCOMMAND Command + ) +{ + if ((Command->pageSize <= Command->offset) || + (Command->logical == gcvNULL)) + { + /* Start at beginning of a new queue. */ + gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); + } + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Nothing to do. */ + return gcvSTATUS_OK; +} + +static gceSTATUS +_StartMCFE( + IN gckCOMMAND Command + ) +{ + if ((Command->pageSize <= Command->offset) || + (Command->logical == gcvNULL)) + { + /* Start at beginning of a new queue. */ + gcmkVERIFY_OK(_NewQueue(Command, gcvTRUE)); + } + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Nothing to do. */ + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckCOMMAND_Start +** +** Start up the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to start. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Start( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Command=%p", Command); + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + gcmkONERROR(_StartWaitLinkFE(Command)); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + gcmkONERROR(_StartMCFE(Command)); + } + else + { + gcmkONERROR(_StartAsyncFE(Command)); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; + +} + +static gceSTATUS +_StopWaitLinkFE( + IN gckCOMMAND Command + ) +{ + gckHARDWARE hardware; + gceSTATUS status; + gctUINT32 idle; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Replace last WAIT with END. */ + gcmkONERROR(gckWLFE_End( + hardware, + Command->waitPos.logical, + Command->waitPos.address, + &Command->waitPos.size + )); + + gcmkDUMP(Command->os, "#[end: fe stop]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + +#if gcdSECURITY + gcmkONERROR(gckKERNEL_SecurityExecute( + Command->kernel, Command->waitPos.logical, 8 + )); +#endif + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, + Command->logical, + Command->offset)); + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + + /* Wait for idle. */ + gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StopAsyncFE( + IN gckCOMMAND Command + ) +{ + gckHARDWARE hardware; + gceSTATUS status; + gctUINT32 idle; + + gcmkHEADER_ARG("Command=%p", Command); + + hardware = Command->kernel->hardware; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, + Command->logical, + Command->offset)); + + /* Wait for idle. */ + gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_StopMCFE( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + + gcmkHEADER_ARG("Command=%p", Command); + + hardware = Command->kernel->hardware; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(hardware, + Command->logical, + Command->offset)); + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stop +** +** Stop the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to stop. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stop( + IN gckCOMMAND Command + ) +{ + if (!Command->running) + { + /* Command queue is not running. */ + return gcvSTATUS_OK; + } + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + return _StopWaitLinkFE(Command); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + return _StopMCFE(Command); + } + else + { + return _StopAsyncFE(Command); + } +} + +static gceSTATUS +_CommitWaitLinkOnce( + IN gckCOMMAND Command, + IN gckCONTEXT Context, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer, + IN gcsSTATE_DELTA_PTR StateDelta, + IN gctUINT32 ProcessID, + IN gctBOOL Shared, + INOUT gctBOOL *contextSwitched + ) +{ + gceSTATUS status; + gctBOOL commitEntered = gcvFALSE; + gctBOOL contextAcquired = gcvFALSE; + gckHARDWARE hardware; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcsCONTEXT_PTR contextBuffer; + gctUINT8_PTR commandBufferLogical = gcvNULL; + gctUINT32 commandBufferAddress = 0; + gckVIDMEM_NODE commandBufferVideoMem = gcvNULL; + gctUINT8_PTR commandBufferTail = gcvNULL; + gctUINT commandBufferSize; + gctUINT32 linkBytes; + gctSIZE_T bytes; + gctUINT32 offset; + gctPOINTER entryLogical; + gctUINT32 entryAddress; + gctUINT32 entryBytes; + gctUINT32 exitAddress; + gctUINT32 exitBytes; + gctPOINTER waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkBytes; + gctUINT32 waitOffset; + gctUINT32 waitSize; + +#if gcdCAPTURE_ONLY_MODE + gctINT i; +#endif + +#ifdef __QNXNTO__ + gctPOINTER userCommandBufferLogical = gcvNULL; + gctBOOL userCommandBufferLogicalMapped = gcvFALSE; +#endif + +#if gcdDUMP_IN_KERNEL + gctPOINTER contextDumpLogical = gcvNULL; +# endif + gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; + gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; + gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; + + gcmkHEADER_ARG("Command=%p CommandBuffer=%p ProcessID=%d", + Command, CommandBuffer, ProcessID + ); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + gcmkASSERT(Command->feType == gcvHW_FE_WAIT_LINK); + + /* Acquire the command queue. */ + gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); + commitEntered = gcvTRUE; + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + contextAcquired = gcvTRUE; + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + + + gcmkONERROR( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + /* Query the size of LINK command. */ + gcmkONERROR(gckWLFE_Link( + hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL + )); + + /* Compute the command buffer entry and the size. */ + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset; + +#ifdef __QNXNTO__ + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Command->kernel, + ProcessID, + CommandBuffer->videoMemNode, + &commandBufferVideoMem + )); + + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + Command->kernel, + commandBufferVideoMem, + gcvFALSE, + gcvFALSE, + &userCommandBufferLogical + )); + + commandBufferLogical = (gctUINT8_PTR)userCommandBufferLogical + CommandBuffer->startOffset; + userCommandBufferLogicalMapped =gcvTRUE; +#endif + + commandBufferSize = CommandBuffer->size; + + gcmkONERROR(_CheckFlushMMU(Command, hardware)); + + if (Command->dummyDraw == gcvTRUE && + Context != gcvNULL) + { + Command->dummyDraw = gcvFALSE; + gcmkONERROR(_DummyDraw(Command)); + } + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && + Command->kernel->asyncCommand && + patchListVar.maxAsyncTimestamp != 0) + { + gcmkONERROR(_WaitForAsyncCommandStamp( + Command, + patchListVar.maxAsyncTimestamp + )); + } + + /* Get the current offset. */ + offset = Command->offset; + + /* Compute number of bytes left in current kernel command queue. */ + bytes = Command->pageSize - offset; + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + gcvNULL, + ~0U, + offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + /* Is there enough space in the current command queue? */ + if (bytes < waitLinkBytes) + { + /* No, create a new one. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Get the new current offset. */ + offset = Command->offset; + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - offset; + gcmkASSERT(bytes >= waitLinkBytes); + } + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; + waitLinkAddress = Command->address + offset; + + /* Context switch required? */ + if (Context == gcvNULL) + { + /* See if we have to switch pipes for the command buffer. */ + if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Compute the entry. */ + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + + Command->currContext = gcvNULL; + } +#if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE + else if (1) +#else + else if (Command->currContext != Context) +#endif + { + /* Get the current context buffer. */ + contextBuffer = Context->buffer; + + /* Yes, merge in the deltas. */ + gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); + + /*************************************************************** + ** SWITCHING CONTEXT. + */ + + /* Determine context buffer entry offset. */ + offset = (Command->pipeSelect == gcvPIPE_3D) + + /* Skip pipe switching sequence. */ + ? Context->entryOffset3D + Context->pipeSelectBytes + + /* Do not skip pipe switching sequence. */ + : Context->entryOffset3D; + + /* Compute the entry. */ + entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; + entryAddress = contextBuffer->address + offset; + entryBytes = Context->bufferSize - offset; + + /* See if we have to switch pipes between the context + and command buffers. */ + if (CommandBuffer->entryPipe == gcvPIPE_3D) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the initial context pipes are + different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Generate a LINK from the context buffer to + the command buffer. */ + gcmkONERROR(gckWLFE_Link( + hardware, + contextBuffer->link3D, + commandBufferAddress + offset, + commandBufferSize - offset, + &linkBytes, + &commandLinkLow, + &commandLinkHigh + )); + +#if gcdCAPTURE_ONLY_MODE + for (i = 0; i < gcdCONTEXT_BUFFER_COUNT; ++i) + { + gcsCONTEXT_PTR buffer = contextBuffer; + + gckOS_CopyToUserData(Command->os, buffer->logical, CommandBuffer->contextLogical[i], Context->bufferSize); + + buffer = buffer->next; + } +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + contextBuffer->videoMem, + entryAddress - contextBuffer->address, + entryLogical, + entryBytes + )); + + /* Update the current context. */ + Command->currContext = Context; + + if (contextSwitched) + { + *contextSwitched = gcvTRUE; + } + +#if gcdDUMP_IN_KERNEL + contextDumpLogical = entryLogical; +#endif + +#if gcdSECURITY + /* Commit context buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + entryLogical, + entryBytes - 8 + ); +#endif + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + gcvNULL, + 0xFFFFFFFF, + entryLogical, + entryBytes + ); +#endif + } + + /* Same context. */ + else + { + /* See if we have to switch pipes for the command buffer. */ + if (CommandBuffer->entryPipe == (gctUINT32)(Command->pipeSelect)) + { + /* Skip reserved head bytes. */ + offset = CommandBuffer->reservedHead; + } + else + { + gctUINT32 pipeBytes = CommandBuffer->reservedHead; + + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + CommandBuffer->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + + /* Reserved bytes in userspace must be exact for a pipeSelect. */ + gcmkASSERT(pipeBytes == CommandBuffer->reservedHead); + } + + /* Compute the entry. */ + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + } + + (void)entryLogical; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + /* Some extra commands (at beginning) are required for new queue. */ + exitAddress = Command->address; + exitBytes = Command->offset + waitLinkBytes; + } + else + { + /* Still within the preexisting command queue, jump to the new + WAIT/LINK command sequence. */ + exitAddress = waitLinkAddress; + exitBytes = waitLinkBytes; + } + + /* Add a new WAIT/LINK command sequence. When the command buffer which is + currently being scheduled is fully executed by the GPU, the FE will + jump to this WAIT/LINK sequence. */ + gcmkONERROR(gckWLFE_WaitLink( + hardware, + waitLinkLogical, + waitLinkAddress, + offset, + &waitLinkBytes, + &waitOffset, + &waitSize + )); + + if (Command->newQueue) + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + Command->logical, + exitBytes + )); + } + else + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + waitLinkLogical, + exitBytes + )); + } + + /* Determine the location of the TAIL in the command buffer. */ + commandBufferTail + = commandBufferLogical + + commandBufferSize + - CommandBuffer->reservedTail; + + /* Generate command which writes out commit stamp. */ + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) + { + gctUINT32 bytes; + + gcmkONERROR(gckHARDWARE_Fence( + hardware, + gcvENGINE_RENDER, + commandBufferTail, + Command->fence->address, + Command->commitStamp, + &bytes + )); + + commandBufferTail += bytes; + } + + /* Generate a LINK from the end of the command buffer being scheduled + back to the kernel command queue. */ +#if !gcdSECURITY + if (Shared == gcvFALSE) + { + gcmkONERROR(gckWLFE_Link( + hardware, + commandBufferTail, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + } + else + { + gctUINT8_PTR link = commandBufferTail + CommandBuffer->exitIndex * 16; + gctSIZE_T bytes = 8; + + gcmkONERROR(gckWLFE_ChipEnable( + hardware, + link, + (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), + &bytes + )); + + link += bytes; + + gcmkONERROR(gckWLFE_Link( + hardware, + link, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + + link += linkBytes; + } +#endif + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Command->kernel, + ProcessID, + CommandBuffer->videoMemNode, + &commandBufferVideoMem + )); + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + commandBufferVideoMem, + CommandBuffer->startOffset, + commandBufferLogical, + commandBufferSize + )); + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + commandBufferLogical + offset, + commandBufferSize - offset, + gcvNULL, + 0xFFFFFFFF + ); + + gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); +#endif + +#if gcdSECURITY + /* Submit command buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + commandBufferLogical + offset, + commandBufferSize - offset - 8 + ); +#else +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* + * Skip link to entryAddress. + * Instead, we directly link to final wait link position. + */ + gcmkONERROR(gckWLFE_Link( + hardware, + Command->waitPos.logical, + waitLinkAddress, + waitLinkBytes, + &Command->waitPos.size, + &entryLinkLow, + &entryLinkHigh + )); +# else + /* Generate a LINK from the previous WAIT/LINK command sequence to the + entry determined above (either the context or the command buffer). + This LINK replaces the WAIT instruction from the previous WAIT/LINK + pair, therefore we use WAIT metrics for generation of this LINK. + This action will execute the entire sequence. */ + gcmkONERROR(gckWLFE_Link( + hardware, + Command->waitPos.logical, + entryAddress, + entryBytes, + &Command->waitPos.size, + &entryLinkLow, + &entryLinkHigh + )); +# endif +#endif + +#if gcdLINK_QUEUE_SIZE + /* TODO: What's it? */ + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = entryAddress; + data.linkData.end = entryAddress + entryBytes; + data.linkData.linkLow = entryLinkLow; + data.linkData.linkHigh = entryLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + + if (commandBufferAddress + offset != entryAddress) + { + data.linkData.start = commandBufferAddress + offset; + data.linkData.end = commandBufferAddress + commandBufferSize; + data.linkData.linkLow = commandLinkLow; + data.linkData.linkHigh = commandLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + data.linkData.start = exitAddress; + data.linkData.end = exitAddress + exitBytes; + data.linkData.linkLow = exitLinkLow; + data.linkData.linkHigh = exitLinkHigh; + + /* Dump kernel command.*/ + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + } +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + + if (entryAddress != commandBufferAddress + offset) + { + gcmkDUMP(Command->os, "#[context]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_CONTEXT, + contextDumpLogical, + entryAddress, + entryBytes + ); + + /* execute context. */ + gcmkDUMP(Command->os, + "@[execute 0 0 0x%08X 0x%08X]", + entryAddress + offset, + entryBytes - offset - 8 + ); + } + + gcmkDUMP(Command->os, "#[command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_COMMAND, + commandBufferLogical + offset, + commandBufferAddress + offset, + commandBufferSize - offset + ); + + /* execute user commands. */ + gcmkDUMP( + Command->os, + "@[execute 0 0 0x%08X 0x%08X]", + commandBufferAddress + + CommandBuffer->reservedHead, + commandBufferSize + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + ); + + gcmkDUMP(Command->os, "#[wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + waitLinkLogical, + waitLinkAddress, + waitLinkBytes + ); + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + gcmkDUMP( + Command->os, + "#[null driver: below command skipped link to 0x%08X 0x%08X]", + entryAddress, + entryBytes + ); +#endif + + gcmkDUMP(Command->os, "#[link: break prev wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + + /* Update the current pipe. */ + Command->pipeSelect = CommandBuffer->exitPipe; + + /* Update command queue offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update address of last WAIT. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = Command->offset - waitLinkBytes + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; + Command->waitPos.address = waitLinkAddress + waitOffset; + Command->waitPos.size = waitSize; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + hardware, Command->logical, Command->offset + )); + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + contextAcquired = gcvFALSE; + + /* Release the command queue. */ + gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + commitEntered = gcvFALSE; + + if (status == gcvSTATUS_INTERRUPTED) + { + gcmkTRACE( + gcvLEVEL_INFO, + "%s(%d): Intterupted in gckEVENT_Submit", + __FUNCTION__, __LINE__ + ); + status = gcvSTATUS_OK; + } + else + { + gcmkONERROR(status); + } + +#ifdef __QNXNTO__ + if(userCommandBufferLogicalMapped) + { + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + Command->kernel, + commandBufferVideoMem, + ProcessID, + gcvFALSE, + gcvFALSE + )); + + userCommandBufferLogicalMapped =gcvFALSE; + } +#endif + + /* Return status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (contextAcquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + } + +#ifdef __QNXNTO__ + if (userCommandBufferLogicalMapped) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userCommandBufferLogical, + 0, + commandBufferLogical)); + } +#endif + + /* Return status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CommitAsyncOnce( + IN gckCOMMAND Command, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer + ) +{ + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + gctBOOL available = gcvFALSE; + gctBOOL acquired = gcvFALSE; + gctUINT8_PTR commandBufferLogical; + gctUINT8_PTR commandBufferTail; + gctUINT commandBufferSize; + gctUINT32 commandBufferAddress; + gctUINT32 fenceBytes; + gctUINT32 oldValue; + gctUINT32 flushBytes; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcmkHEADER(); + + gcmkVERIFY_OK( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + gckOS_AtomicExchange(Command->os, + hardware->pageTableDirty[gcvENGINE_BLT], + 0, + &oldValue); + + if (oldValue) + { + gckHARDWARE_FlushAsyncMMU(hardware, gcvNULL, &flushBytes); + + gcmkASSERT(flushBytes <= CommandBuffer->reservedHead); + + /* Compute the command buffer entry to insert the flushMMU commands. */ + commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead + - flushBytes; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead + - flushBytes; + + gckHARDWARE_FlushAsyncMMU(hardware, commandBufferLogical, &flushBytes); + } + else + { + /* Compute the command buffer entry. */ + commandBufferLogical = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + flushBytes = 0; + } + + commandBufferTail = (gctUINT8_PTR)gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->size + - CommandBuffer->reservedTail; + + gcmkONERROR(gckHARDWARE_Fence( + hardware, + gcvENGINE_BLT, + commandBufferTail, + Command->fence->address, + Command->commitStamp, + &fenceBytes + )); + + gcmkASSERT(fenceBytes <= CommandBuffer->reservedTail); + + commandBufferSize = CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + + flushBytes + + fenceBytes; + + gckOS_AcquireMutex(Command->os, Command->mutexContext, gcvINFINITE); + acquired = gcvTRUE; + + /* Acquire a slot. */ + for (;;) + { + gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); + + if (available) + { + break; + } + else + { + gcmkTRACE_ZONE(gcvLEVEL_INFO, _GC_OBJ_ZONE, "No available slot, have to wait"); + + gckOS_Delay(Command->os, 1); + } + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[async-command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_ASYNC_COMMAND, + commandBufferLogical, + commandBufferAddress, + commandBufferSize + ); + + gcmkDUMP( + Command->os, + "@[execute 1 0 0x%08X 0x%08X]", + commandBufferAddress + + CommandBuffer->reservedHead, + CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail + ); + +#if !gcdNULL_DRIVER + /* Execute command buffer. */ + gckASYNC_FE_Execute(hardware, commandBufferAddress, commandBufferSize); +#endif + + gckOS_ReleaseMutex(Command->os, Command->mutexContext); + acquired = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gckOS_ReleaseMutex(Command->os, Command->mutexContext); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_CommitMultiChannelOnce( + IN gckCOMMAND Command, + IN gckCONTEXT Context, + IN gcsHAL_COMMAND_LOCATION * CommandBuffer + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctUINT8_PTR commandBufferLogical; + gctUINT commandBufferSize; + gctUINT32 commandBufferAddress; + gckHARDWARE hardware; + gctUINT64 bit; + gcsPATCH_LIST_VARIABLE patchListVar = {0, 0}; + + gcmkHEADER_ARG("priority=%d channelId=%d videoMemNode=%u size=0x%x patchHead=%p", + CommandBuffer->priority, CommandBuffer->channelId, + CommandBuffer->videoMemNode, CommandBuffer->size, + gcmUINT64_TO_PTR(CommandBuffer->patchHead)); + + gcmkASSERT(Command->feType == gcvHW_FE_MULTI_CHANNEL); + + hardware = Command->kernel->hardware; + + gcmkVERIFY_OK( + _HandlePatchList(Command, CommandBuffer, &patchListVar)); + + /* Check flush mcfe MMU cache. */ + gcmkONERROR(_CheckFlushMcfeMMU(Command, hardware)); + + /* Compute the command buffer entry and the size. */ + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + commandBufferAddress = CommandBuffer->address + + CommandBuffer->startOffset + + CommandBuffer->reservedHead; + + /* reservedTail bytes are not used, because fence not enable. */ + commandBufferSize + = CommandBuffer->size + - CommandBuffer->reservedHead + - CommandBuffer->reservedTail; + + if (commandBufferSize & 8) + { + /* + * Need 16 byte alignment for MCFE command size. + * command is already 8 byte aligned, if not 16 byte aligned, + * we need append 8 bytes. + */ + gctUINT32 nop[2]; + gctSIZE_T bytes = 8; + gctUINT8_PTR tail = commandBufferLogical + commandBufferSize; + + gckMCFE_Nop(hardware, nop, &bytes); + gcmkASSERT(bytes == 8); + + gckOS_WriteMemory(Command->os, tail, nop[0]); + gckOS_WriteMemory(Command->os, tail + 4, nop[1]); + + commandBufferSize += 8; + } + + /* Large command buffer size does not make sense. */ + gcmkASSERT(commandBufferSize < 0x800000); + + + if (CommandBuffer->channelId != 0) + { + /* Sync from the system channel. */ + gcmkONERROR(_SyncFromSystemChannel( + Command, + (gctBOOL)CommandBuffer->priority, + (gctUINT32)CommandBuffer->channelId + )); + } + + Command->currContext = Context; + + gckOS_AcquireMutex(Command->os, Command->mutexQueue, gcvINFINITE); + acquired = gcvTRUE; + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[mcfe-command: user]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_COMMAND, + commandBufferLogical, + commandBufferAddress, + commandBufferSize + ); + + gcmkDUMP(Command->os, + "@[execute %d %d 0x%08X 0x%08X]", + CommandBuffer->channelId, + CommandBuffer->priority, + commandBufferAddress, + commandBufferSize); + +#if !gcdNULL_DRIVER + /* Execute command buffer. */ + gcmkONERROR(gckMCFE_Execute( + hardware, + (gctBOOL)CommandBuffer->priority, + (gctUINT32)CommandBuffer->channelId, + commandBufferAddress, + commandBufferSize + )); +#endif + + bit = 1ull << CommandBuffer->channelId; + + /* This channel is dirty. */ + Command->dirtyChannel[CommandBuffer->priority ? 1 : 0] |= bit; + + gckOS_ReleaseMutex(Command->os, Command->mutexQueue); + acquired = gcvFALSE; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + gckOS_ReleaseMutex(Command->os, Command->mutexQueue); + } + + gcmkFOOTER(); + return status; +} + + +/******************************************************************************* +** +** gckCOMMAND_Commit +** +** Commit command buffers to the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gcsHAL_SUBCOMMIT * SubCommit +** Commit information, includes context, delta, and command buffer +* locations. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** gctBOOL *contextSwitched +** pass context Switch flag to upper +*/ +gceSTATUS +gckCOMMAND_Commit( + IN gckCOMMAND Command, + IN gcsHAL_SUBCOMMIT * SubCommit, + IN gctUINT32 ProcessId, + IN gctBOOL Shared, + OUT gctUINT64_PTR CommitStamp, + INOUT gctBOOL *contextSwitched + ) +{ + gceSTATUS status; + gcsSTATE_DELTA_PTR delta = gcmUINT64_TO_PTR(SubCommit->delta); + gckCONTEXT context = gcvNULL; + gcsHAL_COMMAND_LOCATION *cmdLoc = &SubCommit->commandBuffer; + gcsHAL_COMMAND_LOCATION _cmdLoc; + gctPOINTER userPtr = gcvNULL; + gctBOOL needCopy = gcvFALSE; + + gcmkHEADER_ARG("Command=%p SubCommit=%p delta=%p context=%u pid=%u", + Command, SubCommit, delta, SubCommit->context, ProcessId); + + gcmkVERIFY_OK(gckOS_QueryNeedCopy(Command->os, ProcessId, &needCopy)); + + if (SubCommit->context) + { + context = gckKERNEL_QueryPointerFromName( + Command->kernel, + (gctUINT32)(SubCommit->context) + ); + } + + do + { + gctUINT64 next; + + /* Skip the first nested one. */ + if (userPtr) + { + /* Copy/map command buffer location from user. */ + if (needCopy) + { + cmdLoc = &_cmdLoc; + + status = gckOS_CopyFromUserData( + Command->os, + cmdLoc, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION) + ); + } + else + { + status = gckOS_MapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + (gctPOINTER *)&cmdLoc + ); + } + + if (gcmIS_ERROR(status)) + { + userPtr = gcvNULL; + + gcmkONERROR(status); + } + } + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + /* Commit command buffers. */ + status = _CommitWaitLinkOnce(Command, + context, + cmdLoc, + delta, + ProcessId, + Shared, + contextSwitched); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + status = _CommitMultiChannelOnce(Command, context, cmdLoc); + } + else + { + gcmkASSERT(Command->feType == gcvHW_FE_ASYNC); + + status = _CommitAsyncOnce(Command, cmdLoc); + } + + if (status != gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(status); + } + + /* Do not need context or delta for later commands. */ + context = gcvNULL; + delta = gcvNULL; + + next = cmdLoc->next; + + /* Unmap user pointer if mapped. */ + if (!needCopy && userPtr) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + cmdLoc + )); + } + + /* Advance to next command buffer location from user. */ + userPtr = gcmUINT64_TO_PTR(next); + } + while (userPtr); + + if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + /* + * Semphore synchronization. + * + * Here we blindly sync dirty other channels to the system channel. + * The scenario to sync channels to the system channel: + * 1. Need to sync channels who sent semaphores. + * 2. Need to sync dirty channels when event(interrupt) is to sent. + * 3. Need to sync dirty channels when system channel need run something + * such as flush mmu. + * + * When power management is on, blindly sync dirty channels is OK because + * there's always a event(intrrupt). + * + * The only condition we sync more than needed is: + * a. power manangement is off. + * b. no user event is attached when commit. + * c. no user event is to be submitted in next ioctl. + * That's a rare condition. + * + * Conclusion is that, blindly sync dirty channels is a good choice for + * now. + */ + gcmkONERROR(_SyncToSystemChannel(Command, Command->dirtyChannel)); + } + + /* Output commit stamp. */ + *CommitStamp = Command->commitStamp; + Command->commitStamp++; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (!needCopy && userPtr) + { + gckOS_UnmapUserPointer( + Command->os, + userPtr, + gcmSIZEOF(gcsHAL_COMMAND_LOCATION), + cmdLoc + ); + } + + gcmkFOOTER(); + return status; +} + + +/******************************************************************************* +** +** gckCOMMAND_Reserve +** +** Reserve space in the command queue. Also acquire the command queue mutex. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** gctPOINTER * Buffer +** Pointer to a variable that will receive the address of the reserved +** space. +** +** gctSIZE_T * BufferSize +** Pointer to a variable that will receive the number of bytes +** available in the command queue. +*/ +gceSTATUS +gckCOMMAND_Reserve( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes, + OUT gctPOINTER * Buffer, + OUT gctUINT32 * BufferSize + ) +{ + gceSTATUS status; + gctUINT32 bytes; + gctUINT32 requiredBytes; + gctUINT32 requestedAligned; + + gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + /* Compute aligned number of reuested bytes. */ + requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); + + /* Another WAIT/LINK command sequence will have to be appended after + the requested area being reserved. Compute the number of bytes + required for WAIT/LINK at the location after the reserved area. */ + gcmkONERROR(gckWLFE_WaitLink( + Command->kernel->hardware, + gcvNULL, + ~0U, + Command->offset + requestedAligned, + &requiredBytes, + gcvNULL, + gcvNULL + )); + + /* Compute total number of bytes required. */ + requiredBytes += requestedAligned; + } + else + { + requiredBytes = gcmALIGN(RequestedBytes, 8); + } + + /* Compute number of bytes available in command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Is there enough space in the current command queue? */ + if (bytes < requiredBytes) + { + /* Create a new command queue. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Still not enough space? */ + if (bytes < requiredBytes) + { + /* Rare case, not enough room in command queue. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + } + + /* Return pointer to empty slot command queue. */ + *Buffer = (gctUINT8 *) Command->logical + Command->offset; + + /* Return number of bytes left in command queue. */ + *BufferSize = bytes; + + /* Success. */ + gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); + return gcvSTATUS_OK; + +OnError: + /* Return status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Execute +** +** Execute a previously reserved command queue by appending a WAIT/LINK command +** sequence after it and modifying the last WAIT into a LINK command. The +** command FIFO mutex will be released whether this function succeeds or not. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Execute( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + + gctUINT8_PTR waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkOffset; + gctUINT32 waitLinkBytes; + + gctUINT32 waitOffset; + gctUINT32 waitBytes; + + gctUINT32 linkLow, linkHigh; + + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + gcmkHEADER_ARG("Command=%p RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Compute offset for WAIT/LINK. */ + waitLinkOffset = Command->offset + RequestedBytes; + + /* Compute number of bytes left in command queue. */ + waitLinkBytes = Command->pageSize - waitLinkOffset; + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; + waitLinkAddress = Command->address + waitLinkOffset; + + /* Append WAIT/LINK in command queue. */ + gcmkONERROR(gckWLFE_WaitLink( + Command->kernel->hardware, + waitLinkLogical, + waitLinkAddress, + waitLinkOffset, + &waitLinkBytes, + &waitOffset, + &waitBytes + )); + + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes + waitLinkBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes + waitLinkBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* + * Skip link to execAddress. + * Instead, we directly link to final wait link position. + */ + gcmkONERROR(gckWLFE_Link( + Command->kernel->hardware, + Command->waitPos.logical, + waitLinkAddress, + waitLinkBytes, + &Command->waitPos.size, + &linkLow, + &linkHigh + )); +#else + /* Convert the last WAIT into a LINK. */ + gcmkONERROR(gckWLFE_Link( + Command->kernel->hardware, + Command->waitPos.logical, + execAddress, + execBytes, + &Command->waitPos.size, + &linkLow, + &linkHigh + )); +#endif + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->waitPos.videoMem, + Command->waitPos.offset, + Command->waitPos.logical, + Command->waitPos.size + )); + +#if gcdLINK_QUEUE_SIZE + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = execAddress; + data.linkData.end = execAddress + execBytes; + data.linkData.linkLow = linkLow; + data.linkData.linkHigh = linkHigh; + + gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); + } +#endif + + gcmkDUMP(Command->os, "#[command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + gcmkDUMP(Command->os, + "#[null driver: below command skipped link to 0x%08X 0x%08X]", + execAddress, + execBytes); +#endif + + gcmkDUMP(Command->os, "#[link: break prev wait-link]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + Command->waitPos.logical, + Command->waitPos.address, + Command->waitPos.size + ); + + /* Update the pointer to the last WAIT. */ + Command->waitPos.videoMem = Command->videoMem; + Command->waitPos.offset = waitLinkOffset + waitOffset; + Command->waitPos.logical = (gctUINT8_PTR)waitLinkLogical + waitOffset; + Command->waitPos.address = waitLinkAddress + waitOffset; + Command->waitPos.size = waitBytes; + + /* Update the command queue. */ + Command->offset += RequestedBytes + waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + Command->kernel->hardware, Command->logical, Command->offset + )); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckCOMMAND_ExecuteAsync( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctBOOL available; + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + hardware = Command->kernel->hardware; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes; + + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + + /* Acquire a slot. */ + for (;;) + { + gcmkONERROR(gckASYNC_FE_ReserveSlot(hardware, &available)); + + if (available) + { + break; + } + else + { + gckOS_Delay(Command->os, 1); + } + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[async-command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + +#if !gcdNULL_DRIVER + /* Send descriptor. */ + gckASYNC_FE_Execute(hardware, execAddress, execBytes); +#endif + + /* Update the command queue. */ + Command->offset += RequestedBytes; + Command->newQueue = gcvFALSE; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +gceSTATUS +gckCOMMAND_ExecuteMultiChannel( + IN gckCOMMAND Command, + IN gctBOOL Priority, + IN gctUINT32 ChannelId, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ + execLogical = Command->logical; + execAddress = Command->address; + execBytes = Command->offset + RequestedBytes; + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes; + } + + if (execBytes & 8) + { + gctSIZE_T bytes = 8; + gctUINT8_PTR tail = (gctUINT8_PTR)execLogical + execBytes; + + gckMCFE_Nop(Command->kernel->hardware, tail, &bytes); + gcmkASSERT(bytes == 8); + + execBytes += 8; + RequestedBytes += 8; + } + + if (Command->newQueue) + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + 0, + execLogical, + execBytes + )); + } + else + { + gcmkONERROR(gckVIDMEM_NODE_CleanCache( + Command->kernel, + Command->videoMem, + Command->offset, + execLogical, + execBytes + )); + } + +#if gcdNULL_DRIVER || gcdCAPTURE_ONLY_MODE + /* Skip submit to hardware for NULL driver. */ + gcmkDUMP(Command->os, "#[null driver: below command is skipped]"); +#endif + + gcmkDUMP(Command->os, "#[mcfe-command: kernel execute]"); + gcmkDUMP_BUFFER( + Command->os, + gcvDUMP_BUFFER_KERNEL_COMMAND, + execLogical, + execAddress, + execBytes + ); + + gcmkDUMP(Command->os, + "@[execute %u %u 0x%08X 0x%08X]", + ChannelId, Priority, execAddress, execBytes); + +#if !gcdNULL_DRIVER + /* Send descriptor. */ + gcmkONERROR( + gckMCFE_Execute(Command->kernel->hardware, + Priority, + ChannelId, + execAddress, + execBytes)); +#endif + + /* Update the command queue. */ + Command->offset += RequestedBytes; + Command->newQueue = gcvFALSE; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stall +** +** The calling thread will be suspended until the command queue has been +** completed. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stall( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gckOS os; + gckHARDWARE hardware; + gckEVENT eventObject; + gceSTATUS status; + gctSIGNAL signal = gcvNULL; + gctUINT timer = 0; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Extract the gckOS object pointer. */ + os = Command->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Extract the gckHARDWARE object pointer. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Extract the gckEVENT object pointer. */ + eventObject = Command->kernel->eventObj; + gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); + + /* Allocate the signal. */ + gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); + + /* Append the EVENT command to trigger the signal. */ + gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); + + /* Submit the event queue. */ + gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); + + gcmkDUMP(Command->os, "#[kernel.stall]"); + + if (status == gcvSTATUS_CHIP_NOT_READY) + { + /* Error. */ + goto OnError; + } + + do + { + /* Wait for the signal. */ + status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); + + if (status == gcvSTATUS_TIMEOUT) + { +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gctUINT32 idle; + + /* Read idle register. */ + gcmkVERIFY_OK(gckHARDWARE_GetIdle( + hardware, gcvFALSE, &idle + )); + + gcmkTRACE( + gcvLEVEL_ERROR, + "%s(%d): idle=%08x", + __FUNCTION__, __LINE__, idle + ); + + gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); +#endif + + /* Advance timer. */ + timer += gcdGPU_ADVANCETIMER; + } + else if (status == gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(gcvSTATUS_INTERRUPTED); + } + + } + while (gcmIS_ERROR(status)); + + /* Bail out on timeout. */ + if (gcmIS_ERROR(status)) + { + /* Broadcast the stuck GPU. */ + gcmkONERROR(gckOS_Broadcast( + os, hardware, gcvBROADCAST_GPU_STUCK + )); + } + + /* Delete the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (signal != gcvNULL) + { + /* Free the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if (gcdENABLE_3D || gcdENABLE_2D) +static gceSTATUS +_AttachWaitLinkFECommand( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Construct( + Command->os, + Command->kernel->hardware, + ProcessID, + Context + )); + + /* Return the number of states in the context. */ + * MaxState = (* Context)->maxState; + * NumStates = (* Context)->numStates; + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER_ARG("*Context=0x%x", *Context); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Attach +** +** Attach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** +** gckCONTEXT * Context +** Pointer to a variable that will receive a pointer to a new +** gckCONTEXT object. +** +** gctSIZE_T * StateCount +** Pointer to a variable that will receive the number of states +** in the context buffer. +*/ +gceSTATUS +gckCOMMAND_Attach( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ) +{ + gctUINT32 allocationSize; + gctPOINTER pointer; + gceSTATUS status; + + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + status = _AttachWaitLinkFECommand(Command, + Context, + MaxState, + NumStates, + ProcessID); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + /* + * For mcfe, we only allocate context which is used to + * store profile counters. + */ + allocationSize = gcmSIZEOF(struct _gckCONTEXT); + + /* Allocate the object. */ + gckOS_Allocate(Command->os, allocationSize, &pointer); + if (!pointer) + { + return gcvSTATUS_OUT_OF_MEMORY; + } + *Context = pointer; + /* Reset the entire object. */ + gckOS_ZeroMemory(*Context, allocationSize); + + /* Initialize the gckCONTEXT object. */ + (*Context)->object.type = gcvOBJ_CONTEXT; + (*Context)->os = Command->os; + (*Context)->hardware = Command->kernel->hardware; + *MaxState = 0; + *NumStates = 0; + + status = gcvSTATUS_OK; + } + else + { + /* Nothing to do. */ + *Context = gcvNULL; + *MaxState = 0; + *NumStates = 0; + + status = gcvSTATUS_OK; + } + + return status; +} +#endif + +static gceSTATUS +_DetachWaitLinkFECommand( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=%p Context=%p", Command, Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Destroy(Context)); + + if (Command->currContext == Context) + { + /* Detach from gckCOMMAND object if the destoryed context is current context. */ + Command->currContext = gcvNULL; + } + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Detach +** +** Detach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gckCONTEXT Context +** Pointer to a gckCONTEXT object to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Detach( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ) +{ + if (Command->feType == gcvHW_FE_WAIT_LINK) + { + return _DetachWaitLinkFECommand(Command, Context); + } + else if (Command->feType == gcvHW_FE_MULTI_CHANNEL) + { + gcmkOS_SAFE_FREE(Context->os, Context); + return gcvSTATUS_OK; + } + else + { + /* Nothing to do. */ + return gcvSTATUS_OK; + } +} + +static void +_DumpBuffer( + IN gctPOINTER Buffer, + IN gctUINT32 GpuAddress, + IN gctSIZE_T Size + ) +{ + gctSIZE_T i, line, left; + gctUINT32_PTR data = Buffer; + + line = Size / 32; + left = Size % 32; + + for (i = 0; i < line; i++) + { + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); + data += 8; + GpuAddress += 8 * 4; + } + + switch(left) + { + case 28: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); + break; + case 24: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); + break; + case 20: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4]); + break; + case 16: + gcmkPRINT("%08X : %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3]); + break; + case 12: + gcmkPRINT("%08X : %08X %08X %08X", + GpuAddress, data[0], data[1], data[2]); + break; + case 8: + gcmkPRINT("%08X : %08X %08X", + GpuAddress, data[0], data[1]); + break; + case 4: + gcmkPRINT("%08X : %08X", + GpuAddress, data[0]); + break; + default: + break; + } +} + +/******************************************************************************* +** +** gckCOMMAND_DumpExecutingBuffer +** +** Dump the command buffer which GPU is executing. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_DumpExecutingBuffer( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject = gcvNULL; + gctUINT32 gpuAddress; + gctPOINTER entry = gcvNULL; + gckOS os = Command->os; + gckKERNEL kernel = Command->kernel; + gctUINT32 i; + gckQUEUE queue = &kernel->hardware->linkQueue; + gctSIZE_T bytes; + gctUINT32 offset; + gctPOINTER entryDump; + gctUINT8 processName[24] = {0}; + + gcmkPRINT("**************************\n"); + gcmkPRINT("**** COMMAND BUF DUMP ****\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); + gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + + gcmkPRINT("DMA Address 0x%08X", gpuAddress); + + /* Find GPU address in video memory list. */ + status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); + + if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) + { + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + nodeObject, + gcvFALSE, + gcvFALSE, + &entryDump + )); + + gcmkVERIFY_OK(gckVIDMEM_NODE_GetSize( + kernel, + nodeObject, + &bytes + )); + + gcmkPRINT("Command buffer around 0x%08X:", gpuAddress); + + /* Align to 4096. */ + offset &= 0xfffff000; + gpuAddress &= 0xfffff000; + + /* Dump max 4096 bytes. */ + bytes = (bytes - offset) > 4096 ? 4096 : (bytes - offset); + + /* Kernel address of page where stall point stay. */ + entryDump = (gctUINT8_PTR)entryDump + offset; + + _DumpBuffer(entryDump, gpuAddress, bytes); + + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + nodeObject, + 0, + gcvFALSE, + gcvFALSE + )); + } + else + { + gcmkPRINT("Can not find command buffer around 0x%08X.\n", gpuAddress); + } + + /* new line. */ + gcmkPRINT(""); + + gcmkPRINT("Kernel command buffers:"); + + for (i = 0; i < gcdCOMMAND_QUEUES; i++) + { + entry = Command->queues[i].logical; + gpuAddress = Command->queues[i].address; + + gcmkPRINT("command buffer %d at 0x%08X size %u", + i, gpuAddress, Command->pageSize); + _DumpBuffer(entry, gpuAddress, Command->pageSize); + } + + /* new line. */ + gcmkPRINT(""); + + if (queue->count) + { + gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", + Command->kernel->stuckDump, queue->count); + } + + for (i = 0; i < queue->count; i++) + { + gcuQUEUEDATA * queueData; + gckLINKDATA linkData; + + gckQUEUE_GetData(queue, i, &queueData); + + linkData = &queueData->linkData; + + /* Get gpu address of this command buffer. */ + gpuAddress = linkData->start; + bytes = linkData->end - gpuAddress; + + processName[0] = '\0'; + gckOS_GetProcessNameByPid(linkData->pid, sizeof(processName), processName); + + gcmkPRINT("Link record %d: [%08X - %08X] from command %08X %08X pid %u (%s):", + i, + linkData->start, + linkData->end, + linkData->linkLow, + linkData->linkHigh, + linkData->pid, + processName); + + /* Find GPU address in video memory list. */ + status = gckVIDMEM_NODE_Find(kernel, gpuAddress, &nodeObject, &offset); + + if (gcmIS_SUCCESS(status) && nodeObject->type == gcvVIDMEM_TYPE_COMMAND) + { + gcmkONERROR(gckVIDMEM_NODE_LockCPU( + kernel, + nodeObject, + gcvFALSE, + gcvFALSE, + &entryDump + )); + + /* Kernel address of page where stall point stay. */ + entryDump = (gctUINT8_PTR)entryDump + offset; + + _DumpBuffer(entryDump, gpuAddress, bytes); + + gcmkVERIFY_OK(gckVIDMEM_NODE_UnlockCPU( + kernel, + nodeObject, + 0, + gcvFALSE, + gcvFALSE + )); + } + else + { + gcmkPRINT("Not found"); + } + + /* new line. */ + gcmkPRINT(""); + } + + return gcvSTATUS_OK; + +OnError: + return status; +} + |