All/some skeletons not properly displayed or missing when many people

gineshidalgo99 · gineshidalgo99 · commit 98e53de17476 · 2020-04-26T16:39:18.000-04:00
diff --git a/doc/release_notes.md b/doc/release_notes.md
@@ -407,6 +407,7 @@ OpenPose Library - Release Notes
     4. Natural sort now works properly with filenames containining numbers longer than the limit of an int.
     5. Optionally auto-generated bin folder only contains the required DLLs (depending on the CMake configuration), instead of all of them.
     6. When WrapperStructFace and WrapperStructHand are not called and configured for Wrapper, setting body to CPU rendering was not working.
+    7. Skelton rendering: All or some skeletons were not properly displayed or completely missing on images with many people.
 4. Changes/additions that affect the compatibility with the OpenPose Unity Plugin:
 
 
diff --git a/include/openpose_private/utilities/render.hu b/include/openpose_private/utilities/render.hu
@@ -8,18 +8,14 @@ namespace op
         const float* const keypointsPtr, const int numberPeople, const int numberParts, const float threshold)
     {
         const auto globalIdx = threadIdx.x;
-        // const auto xIndex = 2*globalIdx;
-        // const auto yIndex = xIndex+1;
-        const auto xIndex = globalIdx;
-        const auto yIndex = numberPeople+globalIdx;
 
         // Fill shared parameters
-        // if (globalIdx < numberPeople)
+        if (globalIdx < numberPeople)
         {
-            auto minValueX = (float)targetWidth;
-            auto minValueY = (float)targetHeight;
-            auto maxValueX = 0.f;
-            auto maxValueY = 0.f;
+            float minValueX = (float)targetWidth;
+            float minValueY = (float)targetHeight;
+            float maxValueX = 0.f;
+            float maxValueY = 0.f;
             for (auto part = 0 ; part < numberParts ; part++)
             {
                 const auto index = 3 * (globalIdx*numberParts + part);
@@ -51,13 +47,18 @@ namespace op
                 minValueY -= constantToAdd;
             }
 
+            // const auto xIndex = 2*globalIdx;
+            // const auto yIndex = xIndex+1;
+            const auto xIndex = globalIdx;
+            const auto yIndex = numberPeople+globalIdx;
             minPtr[xIndex] = minValueX;
             minPtr[yIndex] = minValueY;
             maxPtr[xIndex] = maxValueX;
             maxPtr[yIndex] = maxValueY;
         }
     }
 
+    // Note: renderKeypoints is not working for videos with many people, renderKeypointsOld speed was slightly improved instead
     __inline__ __device__ void renderKeypoints(
         float* targetPtr, float* sharedMaxs, float* sharedMins, float* sharedScaleF, const float* const maxPtr,
         const float* const minPtr, const float* const scalePtr, const int globalIdx, const int x, const int y,
@@ -82,9 +83,9 @@ namespace op
         if (x < targetWidth && y < targetHeight)
         {
             const auto baseIndex = 3*(y * targetWidth + x);
-            auto b = targetPtr[baseIndex];
-            auto g = targetPtr[baseIndex+1];
-            auto r = targetPtr[baseIndex+2];
+            float b = targetPtr[baseIndex];
+            float g = targetPtr[baseIndex+1];
+            float r = targetPtr[baseIndex+2];
             if (!blendOriginalFrame)
             {
                 b = 0.f;
@@ -104,7 +105,7 @@ namespace op
                 const auto xIndex = person;
                 const auto yIndex = numberPeople+person;
                 if (x <= sharedMaxs[xIndex] && x >= sharedMins[xIndex]
-                     && y <= sharedMaxs[yIndex] && y >= sharedMins[yIndex])
+                    && y <= sharedMaxs[yIndex] && y >= sharedMins[yIndex])
                 {
                     // Part pair connections
                     for (auto partPair = 0; partPair < numberPartPairs; partPair++)
@@ -195,7 +196,6 @@ namespace op
                                 if (minr2 <= dist2 && dist2 <= maxr2)
                                     addColorWeighted(r, g, b, &rgbColorsPtr[(part%numberColors)*3], alphaColorToAdd);
                             }
-
                         }
                     }
                 }
@@ -218,10 +218,10 @@ namespace op
         // Fill shared parameters
         if (globalIdx < numberPeople)
         {
-            sharedMins[globalIdx].x = targetWidth;
-            sharedMins[globalIdx].y = targetHeight;
-            sharedMaxs[globalIdx].x = 0.f;
-            sharedMaxs[globalIdx].y = 0.f;
+            float minValueX = (float)targetWidth;
+            float minValueY = (float)targetHeight;
+            float maxValueX = 0.f;
+            float maxValueY = 0.f;
             for (auto part = 0 ; part < numberParts ; part++)
             {
                 const auto index = 3 * (globalIdx*numberParts + part);
@@ -230,38 +230,43 @@ namespace op
                 const auto score = keypointsPtr[index+2];
                 if (score > threshold)
                 {
-                    if (x < sharedMins[globalIdx].x)
-                        sharedMins[globalIdx].x = x;
-                    if (x > sharedMaxs[globalIdx].x)
-                        sharedMaxs[globalIdx].x = x;
-                    if (y < sharedMins[globalIdx].y)
-                        sharedMins[globalIdx].y = y;
-                    if (y > sharedMaxs[globalIdx].y)
-                        sharedMaxs[globalIdx].y = y;
+                    if (x < minValueX)
+                        minValueX = x;
+                    if (x > maxValueX)
+                        maxValueX = x;
+                    if (y < minValueY)
+                        minValueY = y;
+                    if (y > maxValueY)
+                        maxValueY = y;
                 }
             }
-            if (sharedMaxs[globalIdx].x != 0.f && sharedMaxs[globalIdx].y != 0.f)
+            if (maxValueX != 0.f && maxValueY != 0.f)
             {
-                const auto averageX = sharedMaxs[globalIdx].x - sharedMins[globalIdx].x;
-                const auto averageY = sharedMaxs[globalIdx].y - sharedMins[globalIdx].y;
+                const auto averageX = maxValueX - minValueX;
+                const auto averageY = maxValueY - minValueY;
                 // (averageX + averageY) / 2.f / 400.f
                 sharedScaleF[globalIdx] = fastTruncateCuda((averageX + averageY) / 400.f, 0.33f, 1.f);
                 const auto constantToAdd = 50.f;
-                sharedMaxs[globalIdx].x += constantToAdd;
-                sharedMaxs[globalIdx].y += constantToAdd;
-                sharedMins[globalIdx].x -= constantToAdd;
-                sharedMins[globalIdx].y -= constantToAdd;
+                maxValueX += constantToAdd;
+                maxValueY += constantToAdd;
+                minValueX -= constantToAdd;
+                minValueY -= constantToAdd;
             }
+
+            sharedMins[globalIdx].x = minValueX;
+            sharedMins[globalIdx].y = minValueY;
+            sharedMaxs[globalIdx].x = maxValueX;
+            sharedMaxs[globalIdx].y = maxValueY;
         }
         __syncthreads();
 
         // Fill each (x,y) target pixel
         if (x < targetWidth && y < targetHeight)
         {
             const auto baseIndex = 3*(y * targetWidth + x);
-            auto& b = targetPtr[baseIndex];
-            auto& g = targetPtr[baseIndex+1];
-            auto& r = targetPtr[baseIndex+2];
+            float b = targetPtr[baseIndex];
+            float g = targetPtr[baseIndex+1];
+            float r = targetPtr[baseIndex+2];
             if (!blendOriginalFrame)
             {
                 b = 0.f;
@@ -372,6 +377,9 @@ namespace op
                     }
                 }
             }
+            targetPtr[baseIndex] = b;
+            targetPtr[baseIndex+1] = g;
+            targetPtr[baseIndex+2] = r;
         }
     }
 }
diff --git a/src/openpose/pose/poseGpuRenderer.cpp b/src/openpose/pose/poseGpuRenderer.cpp
@@ -74,8 +74,8 @@ namespace op
             opLog("Starting initialization on thread.", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
             // GPU memory allocation for rendering
             #ifdef USE_CUDA
-                cudaMalloc((void**)(&pGpuPose),
-                    POSE_MAX_PEOPLE * getPoseNumberBodyParts(mPoseModel) * 3 * sizeof(float));
+                const auto gpuPoseVolume = POSE_MAX_PEOPLE * getPoseNumberBodyParts(mPoseModel) * 3 * sizeof(float);
+                cudaMalloc((void**)(&pGpuPose), gpuPoseVolume);
                 cudaMalloc((void**)&pMaxPtr, sizeof(float) * 2 * POSE_MAX_PEOPLE);
                 cudaMalloc((void**)&pMinPtr, sizeof(float) * 2 * POSE_MAX_PEOPLE);
                 cudaMalloc((void**)&pScalePtr, sizeof(float) * POSE_MAX_PEOPLE);
@@ -119,9 +119,11 @@ namespace op
                         scaleKeypoints(poseKeypointsRescaled, scaleInputToOutput);
                         // Render keypoints
                         if (!poseKeypoints.empty())
+                        {
+                            const auto gpuPoseVolume = numberPeople * numberBodyParts * 3 * sizeof(float);
                             cudaMemcpy(
-                                pGpuPose, poseKeypointsRescaled.getConstPtr(),
-                                numberPeople * numberBodyParts * 3 * sizeof(float), cudaMemcpyHostToDevice);
+                                pGpuPose, poseKeypointsRescaled.getConstPtr(), gpuPoseVolume, cudaMemcpyHostToDevice);
+                        }
                         renderPoseKeypointsGpu(
                             *spGpuMemory, pMaxPtr, pMinPtr, pScalePtr, mPoseModel, numberPeople, frameSize, pGpuPose,
                             mRenderThreshold, mShowGooglyEyes, mBlendOriginalFrame, getAlphaKeypoint());
diff --git a/src/openpose/pose/renderPose.cu b/src/openpose/pose/renderPose.cu

Original file line number	Diff line number	Diff line change
`@@ -8,18 +8,14 @@ namespace op`
`8`	`8`	`const float* const keypointsPtr, const int numberPeople, const int numberParts, const float threshold)`
`9`	`9`	`{`
`10`	`10`	`const auto globalIdx = threadIdx.x;`
`11`		`- // const auto xIndex = 2*globalIdx;`
`12`		`- // const auto yIndex = xIndex+1;`
`13`		`- const auto xIndex = globalIdx;`
`14`		`- const auto yIndex = numberPeople+globalIdx;`
`15`	`11`
`16`	`12`	`// Fill shared parameters`
`17`		`- // if (globalIdx < numberPeople)`
	`13`	`+ if (globalIdx < numberPeople)`
`18`	`14`	`{`
`19`		`- auto minValueX = (float)targetWidth;`
`20`		`- auto minValueY = (float)targetHeight;`
`21`		`- auto maxValueX = 0.f;`
`22`		`- auto maxValueY = 0.f;`
	`15`	`+ float minValueX = (float)targetWidth;`
	`16`	`+ float minValueY = (float)targetHeight;`
	`17`	`+ float maxValueX = 0.f;`
	`18`	`+ float maxValueY = 0.f;`
`23`	`19`	`for (auto part = 0 ; part < numberParts ; part++)`
`24`	`20`	`{`
`25`	`21`	`const auto index = 3 * (globalIdx*numberParts + part);`
`@@ -51,13 +47,18 @@ namespace op`
`51`	`47`	`minValueY -= constantToAdd;`
`52`	`48`	`}`
`53`	`49`
	`50`	`+ // const auto xIndex = 2*globalIdx;`
	`51`	`+ // const auto yIndex = xIndex+1;`
	`52`	`+ const auto xIndex = globalIdx;`
	`53`	`+ const auto yIndex = numberPeople+globalIdx;`
`54`	`54`	`minPtr[xIndex] = minValueX;`
`55`	`55`	`minPtr[yIndex] = minValueY;`
`56`	`56`	`maxPtr[xIndex] = maxValueX;`
`57`	`57`	`maxPtr[yIndex] = maxValueY;`
`58`	`58`	`}`
`59`	`59`	`}`
`60`	`60`
	`61`	`+ // Note: renderKeypoints is not working for videos with many people, renderKeypointsOld speed was slightly improved instead`
`61`	`62`	`__inline__ __device__ void renderKeypoints(`
`62`	`63`	`float* targetPtr, float* sharedMaxs, float* sharedMins, float* sharedScaleF, const float* const maxPtr,`
`63`	`64`	`const float* const minPtr, const float* const scalePtr, const int globalIdx, const int x, const int y,`
`@@ -82,9 +83,9 @@ namespace op`
`82`	`83`	`if (x < targetWidth && y < targetHeight)`
`83`	`84`	`{`
`84`	`85`	`const auto baseIndex = 3(y targetWidth + x);`
`85`		`- auto b = targetPtr[baseIndex];`
`86`		`- auto g = targetPtr[baseIndex+1];`
`87`		`- auto r = targetPtr[baseIndex+2];`
	`86`	`+ float b = targetPtr[baseIndex];`
	`87`	`+ float g = targetPtr[baseIndex+1];`
	`88`	`+ float r = targetPtr[baseIndex+2];`
`88`	`89`	`if (!blendOriginalFrame)`
`89`	`90`	`{`
`90`	`91`	`b = 0.f;`
`@@ -104,7 +105,7 @@ namespace op`
`104`	`105`	`const auto xIndex = person;`
`105`	`106`	`const auto yIndex = numberPeople+person;`
`106`	`107`	`if (x <= sharedMaxs[xIndex] && x >= sharedMins[xIndex]`
`107`		`- && y <= sharedMaxs[yIndex] && y >= sharedMins[yIndex])`
	`108`	`+ && y <= sharedMaxs[yIndex] && y >= sharedMins[yIndex])`
`108`	`109`	`{`
`109`	`110`	`// Part pair connections`
`110`	`111`	`for (auto partPair = 0; partPair < numberPartPairs; partPair++)`
`@@ -195,7 +196,6 @@ namespace op`
`195`	`196`	`if (minr2 <= dist2 && dist2 <= maxr2)`
`196`	`197`	`addColorWeighted(r, g, b, &rgbColorsPtr[(part%numberColors)*3], alphaColorToAdd);`
`197`	`198`	`}`
`198`		`-`
`199`	`199`	`}`
`200`	`200`	`}`
`201`	`201`	`}`
`@@ -218,10 +218,10 @@ namespace op`
`218`	`218`	`// Fill shared parameters`
`219`	`219`	`if (globalIdx < numberPeople)`
`220`	`220`	`{`
`221`		`- sharedMins[globalIdx].x = targetWidth;`
`222`		`- sharedMins[globalIdx].y = targetHeight;`
`223`		`- sharedMaxs[globalIdx].x = 0.f;`
`224`		`- sharedMaxs[globalIdx].y = 0.f;`
	`221`	`+ float minValueX = (float)targetWidth;`
	`222`	`+ float minValueY = (float)targetHeight;`
	`223`	`+ float maxValueX = 0.f;`
	`224`	`+ float maxValueY = 0.f;`
`225`	`225`	`for (auto part = 0 ; part < numberParts ; part++)`
`226`	`226`	`{`
`227`	`227`	`const auto index = 3 * (globalIdx*numberParts + part);`
`@@ -230,38 +230,43 @@ namespace op`
`230`	`230`	`const auto score = keypointsPtr[index+2];`
`231`	`231`	`if (score > threshold)`
`232`	`232`	`{`
`233`		`- if (x < sharedMins[globalIdx].x)`
`234`		`- sharedMins[globalIdx].x = x;`
`235`		`- if (x > sharedMaxs[globalIdx].x)`
`236`		`- sharedMaxs[globalIdx].x = x;`
`237`		`- if (y < sharedMins[globalIdx].y)`
`238`		`- sharedMins[globalIdx].y = y;`
`239`		`- if (y > sharedMaxs[globalIdx].y)`
`240`		`- sharedMaxs[globalIdx].y = y;`
	`233`	`+ if (x < minValueX)`
	`234`	`+ minValueX = x;`
	`235`	`+ if (x > maxValueX)`
	`236`	`+ maxValueX = x;`
	`237`	`+ if (y < minValueY)`
	`238`	`+ minValueY = y;`
	`239`	`+ if (y > maxValueY)`
	`240`	`+ maxValueY = y;`
`241`	`241`	`}`
`242`	`242`	`}`
`243`		`- if (sharedMaxs[globalIdx].x != 0.f && sharedMaxs[globalIdx].y != 0.f)`
	`243`	`+ if (maxValueX != 0.f && maxValueY != 0.f)`
`244`	`244`	`{`
`245`		`- const auto averageX = sharedMaxs[globalIdx].x - sharedMins[globalIdx].x;`
`246`		`- const auto averageY = sharedMaxs[globalIdx].y - sharedMins[globalIdx].y;`
	`245`	`+ const auto averageX = maxValueX - minValueX;`
	`246`	`+ const auto averageY = maxValueY - minValueY;`
`247`	`247`	`// (averageX + averageY) / 2.f / 400.f`
`248`	`248`	`sharedScaleF[globalIdx] = fastTruncateCuda((averageX + averageY) / 400.f, 0.33f, 1.f);`
`249`	`249`	`const auto constantToAdd = 50.f;`
`250`		`- sharedMaxs[globalIdx].x += constantToAdd;`
`251`		`- sharedMaxs[globalIdx].y += constantToAdd;`
`252`		`- sharedMins[globalIdx].x -= constantToAdd;`
`253`		`- sharedMins[globalIdx].y -= constantToAdd;`
	`250`	`+ maxValueX += constantToAdd;`
	`251`	`+ maxValueY += constantToAdd;`
	`252`	`+ minValueX -= constantToAdd;`
	`253`	`+ minValueY -= constantToAdd;`
`254`	`254`	`}`
	`255`	`+`
	`256`	`+ sharedMins[globalIdx].x = minValueX;`
	`257`	`+ sharedMins[globalIdx].y = minValueY;`
	`258`	`+ sharedMaxs[globalIdx].x = maxValueX;`
	`259`	`+ sharedMaxs[globalIdx].y = maxValueY;`
`255`	`260`	`}`
`256`	`261`	`__syncthreads();`
`257`	`262`
`258`	`263`	`// Fill each (x,y) target pixel`
`259`	`264`	`if (x < targetWidth && y < targetHeight)`
`260`	`265`	`{`
`261`	`266`	`const auto baseIndex = 3(y targetWidth + x);`
`262`		`- auto& b = targetPtr[baseIndex];`
`263`		`- auto& g = targetPtr[baseIndex+1];`
`264`		`- auto& r = targetPtr[baseIndex+2];`
	`267`	`+ float b = targetPtr[baseIndex];`
	`268`	`+ float g = targetPtr[baseIndex+1];`
	`269`	`+ float r = targetPtr[baseIndex+2];`
`265`	`270`	`if (!blendOriginalFrame)`
`266`	`271`	`{`
`267`	`272`	`b = 0.f;`
`@@ -372,6 +377,9 @@ namespace op`
`372`	`377`	`}`
`373`	`378`	`}`
`374`	`379`	`}`
	`380`	`+ targetPtr[baseIndex] = b;`
	`381`	`+ targetPtr[baseIndex+1] = g;`
	`382`	`+ targetPtr[baseIndex+2] = r;`
`375`	`383`	`}`
`376`	`384`	`}`
`377`	`385`	`}`