refactor: upgrade Pinecone SDK, Node.js environment, and fix sync logic

- Upgrade Docker Node.js runtime to v20 to support Pinecone v7 SDK.
- Fix document-processor crash by adding setuptools to requirements.txt.
- Replace deprecated Pinecone REST logic with native SDK describeIndex.
- Resolve 400 Bad Request during Sync by dynamically fetching index dimensions.
- Standardize rawGet implementations across vector providers using native SDKs.
- Update backend models and utilities to support SDK-based queries and retries.

Fixes Mintplex-Labs#170, Fixes Mintplex-Labs#129, Closes Mintplex-Labs#144

Author: snapspecter

.gitignore

@@ -8,4 +8,8 @@ v-env
 .DS_Store
 yarn-error.log
 yarn.lock
-frontend/.env.development
+frontend/.env.development
+
+.agents/
+.sdd/
+.agent/workflows/

backend/endpoints/auth.js

@@ -30,7 +30,7 @@ function authenticationEndpoints(app) {
         user: onboardingUser,
         token: makeJWT(
           { id: onboardingUser.id, email: onboardingUser.email },
-          "1hr"
+          "1hr",
         ),
         message: null,
       });
@@ -93,7 +93,7 @@ function authenticationEndpoints(app) {
         user: existingUser,
         token: makeJWT(
           { id: existingUser.id, email: existingUser.email },
-          "30d"
+          "30d",
         ),
         message: null,
       });

backend/endpoints/system.js

@@ -61,7 +61,7 @@ function systemEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.get(
@@ -102,7 +102,7 @@ function systemEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.post(
@@ -122,7 +122,7 @@ function systemEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.get("/boot", async (_, response) => {

backend/endpoints/v1/document-processor/index.js

@@ -24,7 +24,7 @@ function documentProcessorEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.get(
@@ -49,7 +49,7 @@ function documentProcessorEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 }
 

backend/endpoints/v1/documents/index.js

@@ -54,7 +54,7 @@ function documentEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.delete(
@@ -84,14 +84,14 @@ function documentEndpoints(app) {
           workspace,
           document,
           connector,
-          user
+          user,
         );
         response.sendStatus(200).end();
       } catch (e) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.get(
@@ -113,7 +113,7 @@ function documentEndpoints(app) {
         const fragments = await DocumentVectors.where(
           { document_id: Number(id) },
           pageSize,
-          offset
+          offset,
         );
 
         const totalFragments = await DocumentVectors.count({
@@ -124,7 +124,7 @@ function documentEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.post(
@@ -175,14 +175,14 @@ function documentEndpoints(app) {
           workspace,
           connector,
           user,
-          newText
+          newText,
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.post(
@@ -224,14 +224,14 @@ function documentEndpoints(app) {
           workspace,
           connector,
           user,
-          newMetadata
+          newMetadata,
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.delete(
@@ -269,14 +269,14 @@ function documentEndpoints(app) {
           workspace,
           organization,
           connector,
-          user
+          user,
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.post(
@@ -316,7 +316,7 @@ function documentEndpoints(app) {
         console.log(e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.post(
@@ -370,14 +370,14 @@ function documentEndpoints(app) {
           workspace,
           document,
           connector,
-          user
+          user,
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
         console.log(e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.get(
@@ -407,14 +407,14 @@ function documentEndpoints(app) {
         const { fragments, error } = await documentEmbeddingSearch(
           document,
           method,
-          query
+          query,
         );
         response.status(200).json({ fragments, error });
       } catch (e) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 }
 

backend/endpoints/v1/index.js

@@ -28,7 +28,7 @@ function v1Endpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   userEndpoints(app);

backend/endpoints/v1/jobs/index.js

@@ -31,7 +31,7 @@ function jobEndpoints(app) {
           existingJob.taskName,
           jobData,
           user.id,
-          existingJob.organization_id
+          existingJob.organization_id,
         );
 
         if (!!error) {
@@ -51,7 +51,7 @@ function jobEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 
   app.delete(
@@ -83,7 +83,7 @@ function jobEndpoints(app) {
         console.log(e.message, e);
         response.sendStatus(500).end();
       }
-    }
+    },
   );
 }